@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.70

package/src/server/state-backend-report.ts

@@ -0,0 +1,185 @@
+import {
+	providerCacheRedisResolutionFromEnv,
+	type ProviderRedisUrlResolution,
+	providerStateRedisResolutionFromEnv,
+} from "../config/loader.js";
+
+/**
+ * What a provider store resolved to at boot.
+ *
+ * `endpoint` is deliberately host and port only: the URL can carry a password
+ * (`redis://:secret@host:6379`), and the diagnostic inventory already treats
+ * those credential components — and only those — as sensitive
+ * (`collectDiagnosticSensitiveValues`, `urlCredentialComponents(url, {
+ * includePathAndQuery: false })`). A URL that does not parse contributes no
+ * endpoint at all rather than a redacted-looking fragment of itself.
+ */
+export type ProviderStoreBackendReport = {
+	/**
+	 * `redis` when a URL resolved, `memory` for the in-process fallback,
+	 * `unsupported` when state has no backend and every call fails closed,
+	 * `injected` when the host handed the server a store object.
+	 */
+	readonly backend: "redis" | "memory" | "unsupported" | "injected";
+	/** Env name that supplied the Redis URL. Present only for `redis`. */
+	readonly envName?: string;
+	/**
+	 * `host:port` of the resolved Redis. Never includes credentials, path or
+	 * query. Absent when the URL does not parse, or when its query carries an
+	 * ioredis option that moves the endpoint (`path`, `port`, `host`, `family`)
+	 * so the authority is not where the client connects — see
+	 * `state_redis_url_ambiguous_endpoint`.
+	 */
+	readonly endpoint?: string;
+	/** URL scheme, e.g. `redis` or `rediss`. Present only for `redis`. */
+	readonly scheme?: string;
+	/** True when the URL came from a shared fallback env rather than the store's own. */
+	readonly fallback?: boolean;
+};
+
+/**
+ * One line per process boot naming the stores the provider resolved. Without
+ * it, a provider silently riding the cache Redis for its state is
+ * indistinguishable from one on a durable instance, from outside the pod and
+ * from the pod's own logs (apifuse#2302, the reason apifuse#2144 could only be
+ * verified by inference from the Deployment spec).
+ */
+export type ProviderStateBackendLogEvent = {
+	/** `warn` when a warning was raised, otherwise `info`. */
+	readonly level: "info" | "warn";
+	readonly event: "provider_state_backend";
+	readonly providerId: string;
+	readonly state: ProviderStoreBackendReport;
+	readonly cache: ProviderStoreBackendReport;
+	readonly sdkVersion: string;
+	/**
+	 * Non-fatal findings, e.g. `state_redis_url_fallback` (the state store is
+	 * sharing an instance provisioned for something else),
+	 * `state_redis_url_unparsed`, or `state_redis_url_ambiguous_endpoint` (the
+	 * URL's query carries an ioredis option that moves the endpoint, so the
+	 * authority is not reported as the endpoint). Absent when the resolution
+	 * was unambiguous.
+	 */
+	readonly warnings?: readonly string[];
+};
+
+/**
+ * Whether ioredis' own URL parsing would move the endpoint away from the
+ * authority this function can see. `createProviderRedisClient` hands the raw
+ * URL to `new Redis(url, …)`, and `parseURL` ends with
+ * `defaults(result, queryOptions)` — lodash `defaults`, so a query key only
+ * applies where the authority supplied nothing:
+ *
+ * - `path` is never derived from the authority of a `redis://` URL (that
+ *   pathname is the db index), so `?path=` always applies — and
+ *   `StandaloneConnector` connects to `options.path` INSTEAD of host/port.
+ *   The client is then on a Unix socket the authority says nothing about.
+ * - `port` applies only when the URL carries no explicit port.
+ * - `host` never applies here: we return early unless `parsed.hostname` is
+ *   set, so `result.host` is always already populated.
+ * - `family` selects an address family; it does not move `host:port`.
+ *
+ * Verified against the pinned ioredis 5.11.1. In those cases the endpoint is
+ * WITHHELD and the ambiguity named, rather than reported as the authority — a
+ * diagnostic whose whole job is to answer "which Redis is this provider on"
+ * must not answer it wrongly. The query is never echoed: it can carry a
+ * password, which is why `endpoint` is host-and-port only to begin with.
+ */
+function queryMovesEndpoint(parsed: URL): boolean {
+	if (parsed.searchParams.has("path")) return true;
+	return parsed.searchParams.has("port") && parsed.port === "";
+}
+
+function redisEndpoint(url: string): {
+	endpoint?: string;
+	scheme?: string;
+	ambiguous?: boolean;
+} {
+	let parsed: URL;
+	try {
+		parsed = new URL(url);
+	} catch {
+		return {};
+	}
+	// No authority means this is not the `redis://host:port` shape at all (e.g.
+	// a bare `host:port`, which parses as a scheme). Report nothing rather than
+	// a fragment of a string whose parts we have not identified.
+	if (!parsed.hostname) return {};
+	const scheme = parsed.protocol.replace(/:$/, "");
+	if (queryMovesEndpoint(parsed)) return { scheme, ambiguous: true };
+	return {
+		scheme,
+		endpoint: parsed.port ? `${parsed.hostname}:${parsed.port}` : parsed.hostname,
+	};
+}
+
+function redisReport(resolution: ProviderRedisUrlResolution): {
+	report: ProviderStoreBackendReport;
+	parsed: boolean;
+	ambiguous: boolean;
+} {
+	const { endpoint, scheme, ambiguous } = redisEndpoint(resolution.url);
+	return {
+		parsed: endpoint !== undefined || ambiguous === true,
+		ambiguous: ambiguous === true,
+		report: {
+			backend: "redis",
+			envName: resolution.envName,
+			...(endpoint === undefined ? {} : { endpoint }),
+			...(scheme === undefined ? {} : { scheme }),
+			...(resolution.fallback ? { fallback: true } : {}),
+		},
+	};
+}
+
+/**
+ * Build the boot event from the same env precedence the stores themselves use
+ * (`providerStateRedisResolutionFromEnv` / `providerCacheRedisResolutionFromEnv`),
+ * so the line can never describe a backend other than the one in use.
+ */
+export function providerStateBackendLogEvent(input: {
+	readonly providerId: string;
+	readonly sdkVersion: string;
+	/** The host passed an explicit state store, so no env resolution happened. */
+	readonly injectedState: boolean;
+	/** Mirrors `createProviderRuntimeStateFromEnv({ allowMemoryFallback })`. */
+	readonly allowMemoryFallback: boolean;
+}): ProviderStateBackendLogEvent {
+	const warnings: string[] = [];
+	let state: ProviderStoreBackendReport;
+	if (input.injectedState) {
+		state = { backend: "injected" };
+	} else {
+		const resolution = providerStateRedisResolutionFromEnv();
+		if (resolution) {
+			const { report, parsed, ambiguous } = redisReport(resolution);
+			state = report;
+			if (!parsed) warnings.push("state_redis_url_unparsed");
+			if (ambiguous) warnings.push("state_redis_url_ambiguous_endpoint");
+			if (resolution.fallback) warnings.push("state_redis_url_fallback");
+		} else {
+			state = { backend: input.allowMemoryFallback ? "memory" : "unsupported" };
+		}
+	}
+
+	const cacheResolution = providerCacheRedisResolutionFromEnv();
+	let cache: ProviderStoreBackendReport;
+	if (cacheResolution) {
+		const { report, parsed, ambiguous } = redisReport(cacheResolution);
+		cache = report;
+		if (!parsed) warnings.push("cache_redis_url_unparsed");
+		if (ambiguous) warnings.push("cache_redis_url_ambiguous_endpoint");
+	} else {
+		cache = { backend: "memory" };
+	}
+
+	return {
+		level: warnings.length > 0 ? "warn" : "info",
+		event: "provider_state_backend",
+		providerId: input.providerId,
+		state,
+		cache,
+		sdkVersion: input.sdkVersion,
+		...(warnings.length === 0 ? {} : { warnings }),
+	};
+}

package/src/server/trace-output.ts

@@ -0,0 +1,189 @@
+import { isSensitiveFixtureKey, REDACTED_FIXTURE_VALUE } from "../fixture-sanitization.js";
+import {
+	type DiagnosticRedactor,
+	redactedKeyAllocator,
+	isRedactedKey,
+	REDACTION_FAILED,
+	redactDiagnosticText,
+} from "../runtime/diagnostic-redactor.js";
+import {
+	OTEL_EXPORTER_OTLP_ENDPOINT,
+	OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+	type OTLPExportOptions,
+	type OTLPExportResolution,
+	type OTLPResourceResolution,
+	resolveOTLPExportOptions,
+	resolveOTLPResourceAttributes,
+} from "../runtime/otlp.js";
+import {
+	type CreateTraceContextOptions,
+	resolveTraceContextOptions,
+	type Span,
+} from "../runtime/trace.js";
+import {
+	sanitizeSpanForOutput,
+	sanitizeSpanNameForOutput,
+	sanitizeTraceAttributes,
+} from "../trace-sanitization.js";
+import type { TraceConfig } from "../types.js";
+
+type EnvLike = Record<string, string | undefined>;
+
+const TRACE_RESOURCE_ATTRIBUTE_SANITIZER = Symbol.for(
+	"@apifuse/provider-sdk/runtime/trace-resource-attribute-sanitizer",
+);
+
+type ResourceAttributesWithSanitizer = Record<string, string> & {
+	[TRACE_RESOURCE_ATTRIBUTE_SANITIZER]?: (
+		attributes: Record<string, string>,
+	) => Record<string, string>;
+};
+
+// Warn once per environment object: process.env in production, each injected env in tests.
+const warnedEnvironments = new WeakSet<EnvLike>();
+const warnedResourceEnvironments = new WeakSet<EnvLike>();
+
+function warnDiscardedResourceAttributes(env: EnvLike, resolution: OTLPResourceResolution): void {
+	if (resolution.discarded.length === 0 || warnedResourceEnvironments.has(env)) return;
+	warnedResourceEnvironments.add(env);
+	console.warn(
+		`[apifuse] ${resolution.discarded.join(", ")} could not be parsed as an OTel key=value list and was ignored.`,
+	);
+}
+
+function warnExportDisabled(
+	env: EnvLike,
+	resolution: Exclude<OTLPExportResolution, { status: "resolved" }>,
+): void {
+	if (warnedEnvironments.has(env)) return;
+	warnedEnvironments.add(env);
+	console.warn(
+		resolution.status === "unconfigured"
+			? `[apifuse] OTLP trace export is enabled but no endpoint is configured; set ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} or ${OTEL_EXPORTER_OTLP_ENDPOINT}. Trace export is disabled.`
+			: `[apifuse] OTLP trace export is enabled but ${resolution.source} ${resolution.reason}. Trace export is disabled.`,
+	);
+}
+
+function resolveServerOTLPExportOptions(
+	config: TraceConfig,
+	env: EnvLike,
+): OTLPExportOptions | undefined {
+	const resolution = resolveOTLPExportOptions(
+		{
+			endpoint: config.otlp?.endpoint ?? config.endpoint,
+			headers: config.otlp?.headers,
+			timeout: config.otlp?.timeout,
+		},
+		env,
+	);
+	if (resolution.status === "resolved") return resolution.options;
+	warnExportDisabled(env, resolution);
+	return undefined;
+}
+
+/**
+ * Per-request attributes carry client-supplied values (request_id) and get the full attribute
+ * sanitizer, exactly as on the console path. Operator-configured attributes from the process
+ * environment are identifiers rather than request data: their values are kept, with control
+ * characters neutralized, length bounded, and secret-named keys redacted.
+ */
+function resolveExportResourceAttributes(
+	requestAttributes: Record<string, string>,
+	env: EnvLike,
+	redact?: DiagnosticRedactor,
+): {
+	attributes: Record<string, string>;
+	sanitize: (attributes: Record<string, string>) => Record<string, string>;
+} {
+	const resolution = resolveOTLPResourceAttributes({}, env);
+	warnDiscardedResourceAttributes(env, resolution);
+	const operatorAttributes = resolution.attributes;
+	const requestKeys = new Set(Object.keys(requestAttributes));
+	return {
+		// Keep the raw snapshot private until export. A credential registered after
+		// trace creation must be matched before any key or value is bounded.
+		attributes: { ...operatorAttributes, ...requestAttributes },
+		sanitize(attributes) {
+			const nextKey = redactedKeyAllocator(Object.keys(attributes));
+			const sanitizedOperatorAttributes: Record<string, string> = {};
+			const rawRequestAttributes: Record<string, string> = {};
+			for (const [key, value] of Object.entries(attributes)) {
+				const unchangedOperatorValue = !requestKeys.has(key) && operatorAttributes[key] === value;
+				if (unchangedOperatorValue) {
+					const redactedKey = isRedactedKey(key) ? key : redactDiagnosticText(key, redact);
+					const keyChanged = redactedKey !== key;
+					const sanitizedKey = keyChanged ? nextKey() : sanitizeSpanNameForOutput(key);
+					sanitizedOperatorAttributes[sanitizedKey] = keyChanged
+						? redactedKey === REDACTION_FAILED
+							? REDACTION_FAILED
+							: REDACTED_FIXTURE_VALUE
+						: isSensitiveFixtureKey(key)
+							? REDACTED_FIXTURE_VALUE
+							: sanitizeSpanNameForOutput(value, redact);
+				} else {
+					rawRequestAttributes[key] = value;
+				}
+			}
+			const sanitizedRequestAttributes = Object.fromEntries(
+				Object.entries(
+					sanitizeTraceAttributes(
+						rawRequestAttributes,
+						redact,
+						Object.keys(sanitizedOperatorAttributes),
+					),
+				).map(([key, value]) => [key, String(value)]),
+			);
+			return { ...sanitizedOperatorAttributes, ...sanitizedRequestAttributes };
+		},
+	};
+}
+
+/** Server-only trace output policy. Shared programmatic trace callers stay in-memory. */
+export function resolveServerTraceContextOptions(
+	config: TraceConfig,
+	resourceAttributes: Record<string, string>,
+	env: EnvLike = process.env,
+	redact?: DiagnosticRedactor,
+): CreateTraceContextOptions {
+	const resolved = resolveTraceContextOptions(config);
+	const outputEnabled = config.enabled !== false && config.exporter !== "none";
+	const consoleHook =
+		outputEnabled && (config.exporter === "console" || config.exporter === "json")
+			? (span: Span) =>
+					console.log(JSON.stringify(sanitizeSpanForOutput(span, resourceAttributes, redact)))
+			: undefined;
+	const onSpan =
+		consoleHook && resolved.onSpan
+			? (span: Span) => {
+					consoleHook(span);
+					resolved.onSpan?.(span);
+				}
+			: (consoleHook ?? resolved.onSpan);
+	const exportOptions =
+		outputEnabled && config.exporter === "otlp"
+			? resolveServerOTLPExportOptions(config, env)
+			: undefined;
+	const exportResources = exportOptions
+		? resolveExportResourceAttributes(resourceAttributes, env, redact)
+		: undefined;
+	if (exportResources) {
+		Object.defineProperty(
+			exportResources.attributes as ResourceAttributesWithSanitizer,
+			TRACE_RESOURCE_ATTRIBUTE_SANITIZER,
+			{ value: exportResources.sanitize },
+		);
+	}
+
+	return {
+		maxSpans: resolved.maxSpans,
+		onSpan,
+		...(redact ? { redact } : {}),
+		...(exportOptions
+			? {
+					exportOptions,
+					resourceAttributes: exportResources?.attributes,
+					sanitizeSpanForExport: (span: Span) => sanitizeSpanForOutput(span, undefined, redact),
+				}
+			: {}),
+	};
+}

package/src/server/types.ts

@@ -1,8 +1,15 @@
 import { z } from "zod";
+import { PROVIDER_ERROR_SOURCES } from "../observability.js";
 
 import { HttpRetryPreset } from "../types.js";
 
-export const ConnectionModeSchema = z.enum(["oauth2", "credentials", "platform-managed", "none"]);
+export const ConnectionModeSchema = z.enum([
+	"oauth2",
+	"oauth2_proxied",
+	"credentials",
+	"platform-managed",
+	"none",
+]);
 
 export const OperationConnectionSchema = z.object({
 	id: z.string(),
@@ -18,6 +25,13 @@ export const OperationRequestSchema = z.object({
 	input: z.record(z.string(), z.unknown()),
 	connectionId: z.string().optional(),
 	connection: OperationConnectionSchema.optional(),
+	/**
+	 * Gateway-asserted principal scope: the organization id for customer
+	 * subjects, the service-account id for platform service accounts. Absent
+	 * on direct calls and self-test. Trusted exactly like `connectionId`; it is
+	 * never derived from caller headers or `input`.
+	 */
+	tenantId: z.string().optional(),
 	headers: z.record(z.string(), z.string()).optional(),
 	trace: z.record(z.string(), z.string()).optional(),
 });
@@ -26,6 +40,11 @@ export const ErrorEnvelopeSchema = z.object({
 	code: z.string(),
 	message: z.string(),
 	requestId: z.string().optional(),
+	retryable: z.boolean(),
+	// Who stopped the request (honest-provider-error-contract); optional so
+	// older peers (stateful forwarding owners) remain parseable. Derived from
+	// the PROVIDER_ERROR_SOURCES registry so the enum cannot drift.
+	source: z.enum(PROVIDER_ERROR_SOURCES).optional(),
 	fix: z.string().optional(),
 	details: z.unknown().optional(),
 });
@@ -81,12 +100,26 @@ export const AuthFlowRequestSchema = z.object({
 	headers: z.record(z.string(), z.string()).optional(),
 	input: z.record(z.string(), z.unknown()).optional(),
 	context: z.record(z.string(), z.unknown()).optional(),
+	/** Opaque engine state. This is never projected into FlowContext or provider contextPatch. */
+	engine: z
+		.object({
+			egressLease: z.string().min(1).max(16_384).optional(),
+		})
+		.strict()
+		.optional(),
 	connection: OperationConnectionSchema.optional(),
 });
 
 export const AuthFlowSuccessResponseSchema = z.object({
 	data: z.unknown(),
 	contextPatch: z.record(z.string(), z.unknown().nullable()).optional(),
+	/** Opaque engine state to carry verbatim to the next auth turn. */
+	engine: z
+		.object({
+			egressLease: z.string().min(1).max(16_384).optional(),
+		})
+		.strict()
+		.optional(),
 });
 
 export const AuthFlowErrorResponseSchema = OperationErrorResponseSchema;

package/src/stateful/README.md

@@ -0,0 +1,146 @@
+# Stateful module compatibility note
+
+The `@apifuse/provider-sdk/stateful` entry point is new and unreleased. Its initial contract includes
+the following compatibility-sensitive behavior:
+
+- Session identifiers use the branded `SessionKey` returned by `buildSessionKey`.
+  `parseSessionKey` returns `SessionKeyParts`; validate and brand an external string with
+  `buildSessionKey(parseSessionKey(value))`. Registry inputs, routing requests, event owner fences,
+  and release APIs no longer accept arbitrary strings. A decoded `SessionOwnerRecord.sessionKey`
+  remains a string and must be checked where external data enters trusted SDK code (a trust
+  boundary).
+- Adapters declare `policy.concurrency` (`serialize` or bounded/unbounded `parallel`) and
+  `policy.reconnect` (`resume`, `recreate`, or `unsupported`). `restore`, `snapshot`, and `health`
+  are optional; a declared `snapshot` must be paired with the manager's `checkpointStore`.
+- Code in the process hosting the adapter (the provider pod) publishes through
+  `ProviderEventPublisher`; persistent services that survive provider-pod restarts (durable
+  platform code) implement `ProviderEventPipeline`. An adapter's
+  `subscribe(ctx, session, publish)` callback accepts only an event. The session manager, not the
+  adapter, binds the registry's current write authorization (the authoritative
+  `{ sessionKey, generation, ownerPodId, ownerEndpoint }` fence) before calling the publisher.
+  Adapters must not construct or cache owner fences themselves.
+- `subscribe` must return a disposer. The manager runs it before checkpointing and closing a
+  session. `health()` and `reconcileWrite()` use the same per-session concurrency scheduler and
+  optional ownership validation semantics as invocation.
+- `HttpProviderEventEmitter.flush(timeoutMs)` resolves to
+  `{ delivered: number, failed: number, pending: number }`. Counts are cumulative for the emitter's
+  lifetime; `pending` is the buffer remaining when the flush returns.
+- `serve()` resolves to a `ProviderServerHandle` with `port` and idempotent `close({ timeoutMs? })`.
+  Close stops listeners, runs shutdown hooks in declaration order, and enforces the configured
+  timeout. Signal-enabled handles share one process handler per signal; a signal closes every
+  registered server before it is re-raised. Closing the last handle removes the shared handlers.
+- `ProviderServerStatefulForwardEnvelope` is strict. Its owner fence is flattened into
+  `sessionKey`, `ownerPodId`, and `generation`; a nested `owner` object is not accepted. The
+  envelope also carries operation/session identity, `sourcePodId`, `forwardedAt`, optional
+  `idempotencyKey`, optional ISO `deadlineAt`, and the nested `operationRequest`. Unknown or
+  malformed fields fail closed, and the owner-side executor receives `deadlineAt` on its request.
+- Stateful forwarding is configured as a pair: `internalOperationExecutor` requires
+  `statefulForwarding`, and `statefulForwarding` requires an `internalOperationExecutor`.
+  `statefulForwarding` supplies the signing secret and `validateOwnerFence`; it may also set
+  `maxSkewMs` and `replayCacheMaxEntries`. Replay-cache capacity returns HTTP 503 with
+  `Retry-After`; fail-closed nonce handling means invalid, replayed, or untrackable nonces are
+  rejected instead of allowing an unverifiable request.
+- Stateful signatures bind the method, route, timestamp, nonce, and exact body. Forwarding also
+  rejects provider/source mismatches, stale owner fences, nonce replay, and request-scoped files.
+
+## Minimal composition
+
+Inside the module that defines your `provider`, the following wires an adapter, session manager,
+router, signed owner forwarding, and graceful shutdown. Replace the environment variable names and
+the service-account mapping with your deployment's values.
+
+```ts
+import {
+	buildSessionKey,
+	createStatefulOwnerFenceValidator,
+	HttpSessionOwnerRegistry,
+	HttpStatefulOwnerForwarder,
+	parseSessionKey,
+	statefulForwardingContextFromProviderRequest,
+	StatefulProviderSessionManager,
+	StatefulSessionRouter,
+	type StatefulProviderAdapter,
+	withStatefulLocalProviderContext,
+} from "@apifuse/provider-sdk/stateful";
+import { serve, type ProviderServerOperationExecutor } from "@apifuse/provider-sdk/server";
+
+type Session = { closed: boolean };
+
+const adapter = {
+	providerId: provider.id,
+	policy: { concurrency: { mode: "serialize" }, reconnect: "recreate" },
+	connect: async () => ({ closed: false }),
+	invoke: async (_ctx, _session, request) => ({ output: request.input }),
+	close: async (_ctx, session) => {
+		session.closed = true;
+	},
+} satisfies StatefulProviderAdapter<Session>;
+
+const registry = new HttpSessionOwnerRegistry({
+	baseUrl: process.env.STATEFUL_CONTROL_PLANE_URL as string,
+	secret: process.env.STATEFUL_CONTROL_PLANE_SECRET as string,
+});
+const manager = new StatefulProviderSessionManager({
+	adapter,
+	poolPolicy: { maxSessions: 100, idleTimeoutMs: "unlimited", maxLifetimeMs: "unlimited" },
+});
+const currentPod = {
+	podId: process.env.POD_ID as string,
+	endpoint: process.env.POD_ENDPOINT as string,
+};
+const router = new StatefulSessionRouter({
+	currentPod,
+	registry,
+	forwarder: new HttpStatefulOwnerForwarder({
+		currentPodId: currentPod.podId,
+		secret: process.env.STATEFUL_FORWARDING_SECRET as string,
+	}),
+	executor: manager,
+	leaseDurationMs: 30_000,
+});
+
+const executeStateful: ProviderServerOperationExecutor = async ({
+	ctx,
+	request,
+	operationId,
+	internalStatefulForward,
+}) => {
+	const keyParts = internalStatefulForward
+		? parseSessionKey(internalStatefulForward.sessionKey)
+		: {
+				providerId: provider.id,
+				serviceAccountId: request.connection?.externalRef as string,
+				connectionId: (request.connection?.id ?? request.connectionId) as string,
+			};
+	const result = await router.route({
+		requestId: request.requestId,
+		sessionKey: buildSessionKey(keyParts),
+		providerId: provider.id,
+		operationId,
+		connectionId: keyParts.connectionId,
+		serviceAccountId: keyParts.serviceAccountId,
+		input: request.input,
+		...(request.deadlineAt ? { deadlineAt: request.deadlineAt } : {}),
+		runtimeContext: withStatefulLocalProviderContext(
+			ctx,
+			statefulForwardingContextFromProviderRequest(request),
+		),
+	});
+	return result.output;
+};
+
+await serve(provider, {
+	operationExecutor: executeStateful,
+	internalOperationExecutor: executeStateful,
+	statefulForwarding: {
+		secret: process.env.STATEFUL_FORWARDING_SECRET as string,
+		validateOwnerFence: createStatefulOwnerFenceValidator(registry),
+	},
+	shutdown: {
+		hooks: [
+			() => manager.closeAll("server-shutdown"),
+			() => router.release(),
+		],
+	},
+});
+```

package/src/stateful/errors.ts

@@ -0,0 +1,35 @@
+export type StatefulControlPlaneOperation = "resolve" | "acquire" | "renew" | "release";
+
+export class StatefulRoutingDeadlineError extends Error {
+	readonly requestId: string;
+	readonly deadlineAt: string;
+
+	constructor(requestId: string, deadlineAt: string) {
+		super(`Stateful operation request ${requestId} deadline has expired.`);
+		this.name = "StatefulRoutingDeadlineError";
+		this.requestId = requestId;
+		this.deadlineAt = deadlineAt;
+	}
+}
+
+export class StatefulControlPlaneError extends Error {
+	readonly code: string;
+	readonly operation: StatefulControlPlaneOperation;
+	readonly status?: number;
+	readonly cause?: unknown;
+
+	constructor(input: {
+		readonly code: string;
+		readonly message: string;
+		readonly operation: StatefulControlPlaneOperation;
+		readonly status?: number;
+		readonly cause?: unknown;
+	}) {
+		super(input.message);
+		this.name = "StatefulControlPlaneError";
+		this.code = input.code;
+		this.operation = input.operation;
+		this.status = input.status;
+		this.cause = input.cause;
+	}
+}