@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.51

package/src/server/trace-output.ts

@@ -0,0 +1,129 @@
+import { isSensitiveFixtureKey, REDACTED_FIXTURE_VALUE } from "../fixture-sanitization.js";
+import {
+	OTEL_EXPORTER_OTLP_ENDPOINT,
+	OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+	type OTLPExportOptions,
+	type OTLPExportResolution,
+	type OTLPResourceResolution,
+	resolveOTLPExportOptions,
+	resolveOTLPResourceAttributes,
+} from "../runtime/otlp.js";
+import {
+	type CreateTraceContextOptions,
+	resolveTraceContextOptions,
+	type Span,
+} from "../runtime/trace.js";
+import {
+	sanitizeSpanForOutput,
+	sanitizeSpanNameForOutput,
+	sanitizeTraceAttributes,
+} from "../trace-sanitization.js";
+import type { TraceConfig } from "../types.js";
+
+type EnvLike = Record<string, string | undefined>;
+
+// Warn once per environment object: process.env in production, each injected env in tests.
+const warnedEnvironments = new WeakSet<EnvLike>();
+const warnedResourceEnvironments = new WeakSet<EnvLike>();
+
+function warnDiscardedResourceAttributes(env: EnvLike, resolution: OTLPResourceResolution): void {
+	if (resolution.discarded.length === 0 || warnedResourceEnvironments.has(env)) return;
+	warnedResourceEnvironments.add(env);
+	console.warn(
+		`[apifuse] ${resolution.discarded.join(", ")} could not be parsed as an OTel key=value list and was ignored.`,
+	);
+}
+
+function warnExportDisabled(
+	env: EnvLike,
+	resolution: Exclude<OTLPExportResolution, { status: "resolved" }>,
+): void {
+	if (warnedEnvironments.has(env)) return;
+	warnedEnvironments.add(env);
+	console.warn(
+		resolution.status === "unconfigured"
+			? `[apifuse] OTLP trace export is enabled but no endpoint is configured; set ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} or ${OTEL_EXPORTER_OTLP_ENDPOINT}. Trace export is disabled.`
+			: `[apifuse] OTLP trace export is enabled but ${resolution.source} ${resolution.reason}. Trace export is disabled.`,
+	);
+}
+
+function resolveServerOTLPExportOptions(
+	config: TraceConfig,
+	env: EnvLike,
+): OTLPExportOptions | undefined {
+	const resolution = resolveOTLPExportOptions(
+		{
+			endpoint: config.otlp?.endpoint ?? config.endpoint,
+			headers: config.otlp?.headers,
+			timeout: config.otlp?.timeout,
+		},
+		env,
+	);
+	if (resolution.status === "resolved") return resolution.options;
+	warnExportDisabled(env, resolution);
+	return undefined;
+}
+
+/**
+ * Per-request attributes carry client-supplied values (request_id) and get the full attribute
+ * sanitizer, exactly as on the console path. Operator-configured attributes from the process
+ * environment are identifiers rather than request data: their values are kept, with control
+ * characters neutralized, length bounded, and secret-named keys redacted.
+ */
+function resolveExportResourceAttributes(
+	requestAttributes: Record<string, string>,
+	env: EnvLike,
+): Record<string, string> {
+	const resolution = resolveOTLPResourceAttributes({}, env);
+	warnDiscardedResourceAttributes(env, resolution);
+	const operatorAttributes = Object.fromEntries(
+		Object.entries(resolution.attributes).map(([key, value]) => [
+			sanitizeSpanNameForOutput(key),
+			isSensitiveFixtureKey(key) ? REDACTED_FIXTURE_VALUE : sanitizeSpanNameForOutput(value),
+		]),
+	);
+	const sanitizedRequestAttributes = Object.fromEntries(
+		Object.entries(sanitizeTraceAttributes(requestAttributes)).map(([key, value]) => [
+			key,
+			String(value),
+		]),
+	);
+	return { ...operatorAttributes, ...sanitizedRequestAttributes };
+}
+
+/** Server-only trace output policy. Shared programmatic trace callers stay in-memory. */
+export function resolveServerTraceContextOptions(
+	config: TraceConfig,
+	resourceAttributes: Record<string, string>,
+	env: EnvLike = process.env,
+): CreateTraceContextOptions {
+	const resolved = resolveTraceContextOptions(config);
+	const outputEnabled = config.enabled !== false && config.exporter !== "none";
+	const consoleHook =
+		outputEnabled && (config.exporter === "console" || config.exporter === "json")
+			? (span: Span) => console.log(JSON.stringify(sanitizeSpanForOutput(span, resourceAttributes)))
+			: undefined;
+	const onSpan =
+		consoleHook && resolved.onSpan
+			? (span: Span) => {
+					consoleHook(span);
+					resolved.onSpan?.(span);
+				}
+			: (consoleHook ?? resolved.onSpan);
+	const exportOptions =
+		outputEnabled && config.exporter === "otlp"
+			? resolveServerOTLPExportOptions(config, env)
+			: undefined;
+
+	return {
+		maxSpans: resolved.maxSpans,
+		onSpan,
+		...(exportOptions
+			? {
+					exportOptions,
+					resourceAttributes: resolveExportResourceAttributes(resourceAttributes, env),
+					sanitizeSpanForExport: (span: Span) => sanitizeSpanForOutput(span),
+				}
+			: {}),
+	};
+}

package/src/server/types.ts

@@ -1,8 +1,15 @@
 import { z } from "zod";
+import { PROVIDER_ERROR_SOURCES } from "../observability.js";
 
 import { HttpRetryPreset } from "../types.js";
 
-export const ConnectionModeSchema = z.enum(["oauth2", "credentials", "platform-managed", "none"]);
+export const ConnectionModeSchema = z.enum([
+	"oauth2",
+	"oauth2_proxied",
+	"credentials",
+	"platform-managed",
+	"none",
+]);
 
 export const OperationConnectionSchema = z.object({
 	id: z.string(),
@@ -26,6 +33,11 @@ export const ErrorEnvelopeSchema = z.object({
 	code: z.string(),
 	message: z.string(),
 	requestId: z.string().optional(),
+	retryable: z.boolean(),
+	// Who stopped the request (honest-provider-error-contract); optional so
+	// older peers (stateful forwarding owners) remain parseable. Derived from
+	// the PROVIDER_ERROR_SOURCES registry so the enum cannot drift.
+	source: z.enum(PROVIDER_ERROR_SOURCES).optional(),
 	fix: z.string().optional(),
 	details: z.unknown().optional(),
 });

package/src/stateful/README.md

@@ -0,0 +1,146 @@
+# Stateful module compatibility note
+
+The `@apifuse/provider-sdk/stateful` entry point is new and unreleased. Its initial contract includes
+the following compatibility-sensitive behavior:
+
+- Session identifiers use the branded `SessionKey` returned by `buildSessionKey`.
+  `parseSessionKey` returns `SessionKeyParts`; validate and brand an external string with
+  `buildSessionKey(parseSessionKey(value))`. Registry inputs, routing requests, event owner fences,
+  and release APIs no longer accept arbitrary strings. A decoded `SessionOwnerRecord.sessionKey`
+  remains a string and must be checked where external data enters trusted SDK code (a trust
+  boundary).
+- Adapters declare `policy.concurrency` (`serialize` or bounded/unbounded `parallel`) and
+  `policy.reconnect` (`resume`, `recreate`, or `unsupported`). `restore`, `snapshot`, and `health`
+  are optional; a declared `snapshot` must be paired with the manager's `checkpointStore`.
+- Code in the process hosting the adapter (the provider pod) publishes through
+  `ProviderEventPublisher`; persistent services that survive provider-pod restarts (durable
+  platform code) implement `ProviderEventPipeline`. An adapter's
+  `subscribe(ctx, session, publish)` callback accepts only an event. The session manager, not the
+  adapter, binds the registry's current write authorization (the authoritative
+  `{ sessionKey, generation, ownerPodId, ownerEndpoint }` fence) before calling the publisher.
+  Adapters must not construct or cache owner fences themselves.
+- `subscribe` must return a disposer. The manager runs it before checkpointing and closing a
+  session. `health()` and `reconcileWrite()` use the same per-session concurrency scheduler and
+  optional ownership validation semantics as invocation.
+- `HttpProviderEventEmitter.flush(timeoutMs)` resolves to
+  `{ delivered: number, failed: number, pending: number }`. Counts are cumulative for the emitter's
+  lifetime; `pending` is the buffer remaining when the flush returns.
+- `serve()` resolves to a `ProviderServerHandle` with `port` and idempotent `close({ timeoutMs? })`.
+  Close stops listeners, runs shutdown hooks in declaration order, and enforces the configured
+  timeout. Signal-enabled handles share one process handler per signal; a signal closes every
+  registered server before it is re-raised. Closing the last handle removes the shared handlers.
+- `ProviderServerStatefulForwardEnvelope` is strict. Its owner fence is flattened into
+  `sessionKey`, `ownerPodId`, and `generation`; a nested `owner` object is not accepted. The
+  envelope also carries operation/session identity, `sourcePodId`, `forwardedAt`, optional
+  `idempotencyKey`, optional ISO `deadlineAt`, and the nested `operationRequest`. Unknown or
+  malformed fields fail closed, and the owner-side executor receives `deadlineAt` on its request.
+- Stateful forwarding is configured as a pair: `internalOperationExecutor` requires
+  `statefulForwarding`, and `statefulForwarding` requires an `internalOperationExecutor`.
+  `statefulForwarding` supplies the signing secret and `validateOwnerFence`; it may also set
+  `maxSkewMs` and `replayCacheMaxEntries`. Replay-cache capacity returns HTTP 503 with
+  `Retry-After`; fail-closed nonce handling means invalid, replayed, or untrackable nonces are
+  rejected instead of allowing an unverifiable request.
+- Stateful signatures bind the method, route, timestamp, nonce, and exact body. Forwarding also
+  rejects provider/source mismatches, stale owner fences, nonce replay, and request-scoped files.
+
+## Minimal composition
+
+Inside the module that defines your `provider`, the following wires an adapter, session manager,
+router, signed owner forwarding, and graceful shutdown. Replace the environment variable names and
+the service-account mapping with your deployment's values.
+
+```ts
+import {
+	buildSessionKey,
+	createStatefulOwnerFenceValidator,
+	HttpSessionOwnerRegistry,
+	HttpStatefulOwnerForwarder,
+	parseSessionKey,
+	statefulForwardingContextFromProviderRequest,
+	StatefulProviderSessionManager,
+	StatefulSessionRouter,
+	type StatefulProviderAdapter,
+	withStatefulLocalProviderContext,
+} from "@apifuse/provider-sdk/stateful";
+import { serve, type ProviderServerOperationExecutor } from "@apifuse/provider-sdk/server";
+
+type Session = { closed: boolean };
+
+const adapter = {
+	providerId: provider.id,
+	policy: { concurrency: { mode: "serialize" }, reconnect: "recreate" },
+	connect: async () => ({ closed: false }),
+	invoke: async (_ctx, _session, request) => ({ output: request.input }),
+	close: async (_ctx, session) => {
+		session.closed = true;
+	},
+} satisfies StatefulProviderAdapter<Session>;
+
+const registry = new HttpSessionOwnerRegistry({
+	baseUrl: process.env.STATEFUL_CONTROL_PLANE_URL as string,
+	secret: process.env.STATEFUL_CONTROL_PLANE_SECRET as string,
+});
+const manager = new StatefulProviderSessionManager({
+	adapter,
+	poolPolicy: { maxSessions: 100, idleTimeoutMs: "unlimited", maxLifetimeMs: "unlimited" },
+});
+const currentPod = {
+	podId: process.env.POD_ID as string,
+	endpoint: process.env.POD_ENDPOINT as string,
+};
+const router = new StatefulSessionRouter({
+	currentPod,
+	registry,
+	forwarder: new HttpStatefulOwnerForwarder({
+		currentPodId: currentPod.podId,
+		secret: process.env.STATEFUL_FORWARDING_SECRET as string,
+	}),
+	executor: manager,
+	leaseDurationMs: 30_000,
+});
+
+const executeStateful: ProviderServerOperationExecutor = async ({
+	ctx,
+	request,
+	operationId,
+	internalStatefulForward,
+}) => {
+	const keyParts = internalStatefulForward
+		? parseSessionKey(internalStatefulForward.sessionKey)
+		: {
+				providerId: provider.id,
+				serviceAccountId: request.connection?.externalRef as string,
+				connectionId: (request.connection?.id ?? request.connectionId) as string,
+			};
+	const result = await router.route({
+		requestId: request.requestId,
+		sessionKey: buildSessionKey(keyParts),
+		providerId: provider.id,
+		operationId,
+		connectionId: keyParts.connectionId,
+		serviceAccountId: keyParts.serviceAccountId,
+		input: request.input,
+		...(request.deadlineAt ? { deadlineAt: request.deadlineAt } : {}),
+		runtimeContext: withStatefulLocalProviderContext(
+			ctx,
+			statefulForwardingContextFromProviderRequest(request),
+		),
+	});
+	return result.output;
+};
+
+await serve(provider, {
+	operationExecutor: executeStateful,
+	internalOperationExecutor: executeStateful,
+	statefulForwarding: {
+		secret: process.env.STATEFUL_FORWARDING_SECRET as string,
+		validateOwnerFence: createStatefulOwnerFenceValidator(registry),
+	},
+	shutdown: {
+		hooks: [
+			() => manager.closeAll("server-shutdown"),
+			() => router.release(),
+		],
+	},
+});
+```

package/src/stateful/errors.ts

@@ -0,0 +1,35 @@
+export type StatefulControlPlaneOperation = "resolve" | "acquire" | "renew" | "release";
+
+export class StatefulRoutingDeadlineError extends Error {
+	readonly requestId: string;
+	readonly deadlineAt: string;
+
+	constructor(requestId: string, deadlineAt: string) {
+		super(`Stateful operation request ${requestId} deadline has expired.`);
+		this.name = "StatefulRoutingDeadlineError";
+		this.requestId = requestId;
+		this.deadlineAt = deadlineAt;
+	}
+}
+
+export class StatefulControlPlaneError extends Error {
+	readonly code: string;
+	readonly operation: StatefulControlPlaneOperation;
+	readonly status?: number;
+	readonly cause?: unknown;
+
+	constructor(input: {
+		readonly code: string;
+		readonly message: string;
+		readonly operation: StatefulControlPlaneOperation;
+		readonly status?: number;
+		readonly cause?: unknown;
+	}) {
+		super(input.message);
+		this.name = "StatefulControlPlaneError";
+		this.code = input.code;
+		this.operation = input.operation;
+		this.status = input.status;
+		this.cause = input.cause;
+	}
+}

package/src/stateful/http-provider-event-emitter.ts

@@ -0,0 +1,314 @@
+import { statefulSignedHeaders } from "../stateful-signing.js";
+import type {
+	ProviderEventDeliveryFailureReason,
+	ProviderEventDeliveryFailureRecorder,
+} from "./provider-event-delivery-failures.js";
+import type {
+	ProviderEventPublisher,
+	ProviderEventPublishOptions,
+	PublishAck,
+} from "./provider-event-pipeline.js";
+import {
+	NOOP_PROVIDER_EVENT_METRIC_EMITTER,
+	providerEventMetricLabels,
+	type ProviderEventMetricEmitter,
+} from "./provider-event-pipeline-metrics.js";
+import type { ProviderEvent } from "./provider-events.js";
+
+type FetchTransport = (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export type HttpProviderEventEmitterOptions = {
+	readonly baseUrl: string;
+	readonly secret: string;
+	readonly fetch?: FetchTransport;
+	readonly clock?: () => Date;
+	readonly random?: () => number;
+	readonly metricEmitter?: ProviderEventMetricEmitter;
+	readonly failureRecorder?: ProviderEventDeliveryFailureRecorder;
+	readonly maxBufferedEvents?: number;
+	readonly retryBaseMs?: number;
+	readonly retryMaxMs?: number;
+	readonly maxAttempts?: number;
+	readonly jitterRatio?: number;
+};
+
+type BufferedEvent = {
+	readonly event: ProviderEvent;
+	readonly rawBody: string;
+	readonly idempotencyKey: string;
+	attempts: number;
+	dropped: boolean;
+	abortController?: AbortController;
+};
+
+const DEFAULT_MAX_BUFFERED_EVENTS = 1_000;
+const DEFAULT_RETRY_BASE_MS = 500;
+const DEFAULT_RETRY_MAX_MS = 30_000;
+const DEFAULT_MAX_ATTEMPTS = 10;
+const DEFAULT_JITTER_RATIO = 0.2;
+
+export type ProviderEventFlushReport = {
+	/** Events delivered during this emitter's lifetime. */
+	readonly delivered: number;
+	/** Events permanently dropped during this emitter's lifetime. */
+	readonly failed: number;
+	/** Events still buffered when flush returned. */
+	readonly pending: number;
+};
+
+const CONSOLE_PROVIDER_EVENT_DELIVERY_FAILURE_RECORDER = {
+	record(failure) {
+		console.error(
+			JSON.stringify({
+				event: "provider_event_delivery_failed",
+				eventId: failure.eventId,
+				reason: failure.reason,
+				attempts: failure.attempts,
+			}),
+		);
+	},
+} satisfies ProviderEventDeliveryFailureRecorder;
+
+/**
+ * Provider-pod HTTP publication with a bounded best-effort delivery guarantee and at-least-once
+ * transport retries. This is not durable at-least-once delivery: buffered events can be lost if
+ * the process is killed. Graceful shutdown should drain the in-memory buffer with flush().
+ */
+export class HttpProviderEventEmitter implements ProviderEventPublisher {
+	readonly #baseUrl: string;
+	readonly #secret: string;
+	readonly #fetch: FetchTransport;
+	readonly #clock: () => Date;
+	readonly #random: () => number;
+	readonly #metricEmitter: ProviderEventMetricEmitter;
+	readonly #failureRecorder: ProviderEventDeliveryFailureRecorder;
+	readonly #maxBufferedEvents: number;
+	readonly #retryBaseMs: number;
+	readonly #retryMaxMs: number;
+	readonly #maxAttempts: number;
+	readonly #jitterRatio: number;
+	readonly #queue: BufferedEvent[] = [];
+	#drainPromise?: Promise<void>;
+	#deliveredCount = 0;
+	#failedCount = 0;
+
+	constructor(options: HttpProviderEventEmitterOptions) {
+		if (options.baseUrl.trim().length === 0) {
+			throw new Error("Provider event ingest baseUrl is required.");
+		}
+		if (options.secret.trim().length === 0) {
+			throw new Error("Provider event ingest secret is required.");
+		}
+		this.#baseUrl = options.baseUrl.replace(/\/+$/, "");
+		this.#secret = options.secret;
+		this.#fetch = options.fetch ?? globalThis.fetch;
+		this.#clock = options.clock ?? (() => new Date());
+		this.#random = options.random ?? Math.random;
+		this.#metricEmitter = options.metricEmitter ?? NOOP_PROVIDER_EVENT_METRIC_EMITTER;
+		this.#failureRecorder =
+			options.failureRecorder ?? CONSOLE_PROVIDER_EVENT_DELIVERY_FAILURE_RECORDER;
+		this.#maxBufferedEvents = positiveInteger(
+			options.maxBufferedEvents ?? DEFAULT_MAX_BUFFERED_EVENTS,
+			"maxBufferedEvents",
+		);
+		this.#retryBaseMs = nonnegativeFinite(
+			options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
+			"retryBaseMs",
+		);
+		this.#retryMaxMs = nonnegativeFinite(options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS, "retryMaxMs");
+		this.#maxAttempts = positiveInteger(options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS, "maxAttempts");
+		this.#jitterRatio = boundedRatio(options.jitterRatio ?? DEFAULT_JITTER_RATIO, "jitterRatio");
+		if (this.#retryMaxMs < this.#retryBaseMs) {
+			throw new Error("Provider event retryMaxMs must be greater than or equal to retryBaseMs.");
+		}
+	}
+
+	publish(event: ProviderEvent, options: ProviderEventPublishOptions): PublishAck {
+		const accepted = this.#enqueue(event, options);
+		return { accepted, queued: this.#queue.length };
+	}
+
+	pendingCount(): number {
+		return this.#queue.length;
+	}
+
+	async flush(timeoutMs = 30_000): Promise<ProviderEventFlushReport> {
+		nonnegativeFinite(timeoutMs, "flush timeout");
+		this.#ensureDrain();
+		if (this.#queue.length === 0 || timeoutMs === 0) return this.#flushReport();
+		let timeout: ReturnType<typeof setTimeout> | undefined;
+		const timedOut = new Promise<void>((resolve) => {
+			timeout = setTimeout(resolve, timeoutMs);
+		});
+		await Promise.race([this.#drainPromise ?? Promise.resolve(), timedOut]);
+		if (timeout) clearTimeout(timeout);
+		return this.#flushReport();
+	}
+
+	#flushReport(): ProviderEventFlushReport {
+		return {
+			delivered: this.#deliveredCount,
+			failed: this.#failedCount,
+			pending: this.#queue.length,
+		};
+	}
+
+	#enqueue(event: ProviderEvent, options: ProviderEventPublishOptions): boolean {
+		let rawBody: string;
+		try {
+			rawBody = JSON.stringify({
+				...event,
+				ownerFence: options.ownerFence,
+			});
+		} catch (error) {
+			this.#recordFailure(event, "attempts_exhausted", 0, error);
+			return false;
+		}
+		if (this.#queue.length >= this.#maxBufferedEvents) {
+			const oldest = this.#queue.shift();
+			if (oldest) {
+				oldest.dropped = true;
+				oldest.abortController?.abort();
+				this.#recordFailure(oldest.event, "buffer_overflow", oldest.attempts);
+			}
+		}
+		this.#queue.push({
+			event,
+			rawBody,
+			idempotencyKey: options.idempotencyKey ?? event.eventId,
+			attempts: 0,
+			dropped: false,
+		});
+		this.#ensureDrain();
+		return true;
+	}
+
+	#ensureDrain(): void {
+		if (this.#drainPromise || this.#queue.length === 0) return;
+		this.#drainPromise = Promise.resolve()
+			.then(() => this.#drain())
+			.finally(() => {
+				this.#drainPromise = undefined;
+				if (this.#queue.length > 0) this.#ensureDrain();
+			});
+	}
+
+	async #drain(): Promise<void> {
+		while (this.#queue.length > 0) {
+			const current = this.#queue[0];
+			if (!current) return;
+			current.attempts += 1;
+			const delivered = await this.#send(current);
+			if (current.dropped) continue;
+			if (delivered) {
+				if (this.#queue[0] === current) this.#queue.shift();
+				this.#deliveredCount += 1;
+				continue;
+			}
+			if (current.attempts >= this.#maxAttempts) {
+				if (this.#queue[0] === current) this.#queue.shift();
+				this.#recordFailure(current.event, "attempts_exhausted", current.attempts);
+				continue;
+			}
+			await delay(this.#retryDelay(current.attempts));
+		}
+	}
+
+	async #send(buffered: BufferedEvent): Promise<boolean> {
+		const timestamp = this.#clock().toISOString();
+		const path = "/v1/stateful/events";
+		const abortController = new AbortController();
+		buffered.abortController = abortController;
+		try {
+			const response = await this.#fetch(`${this.#baseUrl}${path}`, {
+				method: "POST",
+				headers: {
+					"content-type": "application/json",
+					"x-apifuse-event-id": buffered.idempotencyKey,
+					...statefulSignedHeaders({
+						secret: this.#secret,
+						timestamp,
+						rawBody: buffered.rawBody,
+						method: "POST",
+						path,
+					}),
+				},
+				body: buffered.rawBody,
+				signal: abortController.signal,
+			});
+			return response.ok;
+		} catch {
+			return false;
+		} finally {
+			if (buffered.abortController === abortController) {
+				buffered.abortController = undefined;
+			}
+		}
+	}
+
+	#retryDelay(attempts: number): number {
+		const exponential = Math.min(
+			this.#retryMaxMs,
+			this.#retryBaseMs * 2 ** Math.max(0, attempts - 1),
+		);
+		const jitter = 1 + (this.#random() * 2 - 1) * this.#jitterRatio;
+		return Math.max(0, Math.round(exponential * jitter));
+	}
+
+	#recordFailure(
+		event: ProviderEvent,
+		reason: ProviderEventDeliveryFailureReason,
+		attempts: number,
+		error?: unknown,
+	): void {
+		this.#failedCount += 1;
+		this.#incrementDropMetric(event);
+		try {
+			const recorded = this.#failureRecorder.record({
+				eventId: event.eventId,
+				reason,
+				attempts,
+				failedAt: this.#clock().toISOString(),
+				...(error ? { error: "Provider event could not be serialized." } : {}),
+			});
+			void Promise.resolve(recorded).catch(() => {});
+		} catch {}
+	}
+
+	#incrementDropMetric(
+		event: ProviderEvent,
+		emitter: ProviderEventMetricEmitter = this.#metricEmitter,
+	): void {
+		try {
+			emitter.increment(
+				"apifuse_stateful_provider_event_drop_total",
+				providerEventMetricLabels(event),
+			);
+		} catch {}
+	}
+}
+
+function delay(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function positiveInteger(value: number, name: string): number {
+	if (!Number.isInteger(value) || value <= 0) {
+		throw new Error(`Provider event ${name} must be a positive integer.`);
+	}
+	return value;
+}
+
+function nonnegativeFinite(value: number, name: string): number {
+	if (!Number.isFinite(value) || value < 0) {
+		throw new Error(`Provider event ${name} must be a non-negative finite number.`);
+	}
+	return value;
+}
+
+function boundedRatio(value: number, name: string): number {
+	if (!Number.isFinite(value) || value < 0 || value > 1) {
+		throw new Error(`Provider event ${name} must be between 0 and 1.`);
+	}
+	return value;
+}