@apifuse/provider-sdk 2.2.0-beta.3 → 2.2.0-beta.31

package/src/stateful/stateful-provider-observability.ts

@@ -0,0 +1,261 @@
+import { createHash } from "node:crypto";
+
+export const STATEFUL_PROVIDER_METRIC_NAMES = [
+	"apifuse_stateful_provider_session_owner_acquire_total",
+	"apifuse_stateful_provider_session_owner_renew_total",
+	"apifuse_stateful_provider_session_owner_release_total",
+	"apifuse_stateful_provider_session_owner_conflict_total",
+	"apifuse_stateful_provider_session_owner_fencing_stale_total",
+	"apifuse_stateful_provider_active_sessions",
+	"apifuse_stateful_provider_lru_evictions_total",
+	"apifuse_stateful_provider_session_invalidations_total",
+	"apifuse_stateful_provider_session_operation_queue_depth",
+	"apifuse_stateful_provider_session_operation_queue_wait_ms",
+	"apifuse_stateful_provider_session_operation_duration_ms",
+	"apifuse_stateful_provider_routing_forward_total",
+	"apifuse_stateful_provider_routing_local_total",
+	"apifuse_stateful_provider_routing_reacquire_total",
+	"apifuse_stateful_provider_routing_deadline_expired_total",
+	"apifuse_stateful_provider_event_append_total",
+	"apifuse_stateful_provider_event_dedupe_total",
+	"apifuse_stateful_provider_event_drop_total",
+	"apifuse_stateful_provider_event_lag_ms",
+	"apifuse_stateful_provider_webhook_enqueue_total",
+	"apifuse_stateful_provider_webhook_fanout_recovered_total",
+	"apifuse_stateful_provider_webhook_delivered_total",
+	"apifuse_stateful_provider_webhook_retry_total",
+	"apifuse_stateful_provider_webhook_dead_letter_total",
+	"apifuse_stateful_provider_webhook_canceled_total",
+	"apifuse_stateful_provider_webhook_replay_total",
+	"apifuse_stateful_provider_webhook_latency_ms",
+	"apifuse_stateful_provider_write_ledger_ambiguous_total",
+	"apifuse_stateful_provider_write_ledger_reconciled_total",
+	"apifuse_stateful_provider_write_ledger_failed_total",
+] as const;
+
+export type StatefulProviderMetricName = (typeof STATEFUL_PROVIDER_METRIC_NAMES)[number];
+
+export const STATEFUL_PROVIDER_LOG_FIELDS = [
+	"requestId",
+	"eventId",
+	"deliveryId",
+	"providerId",
+	"connectionId",
+	"serviceAccountId",
+	"operationId",
+	"sessionKeyHash",
+	"ownerPodId",
+	"generation",
+	"leaseExpiresAt",
+	"capacityClass",
+	"queueDepth",
+	"eventType",
+	"subjectKind",
+	"subjectId",
+	"retryCount",
+	"ledgerState",
+	"healthStatus",
+	"redactedErrorCode",
+] as const;
+
+export type StatefulProviderLogField = (typeof STATEFUL_PROVIDER_LOG_FIELDS)[number];
+
+export const STATEFUL_PROVIDER_AUTOSCALING_SIGNALS = [
+	"active_sessions",
+	"session_queue_depth",
+	"session_queue_wait_ms",
+	"reconnect_rate",
+	"provider_event_lag_ms",
+	"webhook_outbox_depth",
+	"webhook_due_retry_depth",
+	"memory_pressure",
+	"file_descriptor_pressure",
+	"socket_pressure",
+] as const;
+
+export type StatefulProviderAutoscalingSignal =
+	(typeof STATEFUL_PROVIDER_AUTOSCALING_SIGNALS)[number];
+
+export const STATEFUL_PROVIDER_HEALTH_DIMENSIONS = [
+	"registry_connectivity",
+	"owner_lease_freshness",
+	"pool_capacity",
+	"upstream_auth_validity",
+	"push_loop_lag",
+	"event_append_lag",
+	"webhook_backlog",
+] as const;
+
+export type StatefulProviderHealthDimension = (typeof STATEFUL_PROVIDER_HEALTH_DIMENSIONS)[number];
+
+export const STATEFUL_PROVIDER_ROLLOUT_STATES = [
+	"disabled",
+	"tenant_allowlist",
+	"connection_allowlist",
+	"shadow",
+	"canary",
+	"production",
+	"rollback",
+] as const;
+
+export type StatefulProviderRolloutState = (typeof STATEFUL_PROVIDER_ROLLOUT_STATES)[number];
+
+export interface StatefulProviderMetricLabelInput {
+	readonly requestId?: string;
+	readonly eventId?: string;
+	readonly deliveryId?: string;
+	readonly providerId?: string;
+	readonly connectionId?: string;
+	readonly serviceAccountId?: string;
+	readonly operationId?: string;
+	readonly sessionKey?: string;
+	readonly sessionKeyHash?: string;
+	readonly ownerPodId?: string;
+	readonly generation?: number | string;
+	readonly leaseExpiresAt?: string;
+	readonly capacityClass?: string;
+	readonly queueDepth?: number | string;
+	readonly eventType?: string;
+	readonly subjectKind?: string;
+	readonly subjectId?: string;
+	readonly retryCount?: number | string;
+	readonly ledgerState?: string;
+	readonly healthStatus?: string;
+	readonly redactedErrorCode?: string;
+}
+
+export type StatefulProviderMetricLabels = Partial<Record<StatefulProviderLogField, string>>;
+
+export interface StatefulProviderMetricEmitter {
+	increment(
+		name: StatefulProviderMetricName,
+		labels?: StatefulProviderMetricLabelInput,
+		value?: number,
+	): void;
+	gauge(
+		name: StatefulProviderMetricName,
+		labels: StatefulProviderMetricLabelInput,
+		value: number,
+	): void;
+	observe(
+		name: StatefulProviderMetricName,
+		labels: StatefulProviderMetricLabelInput,
+		value: number,
+	): void;
+}
+
+export type StatefulProviderRecordedMetric = {
+	readonly kind: "increment" | "gauge" | "observe";
+	readonly name: StatefulProviderMetricName;
+	readonly labels: StatefulProviderMetricLabels;
+	readonly value: number;
+};
+
+const METRIC_NAME_SET = new Set<string>(STATEFUL_PROVIDER_METRIC_NAMES);
+const LOG_FIELD_SET = new Set<string>(STATEFUL_PROVIDER_LOG_FIELDS);
+const AUTOSCALING_SIGNAL_SET = new Set<string>(STATEFUL_PROVIDER_AUTOSCALING_SIGNALS);
+
+export const NOOP_STATEFUL_PROVIDER_METRIC_EMITTER = {
+	increment() {},
+	gauge() {},
+	observe() {},
+} satisfies StatefulProviderMetricEmitter;
+
+export class RecordingStatefulProviderMetricEmitter implements StatefulProviderMetricEmitter {
+	readonly metrics: StatefulProviderRecordedMetric[] = [];
+
+	increment(
+		name: StatefulProviderMetricName,
+		labels: StatefulProviderMetricLabelInput = {},
+		value = 1,
+	): void {
+		this.metrics.push({
+			kind: "increment",
+			name,
+			labels: statefulProviderMetricLabels(labels),
+			value,
+		});
+	}
+
+	gauge(
+		name: StatefulProviderMetricName,
+		labels: StatefulProviderMetricLabelInput,
+		value: number,
+	): void {
+		this.metrics.push({
+			kind: "gauge",
+			name,
+			labels: statefulProviderMetricLabels(labels),
+			value,
+		});
+	}
+
+	observe(
+		name: StatefulProviderMetricName,
+		labels: StatefulProviderMetricLabelInput,
+		value: number,
+	): void {
+		this.metrics.push({
+			kind: "observe",
+			name,
+			labels: statefulProviderMetricLabels(labels),
+			value,
+		});
+	}
+}
+
+export function isStatefulProviderMetricName(value: unknown): value is StatefulProviderMetricName {
+	return typeof value === "string" && METRIC_NAME_SET.has(value);
+}
+
+export function isStatefulProviderLogField(value: unknown): value is StatefulProviderLogField {
+	return typeof value === "string" && LOG_FIELD_SET.has(value);
+}
+
+export function isStatefulProviderAutoscalingSignal(
+	value: unknown,
+): value is StatefulProviderAutoscalingSignal {
+	return typeof value === "string" && AUTOSCALING_SIGNAL_SET.has(value);
+}
+
+export function sanitizeSessionKeyForLog(sessionKey: string): string {
+	return `session_sha256:${createHash("sha256").update(sessionKey).digest("hex")}`;
+}
+
+export function statefulProviderMetricLabels(
+	input: StatefulProviderMetricLabelInput,
+): StatefulProviderMetricLabels {
+	const labels: StatefulProviderMetricLabels = {};
+	for (const field of STATEFUL_PROVIDER_LOG_FIELDS) {
+		if (field === "sessionKeyHash") continue;
+		const value = input[field];
+		if (value !== undefined) {
+			labels[field] = String(value);
+		}
+	}
+
+	const sessionKeyHash =
+		input.sessionKeyHash ??
+		(input.sessionKey ? sanitizeSessionKeyForLog(input.sessionKey) : undefined);
+	if (sessionKeyHash) {
+		labels.sessionKeyHash = sessionKeyHash;
+	}
+
+	if (input.sessionKey) {
+		assertNoRawSessionKeyInLabels(labels, input.sessionKey);
+	}
+
+	return labels;
+}
+
+export function assertNoRawSessionKeyInLabels(
+	labels: Record<string, string | number | boolean | null | undefined>,
+	rawSessionKey: string,
+): void {
+	if (!rawSessionKey) return;
+	for (const [key, value] of Object.entries(labels)) {
+		if (String(value ?? "").includes(rawSessionKey)) {
+			throw new Error(`Stateful provider metric label "${key}" contains a raw session key.`);
+		}
+	}
+}

package/src/stateful/stateful-provider-owner-forwarder.ts

@@ -0,0 +1,287 @@
+import { z } from "zod";
+import {
+	type OperationConnection,
+	OperationErrorResponseSchema,
+	OperationSuccessResponseSchema,
+} from "../server/index.js";
+import type { ProviderServerStatefulForwardEnvelope } from "../server/serve.js";
+import { signStatefulRequestBody, statefulSignedHeaders } from "../stateful-signing.js";
+import type { SessionKey } from "./session-key.js";
+import type {
+	StatefulOperationRequest,
+	StatefulOperationResult,
+	StatefulOwnerForwarder,
+} from "./stateful-provider-session-routing.js";
+import { forwardingContextFromStatefulRuntimeContext } from "./stateful-provider-session-routing.js";
+import type {
+	SessionOwnerRecord,
+	SessionOwnerRegistry,
+} from "./stateful-provider-session-runtime.js";
+
+export const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
+export const STATEFUL_FORWARDING_SIGNATURE_HEADER = "x-apifuse-stateful-signature";
+export const STATEFUL_FORWARDING_TIMESTAMP_HEADER = "x-apifuse-stateful-timestamp";
+export const STATEFUL_FORWARDING_NONCE_HEADER = "x-apifuse-stateful-nonce";
+export const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
+
+export interface StatefulOwnerForwarderOptions {
+	readonly currentPodId: string;
+	readonly secret: string;
+	readonly fetch?: FetchTransport;
+	readonly clock?: () => Date;
+}
+
+type FetchTransport = (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+const MAX_FORWARDED_HEADERS = 32;
+const MAX_FORWARDED_HEADER_BYTES = 8 * 1024;
+// Inbound compatibility boundary for rolling deploys: older owner pods omit
+// top-level retryable. Emitted responses remain strict via
+// OperationErrorResponseSchema.
+const ForwardedOperationErrorResponseSchema = OperationErrorResponseSchema.extend({
+	error: OperationErrorResponseSchema.shape.error.extend({
+		retryable: z.boolean().optional().default(false),
+	}),
+});
+const SENSITIVE_HEADER_NAMES = new Set([
+	"authorization",
+	"cookie",
+	"set-cookie",
+	STATEFUL_FORWARDING_SIGNATURE_HEADER,
+	STATEFUL_FORWARDING_TIMESTAMP_HEADER,
+	STATEFUL_FORWARDING_NONCE_HEADER,
+	STATEFUL_FORWARDING_SOURCE_POD_HEADER,
+]);
+
+/** Builds the standard fail-closed owner-currentness check for serve(). */
+export function createStatefulOwnerFenceValidator(registry: SessionOwnerRegistry) {
+	return async (
+		fence: Pick<ProviderServerStatefulForwardEnvelope, "sessionKey" | "ownerPodId" | "generation">,
+		signal: AbortSignal,
+	): Promise<boolean> => {
+		const current = await registry.resolve(fence.sessionKey as SessionKey, undefined, signal);
+		return (
+			current !== null &&
+			current.sessionKey === fence.sessionKey &&
+			current.ownerPodId === fence.ownerPodId &&
+			current.generation === fence.generation
+		);
+	};
+}
+
+export class StatefulOwnerForwardingError extends Error {
+	readonly code: string;
+	readonly status?: number;
+
+	constructor(input: {
+		readonly code: string;
+		readonly message: string;
+		readonly status?: number;
+		readonly cause?: unknown;
+	}) {
+		super(input.message, input.cause === undefined ? undefined : { cause: input.cause });
+		this.name = "StatefulOwnerForwardingError";
+		this.code = input.code;
+		this.status = input.status;
+	}
+}
+
+export class HttpStatefulOwnerForwarder implements StatefulOwnerForwarder {
+	readonly #currentPodId: string;
+	readonly #secret: string;
+	readonly #fetch: FetchTransport;
+	readonly #clock: () => Date;
+
+	constructor(options: StatefulOwnerForwarderOptions) {
+		if (options.secret.trim().length === 0) {
+			throw new Error("Stateful owner forwarding secret is required.");
+		}
+		this.#currentPodId = options.currentPodId;
+		this.#secret = options.secret;
+		this.#fetch = options.fetch ?? globalThis.fetch;
+		this.#clock = options.clock ?? (() => new Date());
+	}
+
+	async forward(
+		owner: SessionOwnerRecord,
+		request: StatefulOperationRequest,
+		signal: AbortSignal,
+	): Promise<StatefulOperationResult> {
+		assertNoRequestScopedFiles(request.input);
+		const forwardedAt = this.#clock().toISOString();
+		const envelope = buildForwardingEnvelope({
+			owner,
+			request,
+			sourcePodId: this.#currentPodId,
+			forwardedAt,
+		});
+		const rawBody = JSON.stringify(envelope);
+		let response: Response;
+		try {
+			const target = new URL(
+				`${owner.ownerEndpoint.replace(/\/+$/, "")}${STATEFUL_INTERNAL_OPERATIONS_ROUTE}`,
+			);
+			response = await this.#fetch(target, {
+				method: "POST",
+				headers: {
+					"content-type": "application/json",
+					...statefulSignedHeaders({
+						secret: this.#secret,
+						timestamp: forwardedAt,
+						rawBody,
+						method: "POST",
+						path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
+					}),
+					[STATEFUL_FORWARDING_SOURCE_POD_HEADER]: this.#currentPodId,
+				},
+				body: rawBody,
+				signal,
+			});
+		} catch (cause) {
+			throw new StatefulOwnerForwardingError({
+				code: "STATEFUL_FORWARDING_REQUEST_FAILED",
+				message: "Stateful owner forwarding request failed before a response was received.",
+				cause,
+			});
+		}
+		return parseForwardedResponse(response);
+	}
+}
+
+export function signStatefulForwardingBody(input: {
+	readonly secret: string;
+	readonly timestamp: string;
+	readonly rawBody: string;
+	readonly method: string;
+	readonly path: string;
+	readonly nonce: string;
+}): string {
+	return signStatefulRequestBody(input);
+}
+
+function assertNoRequestScopedFiles(input: unknown): void {
+	if (!containsRequestScopedFile(input)) return;
+	throw new StatefulOwnerForwardingError({
+		code: "STATEFUL_FILE_FORWARDING_UNSUPPORTED",
+		message:
+			"Request-scoped files cannot be forwarded to a remote stateful owner. Use staged upload once available so the forwarding envelope carries only durable file_ref values, or route the request to the owner-local provider process.",
+	});
+}
+
+function containsRequestScopedFile(value: unknown): boolean {
+	if (isRequestScopedFileRef(value)) return true;
+	if (Array.isArray(value)) return value.some((item) => containsRequestScopedFile(item));
+	if (!value || typeof value !== "object") return false;
+	return Object.values(value).some((item) => containsRequestScopedFile(item));
+}
+
+function isRequestScopedFileRef(value: unknown): boolean {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+	const record = Object.fromEntries(Object.entries(value));
+	return record.type === "request_file" && typeof record.id === "string";
+}
+
+function buildForwardingEnvelope(input: {
+	readonly owner: SessionOwnerRecord;
+	readonly request: StatefulOperationRequest;
+	readonly sourcePodId: string;
+	readonly forwardedAt: string;
+}): ProviderServerStatefulForwardEnvelope {
+	const metadata = forwardedMetadata(input.request);
+	return {
+		requestId: input.request.requestId,
+		providerId: input.request.providerId,
+		operationId: input.request.operationId,
+		sessionKey: input.request.sessionKey,
+		connectionId: input.request.connectionId,
+		serviceAccountId: input.request.serviceAccountId,
+		...(input.request.idempotencyKey ? { idempotencyKey: input.request.idempotencyKey } : {}),
+		...(input.request.deadlineAt !== undefined ? { deadlineAt: input.request.deadlineAt } : {}),
+		sourcePodId: input.sourcePodId,
+		forwardedAt: input.forwardedAt,
+		ownerPodId: input.owner.ownerPodId,
+		generation: input.owner.generation,
+		operationRequest: {
+			requestId: input.request.requestId,
+			input: input.request.input as Record<string, unknown>,
+			connection: metadata.connection,
+			headers: sanitizeForwardedHeaders(metadata.headers),
+			...(metadata.trace ? { trace: metadata.trace } : {}),
+		},
+	};
+}
+
+function forwardedMetadata(request: StatefulOperationRequest): {
+	readonly connection: OperationConnection;
+	readonly headers?: Record<string, string>;
+	readonly trace?: Record<string, string>;
+} {
+	const forwardingContext = forwardingContextFromStatefulRuntimeContext(request.runtimeContext);
+	if (!forwardingContext) {
+		throw new StatefulOwnerForwardingError({
+			code: "STATEFUL_FORWARDING_CONTEXT_MISSING",
+			message: "Stateful operation forwarding requires request context metadata.",
+		});
+	}
+	const operationRequest = forwardingContext.operationRequest;
+	const connection = OperationConnectionSchema.parse(operationRequest?.connection);
+	return {
+		connection,
+		...(operationRequest?.headers ? { headers: operationRequest.headers } : {}),
+		...(operationRequest?.trace ? { trace: operationRequest.trace } : {}),
+	};
+}
+
+function sanitizeForwardedHeaders(
+	headers: Record<string, string> | undefined,
+): Record<string, string> {
+	if (!headers) return {};
+	const forwarded: Record<string, string> = {};
+	let totalBytes = 0;
+	for (const [name, value] of Object.entries(headers)) {
+		const normalized = name.toLowerCase();
+		if (SENSITIVE_HEADER_NAMES.has(normalized)) continue;
+		const nextBytes = Buffer.byteLength(name) + Buffer.byteLength(value);
+		if (
+			Object.keys(forwarded).length >= MAX_FORWARDED_HEADERS ||
+			totalBytes + nextBytes > MAX_FORWARDED_HEADER_BYTES
+		) {
+			break;
+		}
+		forwarded[name] = value;
+		totalBytes += nextBytes;
+	}
+	return forwarded;
+}
+
+async function parseForwardedResponse(response: Response): Promise<StatefulOperationResult> {
+	const body = await response.json().catch(() => undefined);
+	const success = OperationSuccessResponseSchema.safeParse(body);
+	if (response.ok && success.success) {
+		return { output: success.data.data };
+	}
+
+	const error = ForwardedOperationErrorResponseSchema.safeParse(body);
+	if (error.success) {
+		throw new StatefulOwnerForwardingError({
+			code: error.data.error.code,
+			message: error.data.error.message,
+			status: response.status,
+		});
+	}
+
+	throw new StatefulOwnerForwardingError({
+		code: "STATEFUL_FORWARDING_BAD_RESPONSE",
+		message: `Stateful owner returned an invalid response with status ${response.status}.`,
+		status: response.status,
+	});
+}
+
+const OperationConnectionSchema = z.object({
+	id: z.string(),
+	mode: z.enum(["oauth2", "credentials", "platform-managed", "none"]),
+	secrets: z.record(z.string(), z.string()),
+	scopes: z.array(z.string()).optional(),
+	metadata: z.record(z.string(), z.unknown()),
+	externalRef: z.string(),
+});

package/src/stateful/stateful-provider-runtime-context.ts

@@ -0,0 +1,92 @@
+export interface StatefulLocalRuntimeContext {
+	readonly providerContext?: unknown;
+}
+
+export interface StatefulForwardingRuntimeContext {
+	readonly operationRequest?: {
+		readonly requestId: string;
+		readonly connection?: unknown;
+		readonly headers?: Record<string, string>;
+		readonly trace?: Record<string, string>;
+	};
+}
+
+export interface StatefulOperationRuntimeContext {
+	readonly local?: StatefulLocalRuntimeContext;
+	readonly forwarding?: StatefulForwardingRuntimeContext;
+}
+
+/**
+ * Attaches provider-process-only context for local execution. This value must
+ * never be serialized into owner-forwarding envelopes.
+ */
+export function withStatefulLocalProviderContext(
+	providerContext: unknown,
+	forwarding?: StatefulForwardingRuntimeContext,
+): StatefulOperationRuntimeContext {
+	return {
+		local: { providerContext },
+		...(forwarding ? { forwarding } : {}),
+	};
+}
+
+/**
+ * Copies only operation metadata that is safe to serialize across owner pods.
+ */
+export function statefulForwardingContextFromProviderRequest(request: {
+	readonly requestId: string;
+	readonly connection?: unknown;
+	readonly headers?: Record<string, string>;
+	readonly trace?: Record<string, string>;
+}): StatefulForwardingRuntimeContext {
+	return {
+		operationRequest: {
+			requestId: request.requestId,
+			...(request.connection !== undefined ? { connection: request.connection } : {}),
+			...(request.headers ? { headers: request.headers } : {}),
+			...(request.trace ? { trace: request.trace } : {}),
+		},
+	};
+}
+
+export function isStatefulOperationRuntimeContext(
+	value: unknown,
+): value is StatefulOperationRuntimeContext {
+	if (value === null || typeof value !== "object") return false;
+	return (
+		"local" in value ||
+		"forwarding" in value ||
+		"operationRequest" in value ||
+		"providerContext" in value
+	);
+}
+
+export function providerContextFromStatefulRuntimeContext(value: unknown): unknown {
+	if (!isStatefulOperationRuntimeContext(value)) return undefined;
+	if ("local" in value) return value.local?.providerContext;
+	// Backward-compatible read for pre-split runtimeContext objects. New code
+	// should always use `withStatefulLocalProviderContext` instead.
+	if ("providerContext" in value) return value.providerContext;
+	return undefined;
+}
+
+export function forwardingContextFromStatefulRuntimeContext(
+	value: unknown,
+): StatefulForwardingRuntimeContext | undefined {
+	if (!isStatefulOperationRuntimeContext(value)) return undefined;
+	if ("forwarding" in value) return value.forwarding;
+	if ("operationRequest" in value) {
+		const operationRequest = value.operationRequest;
+		if (isForwardingOperationRequest(operationRequest)) {
+			return { operationRequest };
+		}
+	}
+	return undefined;
+}
+
+function isForwardingOperationRequest(
+	value: unknown,
+): value is StatefulForwardingRuntimeContext["operationRequest"] {
+	if (value === null || typeof value !== "object") return false;
+	return "requestId" in value && typeof value.requestId === "string";
+}