@fedify/fedify 2.3.0-dev.1184 → 2.3.0-dev.1190

package/dist/{middleware-DWiKzrOa.cjs → middleware-ddMAHsyF.cjs}

@@ -2,10 +2,10 @@ const { Temporal } = require("@js-temporal/polyfill");
 const { URLPattern } = require("urlpattern-polyfill");
 const require_chunk = require("./chunk-DDcVe30Y.cjs");
 const require_transformers = require("./transformers-NeAONrAq.cjs");
-const require_http = require("./http-fmumSl9Q.cjs");
-const require_proof = require("./proof-BgsSe250.cjs");
+const require_http = require("./http-BAarxBe5.cjs");
+const require_proof = require("./proof-CXdtqYKw.cjs");
 const require_types = require("./types-KC4QAoxe.cjs");
-const require_kv_cache = require("./kv-cache-owsCV_hm.cjs");
+const require_kv_cache = require("./kv-cache-Ds1kjvnu.cjs");
 let _logtape_logtape = require("@logtape/logtape");
 let _fedify_uri_template = require("@fedify/uri-template");
 let _fedify_vocab = require("@fedify/vocab");
@@ -653,6 +653,338 @@ function createFederationBuilder() {
 	return new FederationBuilderImpl();
 }
 //#endregion
+//#region src/federation/circuit-breaker.ts
+const MAX_CUSTOM_FAILURE_HISTORY = 100;
+/**
+* Tracks reachability state for remote outbox delivery hosts.
+* @since 2.3.0
+*/
+var CircuitBreaker = class {
+	#kv;
+	#prefix;
+	#options;
+	#now;
+	#stateChangeObserver;
+	constructor(options) {
+		this.#kv = options.kv;
+		this.#prefix = options.prefix;
+		this.#options = normalizeCircuitBreakerOptions(options.options ?? {});
+		this.#now = options.now ?? (() => Temporal.Now.instant());
+		this.#stateChangeObserver = options.stateChangeObserver;
+	}
+	get options() {
+		return this.#options;
+	}
+	capHeldDelay(heldSince, delay) {
+		const now = this.#now();
+		return now.until(this.#capHeldRetryAt(now, heldSince, now.add(delay)));
+	}
+	async beforeSend(remoteHost, message) {
+		const heldSince = parseHeldSince(message.circuitHeldSince);
+		const now = this.#now();
+		if (heldSince != null && Temporal.Instant.compare(heldSince.add(this.#options.heldActivityTtl), now) <= 0) return {
+			type: "drop",
+			heldSince
+		};
+		let lastConflictingState;
+		for (let attempt = 0; attempt < 10; attempt++) {
+			const oldState = await this.#get(remoteHost);
+			if (oldState == null || oldState.state === "closed") return {
+				type: "send",
+				probe: false
+			};
+			if (oldState.state === "half-open") {
+				const halfOpened = oldState.halfOpened == null ? void 0 : Temporal.Instant.from(oldState.halfOpened);
+				if (halfOpened != null) {
+					const staleAt = halfOpened.add(this.#options.recoveryDelay);
+					if (Temporal.Instant.compare(now, staleAt) < 0) {
+						const releaseAt = now.add(this.#options.releaseInterval);
+						const retryAt = Temporal.Instant.compare(releaseAt, staleAt) < 0 ? releaseAt : staleAt;
+						const cappedRetryAt = this.#capHeldRetryAt(now, heldSince, retryAt);
+						return {
+							type: "hold",
+							state: "half-open",
+							delay: now.until(cappedRetryAt),
+							heldSince: heldSince ?? now
+						};
+					}
+				}
+				const newState = {
+					...oldState,
+					state: "half-open",
+					halfOpened: now.toString()
+				};
+				if (await this.#replace(remoteHost, oldState, newState)) return {
+					type: "send",
+					probe: true
+				};
+				lastConflictingState = "half-open";
+				continue;
+			}
+			const probeAt = (oldState.opened == null ? now : Temporal.Instant.from(oldState.opened)).add(this.#options.recoveryDelay);
+			if (Temporal.Instant.compare(now, probeAt) < 0) {
+				const retryAt = this.#capHeldRetryAt(now, heldSince, probeAt);
+				return {
+					type: "hold",
+					state: "open",
+					delay: now.until(retryAt),
+					heldSince: heldSince ?? now
+				};
+			}
+			const newState = {
+				...oldState,
+				state: "half-open",
+				halfOpened: now.toString()
+			};
+			if (await this.#replace(remoteHost, oldState, newState)) {
+				await this.#notifyStateChange(remoteHost, "open", "half-open");
+				return {
+					type: "send",
+					probe: true,
+					stateChange: {
+						previousState: "open",
+						newState: "half-open"
+					}
+				};
+			}
+			lastConflictingState = "open";
+		}
+		if (lastConflictingState != null) {
+			const retryAt = this.#capHeldRetryAt(now, heldSince, now.add(this.#options.releaseInterval));
+			return {
+				type: "hold",
+				state: lastConflictingState,
+				delay: now.until(retryAt),
+				heldSince: heldSince ?? now
+			};
+		}
+		throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
+	}
+	async recordSuccess(remoteHost) {
+		for (let attempt = 0; attempt < 10; attempt++) {
+			const oldState = await this.#get(remoteHost);
+			if (oldState == null) return void 0;
+			if (await this.#replace(remoteHost, oldState, void 0)) {
+				if (oldState.state !== "closed") {
+					await this.#notifyStateChange(remoteHost, oldState.state, "closed");
+					return {
+						previousState: oldState.state,
+						newState: "closed"
+					};
+				}
+				return;
+			}
+		}
+		throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
+	}
+	async recordReachableFailure(remoteHost) {
+		return await this.recordSuccess(remoteHost);
+	}
+	async recordFailure(remoteHost) {
+		const now = this.#now();
+		for (let attempt = 0; attempt < 10; attempt++) {
+			const oldState = await this.#get(remoteHost);
+			if (oldState?.state === "open") return void 0;
+			const oldFailures = oldState?.failures.map(Temporal.Instant.from) ?? [];
+			const failures = this.#options.pruneFailures([...oldFailures, now], now);
+			let newState;
+			let transition;
+			if (oldState?.state === "half-open" || this.#options.failure(failures)) {
+				newState = {
+					state: "open",
+					failures: failures.map((t) => t.toString()),
+					opened: now.toString()
+				};
+				transition = [oldState?.state ?? "closed", "open"];
+			} else newState = {
+				state: "closed",
+				failures: failures.map((t) => t.toString())
+			};
+			if (await this.#replace(remoteHost, oldState, newState)) {
+				if (transition != null) {
+					await this.#notifyStateChange(remoteHost, transition[0], transition[1]);
+					return {
+						previousState: transition[0],
+						newState: transition[1]
+					};
+				}
+				return;
+			}
+		}
+		throw new Error(`Failed to update circuit breaker state for ${remoteHost}`);
+	}
+	async dropActivity(remoteHost, details) {
+		try {
+			await this.#options.onActivityDrop?.(remoteHost, details);
+		} catch (error) {
+			(0, _logtape_logtape.getLogger)([
+				"fedify",
+				"federation",
+				"circuit"
+			]).error("An unexpected error occurred in circuit breaker activity drop handler:\n{error}", {
+				remoteHost,
+				error
+			});
+		}
+	}
+	async getState(remoteHost) {
+		return await this.#get(remoteHost);
+	}
+	#key(remoteHost) {
+		return [...this.#prefix, remoteHost];
+	}
+	#capHeldRetryAt(now, heldSince, retryAt) {
+		const expiresAt = (heldSince ?? now).add(this.#options.heldActivityTtl);
+		return Temporal.Instant.compare(expiresAt, retryAt) < 0 ? expiresAt : retryAt;
+	}
+	async #get(remoteHost) {
+		return parseCircuitBreakerKvState(await this.#kv.get(this.#key(remoteHost)));
+	}
+	async #replace(remoteHost, oldState, newState) {
+		const key = this.#key(remoteHost);
+		if (this.#kv.cas == null) {
+			if (newState == null) await this.#kv.delete(key);
+			else await this.#kv.set(key, newState);
+			return true;
+		}
+		return await this.#kv.cas(key, oldState, newState);
+	}
+	async #notifyStateChange(remoteHost, previousState, newState) {
+		try {
+			await this.#options.onStateChange?.(remoteHost, previousState, newState);
+		} catch (error) {
+			(0, _logtape_logtape.getLogger)([
+				"fedify",
+				"federation",
+				"circuit"
+			]).error("An unexpected error occurred in circuit breaker state change handler:\n{error}", {
+				remoteHost,
+				previousState,
+				newState,
+				error
+			});
+		}
+		try {
+			await this.#stateChangeObserver?.(remoteHost, previousState, newState);
+		} catch (error) {
+			(0, _logtape_logtape.getLogger)([
+				"fedify",
+				"federation",
+				"circuit"
+			]).error("An unexpected error occurred in circuit breaker state change observer:\n{error}", {
+				remoteHost,
+				previousState,
+				newState,
+				error
+			});
+		}
+	}
+};
+/**
+* Normalizes user-provided circuit breaker options into the internal policy
+* shape used while processing queued outbox deliveries.
+*
+* @param options The public circuit breaker options supplied to Fedify.
+* @returns The normalized failure predicate, failure pruning function,
+* duration values, and optional callbacks with defaults applied.
+* @throws {RangeError} If any configured duration is not positive.
+* @throws {TypeError} If `failureThreshold` is not a positive integer.
+*/
+function normalizeCircuitBreakerOptions(options) {
+	const recoveryDelay = toInstantDuration(options.recoveryDelay ?? { minutes: 30 });
+	const heldActivityTtl = toInstantDuration(options.heldActivityTtl ?? { hours: 168 });
+	const releaseInterval = toInstantDuration(options.releaseInterval ?? { seconds: 1 });
+	assertPositiveDuration(recoveryDelay, "recoveryDelay");
+	assertPositiveDuration(heldActivityTtl, "heldActivityTtl");
+	assertPositiveDuration(releaseInterval, "releaseInterval");
+	let failure;
+	let pruneFailures;
+	if (options.failure == null) {
+		const failureThreshold = options.failureThreshold ?? 5;
+		if (!Number.isInteger(failureThreshold) || failureThreshold <= 0) throw new TypeError("failureThreshold must be a positive integer.");
+		const failureWindow = toInstantDuration(options.failureWindow ?? { minutes: 10 });
+		assertPositiveDuration(failureWindow, "failureWindow");
+		pruneFailures = (timestamps, now) => {
+			const earliest = now.subtract(failureWindow);
+			return timestamps.filter((timestamp) => Temporal.Instant.compare(timestamp, earliest) >= 0).slice(-failureThreshold);
+		};
+		failure = (timestamps) => {
+			if (timestamps.length < failureThreshold) return false;
+			const first = timestamps[timestamps.length - failureThreshold];
+			const last = timestamps[timestamps.length - 1];
+			return Temporal.Duration.compare(first.until(last), failureWindow) <= 0;
+		};
+	} else {
+		failure = options.failure;
+		pruneFailures = (timestamps) => timestamps.slice(-MAX_CUSTOM_FAILURE_HISTORY);
+	}
+	return {
+		failure,
+		pruneFailures,
+		recoveryDelay,
+		heldActivityTtl,
+		releaseInterval,
+		onStateChange: options.onStateChange,
+		onActivityDrop: options.onActivityDrop
+	};
+}
+function toInstantDuration(duration) {
+	const parsed = Temporal.Duration.from(duration);
+	return Temporal.Duration.from({ milliseconds: Math.trunc(parsed.total({
+		unit: "millisecond",
+		relativeTo: Temporal.PlainDateTime.from("2026-01-01T00:00:00")
+	})) });
+}
+function assertPositiveDuration(duration, name) {
+	if (Temporal.Duration.compare(duration, { seconds: 0 }) <= 0) throw new RangeError(`${name} must be a positive duration.`);
+}
+function parseHeldSince(value) {
+	if (value == null) return void 0;
+	try {
+		return Temporal.Instant.from(value);
+	} catch (error) {
+		(0, _logtape_logtape.getLogger)([
+			"fedify",
+			"federation",
+			"circuit"
+		]).warn("Invalid circuitHeldSince value in queued outbox message: {value}", {
+			value,
+			error
+		});
+		return;
+	}
+}
+/**
+* Parses a value loaded from the circuit breaker KV store.
+*
+* @param value The raw KV value to validate.
+* @returns A circuit breaker state when `value` has a recognized state and
+* valid instant strings, or `undefined` when the stored value is malformed.
+*/
+function parseCircuitBreakerKvState(value) {
+	const isInstantString = (v) => {
+		if (typeof v !== "string") return false;
+		try {
+			Temporal.Instant.from(v);
+			return true;
+		} catch {
+			return false;
+		}
+	};
+	if (typeof value !== "object" || value == null) return void 0;
+	const record = value;
+	if (record.state !== "closed" && record.state !== "open" && record.state !== "half-open") return;
+	if (!Array.isArray(record.failures) || !record.failures.every((failure) => isInstantString(failure))) return;
+	if (record.opened != null && !isInstantString(record.opened)) return;
+	if (record.halfOpened != null && !isInstantString(record.halfOpened)) return;
+	return {
+		state: record.state,
+		failures: record.failures,
+		...record.opened == null ? {} : { opened: record.opened },
+		...record.halfOpened == null ? {} : { halfOpened: record.halfOpened }
+	};
+}
+//#endregion
 //#region src/federation/collection.ts
 /**
 * Calculates the [partial follower collection digest][1].
@@ -2911,13 +3243,14 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
 			specDeterminer
 		});
 	} catch (error) {
+		const transportError = error instanceof _fedify_vocab_runtime.FetchError ? error : createFetchError(inbox.href, error);
 		logger.error("Failed to send activity {activityId} to {inbox}:\n{error}", {
 			activityId,
 			inbox: inbox.href,
-			error
+			error: transportError
 		});
 		federationMetrics.recordDelivery(inbox, require_http.getDurationMs(started), false, activityType);
-		throw error;
+		throw transportError;
 	}
 	try {
 		if (!response.ok) {
@@ -2934,7 +3267,7 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
 				statusText: response.statusText,
 				error
 			});
-			throw new SendActivityError(inbox, response.status, `Failed to send activity ${activityId} to ${inbox.href} (${response.status} ${response.statusText}):\n${error}`, error);
+			throw new SendActivityError(inbox, response.status, `Failed to send activity ${activityId} to ${inbox.href} (${response.status} ${response.statusText}):\n${error}`, error, response.headers);
 		}
 		deliverySuccess = true;
 		const eventAttributes = {
@@ -2949,6 +3282,11 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
 		federationMetrics.recordDelivery(inbox, require_http.getDurationMs(started), deliverySuccess, activityType);
 	}
 }
+function createFetchError(url, cause) {
+	const error = new _fedify_vocab_runtime.FetchError(url, cause instanceof Error ? cause.message : String(cause));
+	error.cause = cause;
+	return error;
+}
 /**
 * An error that is thrown when an activity fails to send to a remote inbox.
 * It contains structured information about the failure, including the HTTP
@@ -2972,18 +3310,25 @@ var SendActivityError = class extends Error {
 	*/
 	responseBody;
 	/**
+	* The response headers from the inbox.
+	* @since 2.3.0
+	*/
+	responseHeaders;
+	/**
 	* Creates a new {@link SendActivityError}.
 	* @param inbox The inbox URL.
 	* @param statusCode The HTTP status code.
 	* @param message The error message.
 	* @param responseBody The response body.
+	* @param responseHeaders The response headers.
 	*/
-	constructor(inbox, statusCode, message, responseBody) {
+	constructor(inbox, statusCode, message, responseBody, responseHeaders) {
 		super(message);
 		this.name = "SendActivityError";
 		this.inbox = inbox;
 		this.statusCode = statusCode;
 		this.responseBody = responseBody;
+		this.responseHeaders = new Headers(responseHeaders);
 	}
 };
 //#endregion
@@ -3182,6 +3527,57 @@ var middleware_exports = /* @__PURE__ */ require_chunk.__exportAll({
 	OutboxContextImpl: () => OutboxContextImpl,
 	createFederation: () => createFederation
 });
+const circuitBreakerCasWarningKvStores = /* @__PURE__ */ new WeakSet();
+const retryAfterHttpDate = /* @__PURE__ */ new RegExp("^(?:(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \\d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4} \\d{2}:\\d{2}:\\d{2} GMT|(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), \\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\\d{2} \\d{2}:\\d{2}:\\d{2} GMT|(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?: \\d|\\d{2}) \\d{2}:\\d{2}:\\d{2} \\d{4})$");
+function parseRetryAfter(headers, now = Temporal.Now.instant()) {
+	const value = headers.get("Retry-After");
+	if (value == null) return void 0;
+	const trimmed = value.trim();
+	if (/^\d+$/.test(trimmed)) {
+		const seconds = Number(trimmed);
+		if (!Number.isFinite(seconds)) return void 0;
+		return parseRetryAfterDuration({ seconds });
+	}
+	if (!retryAfterHttpDate.test(trimmed)) return void 0;
+	const httpDate = trimmed.endsWith("GMT") ? trimmed : `${trimmed} GMT`;
+	const retryAtMs = Date.parse(httpDate);
+	if (Number.isNaN(retryAtMs)) return void 0;
+	const nowMs = Number(now.epochMilliseconds);
+	return parseRetryAfterDuration({ milliseconds: Math.max(0, retryAtMs - nowMs) });
+}
+function parseRetryAfterDuration(durationLike) {
+	try {
+		return Temporal.Duration.from(durationLike);
+	} catch (error) {
+		if (error instanceof RangeError) return void 0;
+		throw error;
+	}
+}
+function clampNegativeDelay(delay) {
+	return delay.sign < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay;
+}
+function maxDelay(first, second) {
+	return Temporal.Duration.compare(first, second) >= 0 ? first : second;
+}
+function isTransportDeliveryError(error) {
+	return error instanceof _fedify_vocab_runtime.FetchError || require_http.isAbortError(error);
+}
+function toCircuitBreakerMetricState(state) {
+	return state === "half-open" ? "half_open" : state;
+}
+function recordCircuitBreakerSpanEvent(span, remoteHost, change) {
+	span.addEvent("activitypub.circuit_breaker.state_change", {
+		"activitypub.remote.host": remoteHost,
+		"activitypub.circuit_breaker.previous_state": toCircuitBreakerMetricState(change.previousState),
+		"activitypub.circuit_breaker.state": toCircuitBreakerMetricState(change.newState)
+	});
+}
+function recordCircuitBreakerHeldSpanEvent(span, remoteHost, state) {
+	span.addEvent("activitypub.circuit_breaker.held", {
+		"activitypub.remote.host": remoteHost,
+		"activitypub.circuit_breaker.state": toCircuitBreakerMetricState(state)
+	});
+}
 function isRemoteContextLoadingFailure(error) {
 	return error instanceof Error && typeof error.details === "object" && error.details != null && error.details.code === "loading remote context failed";
 }
@@ -3225,6 +3621,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 	skipSignatureVerification;
 	outboxRetryPolicy;
 	inboxRetryPolicy;
+	circuitBreaker;
 	activityTransformers;
 	_tracerProvider;
 	_meterProvider;
@@ -3239,6 +3636,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 			publicKey: ["_fedify", "publicKey"],
 			httpMessageSignaturesSpec: ["_fedify", "httpMessageSignaturesSpec"],
 			acceptSignatureNonce: ["_fedify", "acceptSignatureNonce"],
+			circuitBreaker: ["_fedify", "circuit"],
 			...options.kvPrefixes ?? {}
 		};
 		if (options.queue == null) {
@@ -3254,6 +3652,25 @@ var FederationImpl = class extends FederationBuilderImpl {
 			this.outboxQueue = options.queue.outbox;
 			this.fanoutQueue = options.queue.fanout;
 		}
+		if (options.circuitBreaker !== false && this.outboxQueue != null) {
+			this.circuitBreaker = new CircuitBreaker({
+				kv: options.kv,
+				prefix: this.kvPrefixes.circuitBreaker,
+				options: options.circuitBreaker,
+				stateChangeObserver: (remoteHost, _previousState, newState) => {
+					const metricState = toCircuitBreakerMetricState(newState);
+					require_http.recordCircuitBreakerStateChange(this.meterProvider, remoteHost, metricState);
+				}
+			});
+			if (options.kv.cas == null && !circuitBreakerCasWarningKvStores.has(options.kv)) {
+				circuitBreakerCasWarningKvStores.add(options.kv);
+				(0, _logtape_logtape.getLogger)([
+					"fedify",
+					"federation",
+					"circuit"
+				]).warn("The configured key-value store does not support CAS; outbound delivery circuit breaker updates may race under concurrent workers.");
+			}
+		}
 		this.inboxQueueStarted = false;
 		this.outboxQueueStarted = false;
 		this.fanoutQueueStarted = false;
@@ -3562,19 +3979,129 @@ var FederationImpl = class extends FederationBuilderImpl {
 			if (rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") rsaKeyPair = pair;
 			keys.push(pair);
 		}
+		const loaderOptions = this.#getLoaderOptions(message.baseUrl);
+		let parsedActorIds;
+		const getActorIds = () => {
+			parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => {
+				try {
+					return [new URL(id)];
+				} catch {
+					logger.warn("Invalid actorId URL in OutboxMessage: {id}", { id });
+					return [];
+				}
+			});
+			return parsedActorIds;
+		};
+		const parseActivity = () => _fedify_vocab.Activity.fromJsonLd(message.activity, {
+			contextLoader: this.contextLoaderFactory(loaderOptions),
+			documentLoader: rsaKeyPair == null ? this.documentLoaderFactory(loaderOptions) : this.authenticatedDocumentLoaderFactory(rsaKeyPair, loaderOptions),
+			tracerProvider: this.tracerProvider
+		});
+		const enqueueHeldOutboxMessage = async (delay, heldSince) => {
+			const { outboxQueue } = this;
+			if (outboxQueue == null) return;
+			const heldMessage = {
+				...message,
+				circuitHeld: true,
+				circuitHeldSince: heldSince.toString()
+			};
+			await outboxQueue.enqueue(heldMessage, {
+				delay: clampNegativeDelay(delay),
+				orderingKey: message.orderingKey
+			});
+			require_http.getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+				role: "outbox",
+				queue: outboxQueue,
+				activityType: heldMessage.activityType
+			}, heldMessage.attempt);
+		};
+		const dropHeldOutboxMessage = async (circuit, remoteHost, inbox, heldSince, activity) => {
+			await circuit.dropActivity(remoteHost, {
+				inbox,
+				activity,
+				activityId: message.activityId,
+				activityType: message.activityType,
+				actorIds: getActorIds(),
+				heldSince
+			});
+			if (this.outboxPermanentFailureHandler != null) {
+				const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
+				try {
+					await this.outboxPermanentFailureHandler(ctx, {
+						reason: "circuit-breaker-ttl",
+						inbox,
+						activity,
+						error: new SendActivityError(inbox, 0, "Circuit breaker held activity expired.", ""),
+						statusCode: 0,
+						circuitHeldSince: heldSince,
+						actorIds: getActorIds()
+					});
+				} catch (handlerError) {
+					logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
+						...logData,
+						error: handlerError
+					});
+				}
+			}
+			require_http.recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
+		};
 		try {
+			const inbox = new URL(message.inbox);
+			const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
+			const remoteHost = require_http.getRemoteHost(inbox);
+			let decision;
+			if (circuit != null) try {
+				decision = await circuit.beforeSend(remoteHost, message);
+			} catch (circuitError) {
+				(0, _logtape_logtape.getLogger)([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to check circuit breaker state before sending; proceeding with delivery:\n{error}", {
+					...logData,
+					remoteHost,
+					error: circuitError
+				});
+			}
+			if (decision != null && circuit != null) {
+				if (decision.type === "hold") {
+					recordCircuitBreakerHeldSpanEvent(span, remoteHost, decision.state);
+					await enqueueHeldOutboxMessage(decision.delay, decision.heldSince);
+					return;
+				}
+				if (decision.type === "drop") {
+					const activity = await parseActivity();
+					await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
+					return;
+				}
+				if (decision.stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, decision.stateChange);
+			}
 			await sendActivity({
 				keys,
 				activity: message.activity,
 				activityId: message.activityId,
 				activityType: message.activityType,
-				inbox: new URL(message.inbox),
+				inbox,
 				sharedInbox: message.sharedInbox,
 				headers: new Headers(message.headers),
 				specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock),
 				meterProvider: this.meterProvider,
 				tracerProvider: this.tracerProvider
 			});
+			if (circuit != null) try {
+				const stateChange = await circuit.recordSuccess(remoteHost);
+				if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+			} catch (error) {
+				(0, _logtape_logtape.getLogger)([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to record successful delivery in circuit breaker state; the activity was already delivered:\n{error}", {
+					...logData,
+					remoteHost,
+					error
+				});
+			}
 		} catch (error) {
 			span.setStatus({
 				code: _opentelemetry_api.SpanStatusCode.ERROR,
@@ -3589,18 +4116,65 @@ var FederationImpl = class extends FederationBuilderImpl {
 					return;
 				}
 			})();
+			let retryAfterDelay;
+			let circuitHold;
+			let circuitDrop;
+			let retryPolicyDelay;
+			let policyDelayCalculated = false;
+			const getPolicyDelay = () => {
+				if (!policyDelayCalculated) {
+					retryPolicyDelay = this.outboxRetryPolicy({
+						elapsedTime: Temporal.Instant.from(message.started).until(Temporal.Now.instant()),
+						attempts: message.attempt
+					});
+					policyDelayCalculated = true;
+				}
+				return retryPolicyDelay;
+			};
+			const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode);
+			if (!isPermanentFailure && error instanceof SendActivityError && (error.statusCode === 429 || error.statusCode === 503)) retryAfterDelay = parseRetryAfter(error.responseHeaders);
+			if (remoteHost != null && this.outboxQueue != null && this.circuitBreaker != null) try {
+				if (error instanceof SendActivityError) {
+					const { statusCode } = error;
+					const stateChange = isPermanentFailure || statusCode === 429 || statusCode >= 400 && statusCode < 500 ? await this.circuitBreaker.recordReachableFailure(remoteHost) : statusCode >= 500 ? await this.circuitBreaker.recordFailure(remoteHost) : void 0;
+					if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+				} else if (isTransportDeliveryError(error)) {
+					const stateChange = await this.circuitBreaker.recordFailure(remoteHost);
+					if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+				}
+				if (!isPermanentFailure) {
+					const circuitDecision = await this.circuitBreaker.beforeSend(remoteHost, message);
+					if (circuitDecision.type === "hold") circuitHold = {
+						delay: circuitDecision.delay,
+						heldSince: circuitDecision.heldSince,
+						remoteHost,
+						state: circuitDecision.state
+					};
+					else if (circuitDecision.type === "drop") circuitDrop = {
+						circuit: this.circuitBreaker,
+						remoteHost,
+						inbox: new URL(message.inbox),
+						heldSince: circuitDecision.heldSince
+					};
+				}
+			} catch (circuitError) {
+				(0, _logtape_logtape.getLogger)([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to update circuit breaker state after delivery failure; falling back to normal failure handling:\n{error}", {
+					...logData,
+					remoteHost,
+					error: circuitError
+				});
+			}
 			span.addEvent("activitypub.delivery.failed", {
 				...remoteHost == null ? {} : { "activitypub.remote.host": remoteHost },
 				"activitypub.delivery.attempt": message.attempt,
-				"activitypub.delivery.permanent_failure": error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode),
+				"activitypub.delivery.permanent_failure": isPermanentFailure,
 				...error instanceof SendActivityError ? { "http.response.status_code": error.statusCode } : {}
 			});
-			const loaderOptions = this.#getLoaderOptions(message.baseUrl);
-			const activity = await _fedify_vocab.Activity.fromJsonLd(message.activity, {
-				contextLoader: this.contextLoaderFactory(loaderOptions),
-				documentLoader: rsaKeyPair == null ? this.documentLoaderFactory(loaderOptions) : this.authenticatedDocumentLoaderFactory(rsaKeyPair, loaderOptions),
-				tracerProvider: this.tracerProvider
-			});
+			const activity = await parseActivity();
 			try {
 				await this.onOutboxError?.(error, activity);
 			} catch (error) {
@@ -3609,7 +4183,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 					error
 				});
 			}
-			if (error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode)) {
+			if (circuitDrop != null) {
+				await dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
+				return;
+			}
+			if (isPermanentFailure) {
 				require_http.getFederationMetrics(this.meterProvider).recordPermanentFailure(error.inbox, error.statusCode);
 				logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not retrying.", {
 					...logData,
@@ -3619,18 +4197,12 @@ var FederationImpl = class extends FederationBuilderImpl {
 					const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
 					try {
 						await this.outboxPermanentFailureHandler(ctx, {
+							reason: "http",
 							inbox: new URL(message.inbox),
 							activity,
 							error,
 							statusCode: error.statusCode,
-							actorIds: (message.actorIds ?? []).flatMap((id) => {
-								try {
-									return [new URL(id)];
-								} catch {
-									logger.warn("Invalid actorId URL in OutboxMessage: {id}", { id });
-									return [];
-								}
-							})
+							actorIds: getActorIds()
 						});
 					} catch (handlerError) {
 						logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
@@ -3642,17 +4214,25 @@ var FederationImpl = class extends FederationBuilderImpl {
 				require_http.recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
 				return;
 			}
-			if (this.outboxQueue?.nativeRetrial) {
+			if (circuitHold != null && getPolicyDelay() != null) {
+				logger.error("Failed to send activity {activityId} to {inbox}; holding because the remote host circuit is open:\n{error}", {
+					...logData,
+					error
+				});
+				recordCircuitBreakerHeldSpanEvent(span, circuitHold.remoteHost, circuitHold.state);
+				const circuit = this.circuitBreaker;
+				await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince);
+				return;
+			}
+			if (this.outboxQueue?.nativeRetrial && retryAfterDelay == null) {
 				logger.error("Failed to send activity {activityId} to {inbox}; backend will handle retry:\n{error}", {
 					...logData,
 					error
 				});
 				throw error;
 			}
-			const delay = this.outboxRetryPolicy({
-				elapsedTime: Temporal.Instant.from(message.started).until(Temporal.Now.instant()),
-				attempts: message.attempt
-			});
+			const policyDelay = getPolicyDelay();
+			const delay = policyDelay == null ? null : retryAfterDelay ?? policyDelay;
 			if (delay != null) {
 				logger.error("Failed to send activity {activityId} to {inbox} (attempt #{attempt}); retry...:\n{error}", {
 					...logData,
@@ -3664,7 +4244,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 				};
 				const { outboxQueue } = this;
 				if (outboxQueue != null) {
-					await outboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					await outboxQueue.enqueue(retryMessage, {
+						delay: clampNegativeDelay(delay),
+						orderingKey: message.orderingKey
+					});
 					require_http.getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
 						role: "outbox",
 						queue: outboxQueue,
@@ -3753,7 +4336,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						...message,
 						attempt: message.attempt + 1
 					};
-					await this.inboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					await this.inboxQueue.enqueue(retryMessage, { delay: clampNegativeDelay(delay) });
 					if (activityType != null) {
 						require_http.getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
 							role: "inbox",
@@ -5506,6 +6089,12 @@ function getRequestId(request) {
 	return `req_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
 }
 //#endregion
+Object.defineProperty(exports, "CircuitBreaker", {
+	enumerable: true,
+	get: function() {
+		return CircuitBreaker;
+	}
+});
 Object.defineProperty(exports, "SendActivityError", {
 	enumerable: true,
 	get: function() {
@@ -5554,6 +6143,18 @@ Object.defineProperty(exports, "middleware_exports", {
 		return middleware_exports;
 	}
 });
+Object.defineProperty(exports, "normalizeCircuitBreakerOptions", {
+	enumerable: true,
+	get: function() {
+		return normalizeCircuitBreakerOptions;
+	}
+});
+Object.defineProperty(exports, "parseCircuitBreakerKvState", {
+	enumerable: true,
+	get: function() {
+		return parseCircuitBreakerKvState;
+	}
+});
 Object.defineProperty(exports, "respondWithObject", {
 	enumerable: true,
 	get: function() {