@fedify/fedify 2.3.0-dev.1189 → 2.3.0-dev.1212

package/dist/{middleware-DK0thDHX.mjs → middleware-BmSzD5U9.mjs}

@@ -1,26 +1,27 @@
 import { Temporal } from "@js-temporal/polyfill";
 import "urlpattern-polyfill";
 globalThis.addEventListener = () => {};
-import { n as version, t as name } from "./deno-BomxIkHS.mjs";
-import { _ as recordOutboxActivity, a as instrumentDocumentLoader, c as recordCollectionDispatchDuration, d as recordCollectionTotalItems, h as recordInboxActivity, i as getRemoteHost, l as recordCollectionPageItems, m as recordFanoutRecipients, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionRequest, v as recordOutboxEnqueue, y as recordWebFingerHandle } from "./metrics-xgr0P4hO.mjs";
+import { n as version, t as name } from "./deno-DTaoLXHr.mjs";
+import { a as instrumentDocumentLoader, b as recordWebFingerHandle, c as recordCircuitBreakerStateChange, d as recordCollectionRequest, f as recordCollectionTotalItems, g as recordInboxActivity, h as recordFanoutRecipients, i as getRemoteHost, l as recordCollectionDispatchDuration, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionPageItems, v as recordOutboxActivity, y as recordOutboxEnqueue } from "./metrics-uwSF8DLC.mjs";
 import { t as formatAcceptSignature } from "./accept-CceiKpCy.mjs";
-import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-CT2NnJuR.mjs";
-import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-DtWN_XvX.mjs";
-import { t as getAuthenticatedDocumentLoader } from "./docloader-CzS6F5sZ.mjs";
-import { n as kvCache } from "./kv-cache-Bf8AoV6C.mjs";
-import { _ as wrapContextLoaderForJsonLd, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, l as hasSignature, m as verifyCompactJsonLd, p as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike } from "./ld-DCyQasTE.mjs";
-import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-BIU_Sl7y.mjs";
+import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-Bhsx9PrC.mjs";
+import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-ByCfCX5K.mjs";
+import { t as getAuthenticatedDocumentLoader } from "./docloader-CdNiXmNg.mjs";
+import { n as kvCache } from "./kv-cache-D4jzgeYW.mjs";
+import { _ as wrapContextLoaderForJsonLd, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, l as hasSignature, m as verifyCompactJsonLd, p as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike } from "./ld-CHtLb_Uh.mjs";
+import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-B0Zrhs0w.mjs";
 import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-BgFLCJQ_.mjs";
-import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-DDs7BRl7.mjs";
+import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-DbJFxpzD.mjs";
 import { t as getNodeInfo } from "./client-B_A6mfn3.mjs";
 import { t as nodeInfoToJson } from "./types-BFowWFTT.mjs";
-import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-Dc6s3gPe.mjs";
-import { t as buildCollectionSynchronizationHeader } from "./collection-CA3V5zyK.mjs";
-import { t as KvKeyCache } from "./keycache-BYMd8q7F.mjs";
-import { t as acceptsJsonLd } from "./negotiation-CDW-_gUU.mjs";
-import { t as hasMalformedKnownTemporalLiteral } from "./temporal-DHgeMWiP.mjs";
-import { t as createExponentialBackoffPolicy } from "./retry-_VvV0h9f.mjs";
-import { n as extractInboxes, r as sendActivity, t as SendActivityError } from "./send-BuxDCpxz.mjs";
+import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-DdbtvTFp.mjs";
+import { t as CircuitBreaker } from "./circuit-breaker-CSWsyoef.mjs";
+import { t as buildCollectionSynchronizationHeader } from "./collection-Cc3DVAhE.mjs";
+import { t as KvKeyCache } from "./keycache-BeU0LCII.mjs";
+import { t as acceptsJsonLd } from "./negotiation-DDstyBvc.mjs";
+import { t as hasMalformedKnownTemporalLiteral } from "./temporal-CcGypkzd.mjs";
+import { t as createExponentialBackoffPolicy } from "./retry-CXg_MBI-.mjs";
+import { n as extractInboxes, r as sendActivity, t as SendActivityError } from "./send-kst2L0Df.mjs";
 import { getLogger, withContext } from "@logtape/logtape";
 import { RouterError } from "@fedify/uri-template";
 import { Activity, Collection, CollectionPage, CryptographicKey, Link, Multikey, Object as Object$1, OrderedCollection, OrderedCollectionPage, Tombstone, getTypeId, lookupObject, traverseCollection } from "@fedify/vocab";
@@ -2083,6 +2084,57 @@ async function handleWebFingerInternal(request, { context, host, actorDispatcher
 }
 //#endregion
 //#region src/federation/middleware.ts
+const circuitBreakerCasWarningKvStores = /* @__PURE__ */ new WeakSet();
+const retryAfterHttpDate = /* @__PURE__ */ new RegExp("^(?:(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \\d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4} \\d{2}:\\d{2}:\\d{2} GMT|(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), \\d{2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\\d{2} \\d{2}:\\d{2}:\\d{2} GMT|(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?: \\d|\\d{2}) \\d{2}:\\d{2}:\\d{2} \\d{4})$");
+function parseRetryAfter(headers, now = Temporal.Now.instant()) {
+	const value = headers.get("Retry-After");
+	if (value == null) return void 0;
+	const trimmed = value.trim();
+	if (/^\d+$/.test(trimmed)) {
+		const seconds = Number(trimmed);
+		if (!Number.isFinite(seconds)) return void 0;
+		return parseRetryAfterDuration({ seconds });
+	}
+	if (!retryAfterHttpDate.test(trimmed)) return void 0;
+	const httpDate = trimmed.endsWith("GMT") ? trimmed : `${trimmed} GMT`;
+	const retryAtMs = Date.parse(httpDate);
+	if (Number.isNaN(retryAtMs)) return void 0;
+	const nowMs = Number(now.epochMilliseconds);
+	return parseRetryAfterDuration({ milliseconds: Math.max(0, retryAtMs - nowMs) });
+}
+function parseRetryAfterDuration(durationLike) {
+	try {
+		return Temporal.Duration.from(durationLike);
+	} catch (error) {
+		if (error instanceof RangeError) return void 0;
+		throw error;
+	}
+}
+function clampNegativeDelay(delay) {
+	return delay.sign < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay;
+}
+function maxDelay(first, second) {
+	return Temporal.Duration.compare(first, second) >= 0 ? first : second;
+}
+function isTransportDeliveryError(error) {
+	return error instanceof FetchError || isAbortError(error);
+}
+function toCircuitBreakerMetricState(state) {
+	return state === "half-open" ? "half_open" : state;
+}
+function recordCircuitBreakerSpanEvent(span, remoteHost, change) {
+	span.addEvent("activitypub.circuit_breaker.state_change", {
+		"activitypub.remote.host": remoteHost,
+		"activitypub.circuit_breaker.previous_state": toCircuitBreakerMetricState(change.previousState),
+		"activitypub.circuit_breaker.state": toCircuitBreakerMetricState(change.newState)
+	});
+}
+function recordCircuitBreakerHeldSpanEvent(span, remoteHost, state) {
+	span.addEvent("activitypub.circuit_breaker.held", {
+		"activitypub.remote.host": remoteHost,
+		"activitypub.circuit_breaker.state": toCircuitBreakerMetricState(state)
+	});
+}
 function isRemoteContextLoadingFailure(error) {
 	return error instanceof Error && typeof error.details === "object" && error.details != null && error.details.code === "loading remote context failed";
 }
@@ -2126,6 +2178,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 	skipSignatureVerification;
 	outboxRetryPolicy;
 	inboxRetryPolicy;
+	circuitBreaker;
 	activityTransformers;
 	_tracerProvider;
 	_meterProvider;
@@ -2140,6 +2193,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 			publicKey: ["_fedify", "publicKey"],
 			httpMessageSignaturesSpec: ["_fedify", "httpMessageSignaturesSpec"],
 			acceptSignatureNonce: ["_fedify", "acceptSignatureNonce"],
+			circuitBreaker: ["_fedify", "circuit"],
 			...options.kvPrefixes ?? {}
 		};
 		if (options.queue == null) {
@@ -2155,6 +2209,25 @@ var FederationImpl = class extends FederationBuilderImpl {
 			this.outboxQueue = options.queue.outbox;
 			this.fanoutQueue = options.queue.fanout;
 		}
+		if (options.circuitBreaker !== false && this.outboxQueue != null) {
+			this.circuitBreaker = new CircuitBreaker({
+				kv: options.kv,
+				prefix: this.kvPrefixes.circuitBreaker,
+				options: options.circuitBreaker,
+				stateChangeObserver: (remoteHost, _previousState, newState) => {
+					const metricState = toCircuitBreakerMetricState(newState);
+					recordCircuitBreakerStateChange(this.meterProvider, remoteHost, metricState);
+				}
+			});
+			if (options.kv.cas == null && !circuitBreakerCasWarningKvStores.has(options.kv)) {
+				circuitBreakerCasWarningKvStores.add(options.kv);
+				getLogger([
+					"fedify",
+					"federation",
+					"circuit"
+				]).warn("The configured key-value store does not support CAS; outbound delivery circuit breaker updates may race under concurrent workers.");
+			}
+		}
 		this.inboxQueueStarted = false;
 		this.outboxQueueStarted = false;
 		this.fanoutQueueStarted = false;
@@ -2463,19 +2536,129 @@ var FederationImpl = class extends FederationBuilderImpl {
 			if (rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") rsaKeyPair = pair;
 			keys.push(pair);
 		}
+		const loaderOptions = this.#getLoaderOptions(message.baseUrl);
+		let parsedActorIds;
+		const getActorIds = () => {
+			parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => {
+				try {
+					return [new URL(id)];
+				} catch {
+					logger.warn("Invalid actorId URL in OutboxMessage: {id}", { id });
+					return [];
+				}
+			});
+			return parsedActorIds;
+		};
+		const parseActivity = () => Activity.fromJsonLd(message.activity, {
+			contextLoader: this.contextLoaderFactory(loaderOptions),
+			documentLoader: rsaKeyPair == null ? this.documentLoaderFactory(loaderOptions) : this.authenticatedDocumentLoaderFactory(rsaKeyPair, loaderOptions),
+			tracerProvider: this.tracerProvider
+		});
+		const enqueueHeldOutboxMessage = async (delay, heldSince) => {
+			const { outboxQueue } = this;
+			if (outboxQueue == null) return;
+			const heldMessage = {
+				...message,
+				circuitHeld: true,
+				circuitHeldSince: heldSince.toString()
+			};
+			await outboxQueue.enqueue(heldMessage, {
+				delay: clampNegativeDelay(delay),
+				orderingKey: message.orderingKey
+			});
+			getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+				role: "outbox",
+				queue: outboxQueue,
+				activityType: heldMessage.activityType
+			}, heldMessage.attempt);
+		};
+		const dropHeldOutboxMessage = async (circuit, remoteHost, inbox, heldSince, activity) => {
+			await circuit.dropActivity(remoteHost, {
+				inbox,
+				activity,
+				activityId: message.activityId,
+				activityType: message.activityType,
+				actorIds: getActorIds(),
+				heldSince
+			});
+			if (this.outboxPermanentFailureHandler != null) {
+				const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
+				try {
+					await this.outboxPermanentFailureHandler(ctx, {
+						reason: "circuit-breaker-ttl",
+						inbox,
+						activity,
+						error: new SendActivityError(inbox, 0, "Circuit breaker held activity expired.", ""),
+						statusCode: 0,
+						circuitHeldSince: heldSince,
+						actorIds: getActorIds()
+					});
+				} catch (handlerError) {
+					logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
+						...logData,
+						error: handlerError
+					});
+				}
+			}
+			recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
+		};
 		try {
+			const inbox = new URL(message.inbox);
+			const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
+			const remoteHost = getRemoteHost(inbox);
+			let decision;
+			if (circuit != null) try {
+				decision = await circuit.beforeSend(remoteHost, message);
+			} catch (circuitError) {
+				getLogger([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to check circuit breaker state before sending; proceeding with delivery:\n{error}", {
+					...logData,
+					remoteHost,
+					error: circuitError
+				});
+			}
+			if (decision != null && circuit != null) {
+				if (decision.type === "hold") {
+					recordCircuitBreakerHeldSpanEvent(span, remoteHost, decision.state);
+					await enqueueHeldOutboxMessage(decision.delay, decision.heldSince);
+					return;
+				}
+				if (decision.type === "drop") {
+					const activity = await parseActivity();
+					await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
+					return;
+				}
+				if (decision.stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, decision.stateChange);
+			}
 			await sendActivity({
 				keys,
 				activity: message.activity,
 				activityId: message.activityId,
 				activityType: message.activityType,
-				inbox: new URL(message.inbox),
+				inbox,
 				sharedInbox: message.sharedInbox,
 				headers: new Headers(message.headers),
 				specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock),
 				meterProvider: this.meterProvider,
 				tracerProvider: this.tracerProvider
 			});
+			if (circuit != null) try {
+				const stateChange = await circuit.recordSuccess(remoteHost);
+				if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+			} catch (error) {
+				getLogger([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to record successful delivery in circuit breaker state; the activity was already delivered:\n{error}", {
+					...logData,
+					remoteHost,
+					error
+				});
+			}
 		} catch (error) {
 			span.setStatus({
 				code: SpanStatusCode.ERROR,
@@ -2490,18 +2673,65 @@ var FederationImpl = class extends FederationBuilderImpl {
 					return;
 				}
 			})();
+			let retryAfterDelay;
+			let circuitHold;
+			let circuitDrop;
+			let retryPolicyDelay;
+			let policyDelayCalculated = false;
+			const getPolicyDelay = () => {
+				if (!policyDelayCalculated) {
+					retryPolicyDelay = this.outboxRetryPolicy({
+						elapsedTime: Temporal.Instant.from(message.started).until(Temporal.Now.instant()),
+						attempts: message.attempt
+					});
+					policyDelayCalculated = true;
+				}
+				return retryPolicyDelay;
+			};
+			const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode);
+			if (!isPermanentFailure && error instanceof SendActivityError && (error.statusCode === 429 || error.statusCode === 503)) retryAfterDelay = parseRetryAfter(error.responseHeaders);
+			if (remoteHost != null && this.outboxQueue != null && this.circuitBreaker != null) try {
+				if (error instanceof SendActivityError) {
+					const { statusCode } = error;
+					const stateChange = isPermanentFailure || statusCode === 429 || statusCode >= 400 && statusCode < 500 ? await this.circuitBreaker.recordReachableFailure(remoteHost) : statusCode >= 500 ? await this.circuitBreaker.recordFailure(remoteHost) : void 0;
+					if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+				} else if (isTransportDeliveryError(error)) {
+					const stateChange = await this.circuitBreaker.recordFailure(remoteHost);
+					if (stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, stateChange);
+				}
+				if (!isPermanentFailure) {
+					const circuitDecision = await this.circuitBreaker.beforeSend(remoteHost, message);
+					if (circuitDecision.type === "hold") circuitHold = {
+						delay: circuitDecision.delay,
+						heldSince: circuitDecision.heldSince,
+						remoteHost,
+						state: circuitDecision.state
+					};
+					else if (circuitDecision.type === "drop") circuitDrop = {
+						circuit: this.circuitBreaker,
+						remoteHost,
+						inbox: new URL(message.inbox),
+						heldSince: circuitDecision.heldSince
+					};
+				}
+			} catch (circuitError) {
+				getLogger([
+					"fedify",
+					"federation",
+					"circuit"
+				]).error("Failed to update circuit breaker state after delivery failure; falling back to normal failure handling:\n{error}", {
+					...logData,
+					remoteHost,
+					error: circuitError
+				});
+			}
 			span.addEvent("activitypub.delivery.failed", {
 				...remoteHost == null ? {} : { "activitypub.remote.host": remoteHost },
 				"activitypub.delivery.attempt": message.attempt,
-				"activitypub.delivery.permanent_failure": error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode),
+				"activitypub.delivery.permanent_failure": isPermanentFailure,
 				...error instanceof SendActivityError ? { "http.response.status_code": error.statusCode } : {}
 			});
-			const loaderOptions = this.#getLoaderOptions(message.baseUrl);
-			const activity = await Activity.fromJsonLd(message.activity, {
-				contextLoader: this.contextLoaderFactory(loaderOptions),
-				documentLoader: rsaKeyPair == null ? this.documentLoaderFactory(loaderOptions) : this.authenticatedDocumentLoaderFactory(rsaKeyPair, loaderOptions),
-				tracerProvider: this.tracerProvider
-			});
+			const activity = await parseActivity();
 			try {
 				await this.onOutboxError?.(error, activity);
 			} catch (error) {
@@ -2510,7 +2740,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 					error
 				});
 			}
-			if (error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode)) {
+			if (circuitDrop != null) {
+				await dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
+				return;
+			}
+			if (isPermanentFailure) {
 				getFederationMetrics(this.meterProvider).recordPermanentFailure(error.inbox, error.statusCode);
 				logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not retrying.", {
 					...logData,
@@ -2520,18 +2754,12 @@ var FederationImpl = class extends FederationBuilderImpl {
 					const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
 					try {
 						await this.outboxPermanentFailureHandler(ctx, {
+							reason: "http",
 							inbox: new URL(message.inbox),
 							activity,
 							error,
 							statusCode: error.statusCode,
-							actorIds: (message.actorIds ?? []).flatMap((id) => {
-								try {
-									return [new URL(id)];
-								} catch {
-									logger.warn("Invalid actorId URL in OutboxMessage: {id}", { id });
-									return [];
-								}
-							})
+							actorIds: getActorIds()
 						});
 					} catch (handlerError) {
 						logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
@@ -2543,17 +2771,25 @@ var FederationImpl = class extends FederationBuilderImpl {
 				recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
 				return;
 			}
-			if (this.outboxQueue?.nativeRetrial) {
+			if (circuitHold != null && getPolicyDelay() != null) {
+				logger.error("Failed to send activity {activityId} to {inbox}; holding because the remote host circuit is open:\n{error}", {
+					...logData,
+					error
+				});
+				recordCircuitBreakerHeldSpanEvent(span, circuitHold.remoteHost, circuitHold.state);
+				const circuit = this.circuitBreaker;
+				await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince);
+				return;
+			}
+			if (this.outboxQueue?.nativeRetrial && retryAfterDelay == null) {
 				logger.error("Failed to send activity {activityId} to {inbox}; backend will handle retry:\n{error}", {
 					...logData,
 					error
 				});
 				throw error;
 			}
-			const delay = this.outboxRetryPolicy({
-				elapsedTime: Temporal.Instant.from(message.started).until(Temporal.Now.instant()),
-				attempts: message.attempt
-			});
+			const policyDelay = getPolicyDelay();
+			const delay = policyDelay == null ? null : retryAfterDelay ?? policyDelay;
 			if (delay != null) {
 				logger.error("Failed to send activity {activityId} to {inbox} (attempt #{attempt}); retry...:\n{error}", {
 					...logData,
@@ -2565,7 +2801,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 				};
 				const { outboxQueue } = this;
 				if (outboxQueue != null) {
-					await outboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					await outboxQueue.enqueue(retryMessage, {
+						delay: clampNegativeDelay(delay),
+						orderingKey: message.orderingKey
+					});
 					getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
 						role: "outbox",
 						queue: outboxQueue,
@@ -2654,7 +2893,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						...message,
 						attempt: message.attempt + 1
 					};
-					await this.inboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					await this.inboxQueue.enqueue(retryMessage, { delay: clampNegativeDelay(delay) });
 					if (activityType != null) {
 						getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
 							role: "inbox",