@fedify/fedify 2.3.0-dev.1069 → 2.3.0-dev.1099

package/dist/{middleware-DsGmXfXz.mjs → middleware-BXnhAGF9.mjs}

@@ -1,31 +1,31 @@
 import { Temporal } from "@js-temporal/polyfill";
 import "urlpattern-polyfill";
 globalThis.addEventListener = () => {};
-import { n as RouterError } from "./router-CrMLXoOr.mjs";
-import { n as version, t as name } from "./deno-CFXqOz6w.mjs";
+import { n as RouterError } from "./router-BT_F5748.mjs";
+import { n as version, t as name } from "./deno-DBabeupC.mjs";
 import { t as formatAcceptSignature } from "./accept-CceiKpCy.mjs";
-import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-CpAxygvh.mjs";
-import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-QHJGzUe8.mjs";
-import { t as getAuthenticatedDocumentLoader } from "./docloader-QNtAtTZF.mjs";
-import { n as kvCache } from "./kv-cache-Bmv7tUzz.mjs";
-import { a as signJsonLd, i as hasSignatureLike, o as verifyJsonLd, r as detachSignature } from "./ld-Ce_vkKjG.mjs";
-import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-DmgzyItA.mjs";
-import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-Bi7n-dEy.mjs";
-import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-C3q2IhUr.mjs";
-import { t as getNodeInfo } from "./client-CIiz1WX7.mjs";
+import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-D9dUsyow.mjs";
+import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-5G18W3NP.mjs";
+import { t as getAuthenticatedDocumentLoader } from "./docloader-DA5FzJOR.mjs";
+import { n as kvCache } from "./kv-cache-CiiNwT6W.mjs";
+import { a as signJsonLd, i as hasSignatureLike, o as verifyJsonLd, r as detachSignature } from "./ld-hbxDLO1k.mjs";
+import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-DwJe0BH9.mjs";
+import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-BgFLCJQ_.mjs";
+import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-erpV_J_n.mjs";
+import { t as getNodeInfo } from "./client-B_A6mfn3.mjs";
 import { t as nodeInfoToJson } from "./types-BFowWFTT.mjs";
-import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-DQ2zYTeA.mjs";
+import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-BkRRjxzb.mjs";
 import { t as buildCollectionSynchronizationHeader } from "./collection-CA3V5zyK.mjs";
-import { a as getFederationMetrics, i as getDurationMs, n as extractInboxes, o as getRemoteHost, r as sendActivity, t as SendActivityError } from "./send-CTi2iPXp.mjs";
+import { a as getFederationMetrics, c as recordOutboxEnqueue, i as getDurationMs, n as extractInboxes, o as getRemoteHost, r as sendActivity, s as isAbortError, t as SendActivityError } from "./send-BOwz4Hw5.mjs";
 import { t as KvKeyCache } from "./keycache-BeU0LCII.mjs";
-import { t as acceptsJsonLd } from "./negotiation-VnHNB0Q5.mjs";
+import { t as acceptsJsonLd } from "./negotiation-DDstyBvc.mjs";
 import { t as createExponentialBackoffPolicy } from "./retry-_VvV0h9f.mjs";
+import { getLogger, withContext } from "@logtape/logtape";
 import { Activity, Collection, CollectionPage, CryptographicKey, Link, Multikey, Object as Object$1, OrderedCollection, OrderedCollectionPage, Tombstone, getTypeId, lookupObject, traverseCollection } from "@fedify/vocab";
 import { lookupWebFinger } from "@fedify/webfinger";
 import { SpanKind, SpanStatusCode, context, metrics, propagation, trace } from "@opentelemetry/api";
 import { uniq } from "es-toolkit";
 import { FetchError, getDocumentLoader } from "@fedify/vocab-runtime";
-import { getLogger, withContext } from "@logtape/logtape";
 import { ATTR_HTTP_REQUEST_HEADER, ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_HEADER, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_URL_FULL } from "@opentelemetry/semantic-conventions";
 import { domainToASCII } from "node:url";
 //#region src/compat/transformers.ts
@@ -230,6 +230,11 @@ async function routeActivity({ context: ctx, json, activity, recipient, inboxLis
 			});
 			throw error;
 		}
+		getFederationMetrics(meterProvider).recordQueueTaskEnqueued({
+			role: "inbox",
+			queue,
+			activityType: getTypeId(activity).href
+		}, 0);
 		logger.info("Activity {activityId} is enqueued.", {
 			activityId: activity.id?.href,
 			activity: json,
@@ -1117,6 +1122,14 @@ async function _handleOrderedCollection(request, { name, values, context, tracer
 * @since 1.8.0
 */
 var CustomCollectionHandler = class {
+	name;
+	values;
+	context;
+	callbacks;
+	tracerProvider;
+	Collection;
+	CollectionPage;
+	filterPredicate;
 	/**
 	* The tracer for telemetry.
 	* @type {Tracer}
@@ -1863,78 +1876,123 @@ var FederationImpl = class extends FederationBuilderImpl {
 	processQueuedTask(contextData, message) {
 		const tracer = this._getTracer();
 		const extractedContext = propagation.extract(context.active(), message.traceContext);
+		const meter = getFederationMetrics(this.meterProvider);
 		return withContext({ messageId: message.id }, async () => {
-			if (message.type === "fanout") await tracer.startActiveSpan("activitypub.fanout", {
-				kind: SpanKind.CONSUMER,
-				attributes: { "activitypub.activity.type": message.activityType }
-			}, extractedContext, async (span) => {
-				const spanCtx = span.spanContext();
-				return await withContext({
-					traceId: spanCtx.traceId,
-					spanId: spanCtx.spanId
-				}, async () => {
-					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-					try {
-						await this.#listenFanoutMessage(contextData, message);
-					} catch (e) {
-						span.setStatus({
-							code: SpanStatusCode.ERROR,
-							message: String(e)
-						});
-						throw e;
-					} finally {
-						span.end();
-					}
+			if (message.type === "fanout") {
+				const common = {
+					role: "fanout",
+					queue: this.fanoutQueue,
+					activityType: message.activityType
+				};
+				await tracer.startActiveSpan("activitypub.fanout", {
+					kind: SpanKind.CONSUMER,
+					attributes: { "activitypub.activity.type": message.activityType }
+				}, extractedContext, async (span) => {
+					const spanCtx = span.spanContext();
+					return await withContext({
+						traceId: spanCtx.traceId,
+						spanId: spanCtx.spanId
+					}, async () => {
+						if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
+						meter.recordQueueTaskStarted(common);
+						meter.incrementQueueTaskInFlight(common);
+						const startedAt = performance.now();
+						let outcome = "completed";
+						try {
+							await this.#listenFanoutMessage(contextData, message);
+						} catch (e) {
+							const aborted = isAbortError(e);
+							outcome = aborted ? "aborted" : "failed";
+							if (!aborted) span.setStatus({
+								code: SpanStatusCode.ERROR,
+								message: String(e)
+							});
+							throw e;
+						} finally {
+							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
+							meter.decrementQueueTaskInFlight(common);
+							span.end();
+						}
+					});
 				});
-			});
-			else if (message.type === "outbox") await tracer.startActiveSpan("activitypub.outbox", {
-				kind: SpanKind.CONSUMER,
-				attributes: {
-					"activitypub.activity.type": message.activityType,
-					"activitypub.activity.retries": message.attempt
-				}
-			}, extractedContext, async (span) => {
-				const spanCtx = span.spanContext();
-				return await withContext({
-					traceId: spanCtx.traceId,
-					spanId: spanCtx.spanId
-				}, async () => {
-					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-					try {
-						await this.#listenOutboxMessage(contextData, message, span);
-					} catch (e) {
-						span.setStatus({
-							code: SpanStatusCode.ERROR,
-							message: String(e)
-						});
-						throw e;
-					} finally {
-						span.end();
+			} else if (message.type === "outbox") {
+				const common = {
+					role: "outbox",
+					queue: this.outboxQueue,
+					activityType: message.activityType
+				};
+				await tracer.startActiveSpan("activitypub.outbox", {
+					kind: SpanKind.CONSUMER,
+					attributes: {
+						"activitypub.activity.type": message.activityType,
+						"activitypub.activity.retries": message.attempt
 					}
+				}, extractedContext, async (span) => {
+					const spanCtx = span.spanContext();
+					return await withContext({
+						traceId: spanCtx.traceId,
+						spanId: spanCtx.spanId
+					}, async () => {
+						if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
+						meter.recordQueueTaskStarted(common);
+						meter.incrementQueueTaskInFlight(common);
+						const startedAt = performance.now();
+						let outcome = "completed";
+						try {
+							await this.#listenOutboxMessage(contextData, message, span);
+						} catch (e) {
+							const aborted = isAbortError(e);
+							outcome = aborted ? "aborted" : "failed";
+							if (!aborted) span.setStatus({
+								code: SpanStatusCode.ERROR,
+								message: String(e)
+							});
+							throw e;
+						} finally {
+							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
+							meter.decrementQueueTaskInFlight(common);
+							span.end();
+						}
+					});
 				});
-			});
-			else if (message.type === "inbox") await tracer.startActiveSpan("activitypub.inbox", {
-				kind: SpanKind.CONSUMER,
-				attributes: { "activitypub.shared_inbox": message.identifier == null }
-			}, extractedContext, async (span) => {
-				const spanCtx = span.spanContext();
-				return await withContext({
-					traceId: spanCtx.traceId,
-					spanId: spanCtx.spanId
-				}, async () => {
-					try {
-						await this.#listenInboxMessage(contextData, message, span);
-					} catch (e) {
-						span.setStatus({
-							code: SpanStatusCode.ERROR,
-							message: String(e)
-						});
-						throw e;
-					} finally {
-						span.end();
-					}
+			} else if (message.type === "inbox") {
+				const common = {
+					role: "inbox",
+					queue: this.inboxQueue
+				};
+				await tracer.startActiveSpan("activitypub.inbox", {
+					kind: SpanKind.CONSUMER,
+					attributes: { "activitypub.shared_inbox": message.identifier == null }
+				}, extractedContext, async (span) => {
+					const spanCtx = span.spanContext();
+					return await withContext({
+						traceId: spanCtx.traceId,
+						spanId: spanCtx.spanId
+					}, async () => {
+						meter.recordQueueTaskStarted(common);
+						meter.incrementQueueTaskInFlight(common);
+						const startedAt = performance.now();
+						let outcome = "completed";
+						try {
+							await this.#listenInboxMessage(contextData, message, span, (activityType) => {
+								common.activityType = activityType;
+							});
+						} catch (e) {
+							const aborted = isAbortError(e);
+							outcome = aborted ? "aborted" : "failed";
+							if (!aborted) span.setStatus({
+								code: SpanStatusCode.ERROR,
+								message: String(e)
+							});
+							throw e;
+						} finally {
+							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
+							meter.decrementQueueTaskInFlight(common);
+							span.end();
+						}
+					});
 				});
-			});
+			}
 		});
 	}
 	async #listenFanoutMessage(data, message) {
@@ -2091,10 +2149,19 @@ var FederationImpl = class extends FederationBuilderImpl {
 					...logData,
 					error
 				});
-				await this.outboxQueue?.enqueue({
+				const retryMessage = {
 					...message,
 					attempt: message.attempt + 1
-				}, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+				};
+				const { outboxQueue } = this;
+				if (outboxQueue != null) {
+					await outboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+						role: "outbox",
+						queue: outboxQueue,
+						activityType: retryMessage.activityType
+					}, retryMessage.attempt);
+				}
 			} else logger.error("Failed to send activity {activityId} to {inbox} after {attempt} attempts; giving up:\n{error}", {
 				...logData,
 				error
@@ -2103,7 +2170,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}
 		logger.info("Successfully sent activity {activityId} to {inbox}.", { ...logData });
 	}
-	async #listenInboxMessage(ctxData, message, span) {
+	async #listenInboxMessage(ctxData, message, span, onActivityType) {
 		const logger = getLogger([
 			"fedify",
 			"federation",
@@ -2117,7 +2184,9 @@ var FederationImpl = class extends FederationBuilderImpl {
 			if (identity != null) context = this.#createContext(baseUrl, ctxData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
 		}
 		const activity = await Activity.fromJsonLd(message.activity, context);
-		span.setAttribute("activitypub.activity.type", getTypeId(activity).href);
+		const activityType = getTypeId(activity).href;
+		span.setAttribute("activitypub.activity.type", activityType);
+		onActivityType?.(activityType);
 		if (activity.id != null) span.setAttribute("activitypub.activity.id", activity.id.href);
 		const cacheKey = activity.id == null ? null : [
 			...this.kvPrefixes.activityIdempotence,
@@ -2145,7 +2214,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				});
 				span.setStatus({
 					code: SpanStatusCode.ERROR,
-					message: `Unsupported activity type: ${getTypeId(activity).href}`
+					message: `Unsupported activity type: ${activityType}`
 				});
 				span.end();
 				return;
@@ -2153,7 +2222,6 @@ var FederationImpl = class extends FederationBuilderImpl {
 			const { class: cls, listener } = dispatched;
 			span.updateName(`activitypub.dispatch_inbox_listener ${cls.name}`);
 			try {
-				const activityType = getTypeId(activity).href;
 				const started = performance.now();
 				try {
 					await listener(context.toInboxContext(message.identifier, message.activity, activity.id?.href, activityType), activity);
@@ -2198,10 +2266,19 @@ var FederationImpl = class extends FederationBuilderImpl {
 						activity: message.activity,
 						recipient: message.identifier
 					});
-					await this.inboxQueue?.enqueue({
+					const retryMessage = {
 						...message,
 						attempt: message.attempt + 1
-					}, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+					};
+					const { inboxQueue } = this;
+					if (inboxQueue != null) {
+						await inboxQueue.enqueue(retryMessage, { delay: Temporal.Duration.compare(delay, { seconds: 0 }) < 0 ? Temporal.Duration.from({ seconds: 0 }) : delay });
+						getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+							role: "inbox",
+							queue: inboxQueue,
+							activityType
+						}, retryMessage.attempt);
+					}
 				} else logger.error("Failed to process the incoming activity {activityId} after {trial} attempts; giving up:\n{error}", {
 					error,
 					activityId: activity.id?.href,
@@ -2390,9 +2467,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 			});
 		}
 		const { outboxQueue } = this;
-		if (outboxQueue.enqueueMany == null) {
-			const promises = messages.map((m) => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }));
-			const errors = (await Promise.allSettled(promises)).filter((r) => r.status === "rejected").map((r) => r.reason);
+		if (outboxQueue.enqueueMany == null || orderingKey != null) {
+			const errors = (await Promise.allSettled(messages.map(async (m) => {
+				await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
+				recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message);
+			}))).filter((r) => r.status === "rejected").map((r) => r.reason);
 			if (errors.length > 0) {
 				logger.error("Failed to enqueue activity {activityId} to send later: {errors}", {
 					activityId: activity.id.href,
@@ -2401,25 +2480,17 @@ var FederationImpl = class extends FederationBuilderImpl {
 				if (errors.length > 1) throw new AggregateError(errors, `Failed to enqueue activity ${activityId} to send later.`);
 				throw errors[0];
 			}
-		} else if (orderingKey != null) {
-			const promises = messages.map((m) => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }));
-			const errors = (await Promise.allSettled(promises)).filter((r) => r.status === "rejected").map((r) => r.reason);
-			if (errors.length > 0) {
-				logger.error("Failed to enqueue activity {activityId} to send later: {errors}", {
+		} else {
+			try {
+				await outboxQueue.enqueueMany(messages.map((m) => m.message));
+			} catch (error) {
+				logger.error("Failed to enqueue activity {activityId} to send later: {error}", {
 					activityId: activity.id.href,
-					errors
+					error
 				});
-				if (errors.length > 1) throw new AggregateError(errors, `Failed to enqueue activity ${activityId} to send later.`);
-				throw errors[0];
+				throw error;
 			}
-		} else try {
-			await outboxQueue.enqueueMany(messages.map((m) => m.message));
-		} catch (error) {
-			logger.error("Failed to enqueue activity {activityId} to send later: {error}", {
-				activityId: activity.id.href,
-				error
-			});
-			throw error;
+			for (const m of messages) recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message);
 		}
 	}
 	fetch(request, options) {
@@ -3235,6 +3306,11 @@ var ContextImpl = class ContextImpl {
 		};
 		if (!this.federation.manuallyStartQueue) this.federation._startQueueInternal(this.data);
 		await this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey });
+		getFederationMetrics(this.federation.meterProvider).recordQueueTaskEnqueued({
+			role: "fanout",
+			queue: this.federation.fanoutQueue,
+			activityType: message.activityType
+		}, 0);
 		return true;
 	}
 	async *getFollowers(identifier) {
@@ -3365,6 +3441,7 @@ var ContextImpl = class ContextImpl {
 			kvPrefixes: this.federation.kvPrefixes,
 			queue: this.federation.inboxQueue,
 			span,
+			meterProvider: this.federation.meterProvider,
 			tracerProvider: options.tracerProvider ?? this.tracerProvider,
 			idempotencyStrategy: this.federation.idempotencyStrategy
 		});
@@ -3614,8 +3691,10 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 	}
 	const { outboxQueue } = ctx.federation;
 	if (outboxQueue.enqueueMany == null || orderingKey != null) {
-		const promises = messages.map((m) => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }));
-		const errors = (await Promise.allSettled(promises)).filter((r) => r.status === "rejected").map((r) => r.reason);
+		const errors = (await Promise.allSettled(messages.map(async (m) => {
+			await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
+			recordOutboxEnqueue(ctx.federation.meterProvider, outboxQueue, m.message);
+		}))).filter((r) => r.status === "rejected").map((r) => r.reason);
 		if (errors.length > 0) {
 			logger.error("Failed to enqueue activity {activityId} to forward later:\n{errors}", {
 				activityId: ctx.activityId,
@@ -3624,14 +3703,17 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			if (errors.length > 1) throw new AggregateError(errors, `Failed to enqueue activity ${ctx.activityId} to forward later.`);
 			throw errors[0];
 		}
-	} else try {
-		await outboxQueue.enqueueMany(messages.map((m) => m.message));
-	} catch (error) {
-		logger.error("Failed to enqueue activity {activityId} to forward later:\n{error}", {
-			activityId: ctx.activityId,
-			error
-		});
-		throw error;
+	} else {
+		try {
+			await outboxQueue.enqueueMany(messages.map((m) => m.message));
+		} catch (error) {
+			logger.error("Failed to enqueue activity {activityId} to forward later:\n{error}", {
+				activityId: ctx.activityId,
+				error
+			});
+			throw error;
+		}
+		for (const m of messages) recordOutboxEnqueue(ctx.federation.meterProvider, outboxQueue, m.message);
 	}
 	return true;
 }