@fedify/fedify 2.4.0-dev.1567 → 2.4.0-dev.1570

package/dist/{middleware-DfBPuuvp.mjs → middleware-CjHqy-1i.mjs}

@@ -1,39 +1,41 @@
 import { Temporal } from "@js-temporal/polyfill";
 import "urlpattern-polyfill";
 globalThis.addEventListener = () => {};
-import { n as version, t as name } from "./deno-Dho0TW0X.mjs";
-import { a as instrumentDocumentLoader, b as recordWebFingerHandle, c as recordCircuitBreakerStateChange, d as recordCollectionRequest, f as recordCollectionTotalItems, g as recordInboxActivity, h as recordFanoutRecipients, i as getRemoteHost, l as recordCollectionDispatchDuration, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionPageItems, v as recordOutboxActivity, x as registerQueueDepthGauge, y as recordOutboxEnqueue } from "./metrics-CR4UD_gF.mjs";
-import { t as formatAcceptSignature } from "./accept-CPkZzmGN.mjs";
-import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-3rYgSl9f.mjs";
-import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-Bf8YhWOi.mjs";
-import { t as getAuthenticatedDocumentLoader } from "./docloader-DCQSgDOg.mjs";
-import { n as kvCache } from "./kv-cache-P3scUO6t.mjs";
-import { _ as wrapContextLoaderForJsonLd, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, l as hasSignature, m as verifyCompactJsonLd, p as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike } from "./ld-CCOPy-Co.mjs";
-import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-TUz_H-M9.mjs";
-import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-L_DbOaFe.mjs";
-import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-CrI3UpuA.mjs";
-import { t as getNodeInfo } from "./client-ByXmQhYD.mjs";
-import { t as nodeInfoToJson } from "./types-J53Kw7so.mjs";
-import { n as extractInboxes, r as sendActivity, t as SendActivityError } from "./send-BBs_erdP.mjs";
-import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-DGWCUaNQ.mjs";
-import { t as CircuitBreaker } from "./circuit-breaker-CSWsyoef.mjs";
-import { t as buildCollectionSynchronizationHeader } from "./collection-Cc3DVAhE.mjs";
-import { t as KvKeyCache } from "./keycache-BeU0LCII.mjs";
-import { t as acceptsJsonLd } from "./negotiation-DDstyBvc.mjs";
-import { t as hasMalformedKnownTemporalLiteral } from "./temporal-bBIy2G-F.mjs";
-import { t as createExponentialBackoffPolicy } from "./retry-CXg_MBI-.mjs";
+import { n as version, t as name } from "./deno-klNxGW7W.mjs";
+import { a as instrumentDocumentLoader, b as recordWebFingerHandle, c as recordCircuitBreakerStateChange, d as recordCollectionRequest, f as recordCollectionTotalItems, g as recordInboxActivity, h as recordFanoutRecipients, i as getRemoteHost, l as recordCollectionDispatchDuration, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionPageItems, v as recordOutboxActivity, x as registerQueueDepthGauge, y as recordOutboxEnqueue } from "./metrics-CLSPm4dZ.mjs";
+import { t as formatAcceptSignature } from "./accept-CgDcxvjV.mjs";
+import { a as importJwk, o as validateCryptoKey, t as exportJwk } from "./key-o0_ipvUR.mjs";
+import { l as verifyRequest, o as parseRfc9421SignatureInput, u as verifyRequestDetailed } from "./http-JdRPUYku.mjs";
+import { t as getAuthenticatedDocumentLoader } from "./docloader-CuFH35j1.mjs";
+import { n as kvCache } from "./kv-cache-CM7w20U5.mjs";
+import { _ as wrapContextLoaderForJsonLd, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, l as hasSignature, m as verifyCompactJsonLd, p as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike } from "./ld-C__CEgQH.mjs";
+import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-gz4TUfKG.mjs";
+import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-BNL8AC14.mjs";
+import { i as verifyObject, n as hasProofLike, r as signObject } from "./proof-CuQpZ_AT.mjs";
+import { t as getNodeInfo } from "./client-B_A6mfn3.mjs";
+import { t as nodeInfoToJson } from "./types-BFowWFTT.mjs";
+import { n as extractInboxes, r as sendActivity, t as SendActivityError } from "./send-B60teucv.mjs";
+import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-CJnWbgbj.mjs";
+import { t as CircuitBreaker } from "./circuit-breaker-hJBB6jwA.mjs";
+import { t as buildCollectionSynchronizationHeader } from "./collection-Dp5ky45w.mjs";
+import { t as KvKeyCache } from "./keycache-CaOR6NYg.mjs";
+import { t as acceptsJsonLd } from "./negotiation-m_L2nyS3.mjs";
+import { t as hasMalformedKnownTemporalLiteral } from "./temporal-CvVMrvJJ.mjs";
+import { t as createExponentialBackoffPolicy } from "./retry-DI4O-zYl.mjs";
+import { t as TaskCodec } from "./codec-nfBU4Up4.mjs";
+import { n as ParallelMessageQueue } from "./mq-03vS-C9P.mjs";
 import { getLogger, withContext } from "@logtape/logtape";
 import { RouterError } from "@fedify/uri-template";
 import { Activity, Collection, CollectionPage, CryptographicKey, Link, Multikey, Object as Object$1, OrderedCollection, OrderedCollectionPage, Tombstone, getTypeId, lookupObject, traverseCollection } from "@fedify/vocab";
 import { lookupWebFinger } from "@fedify/webfinger";
 import { SpanKind, SpanStatusCode, context, metrics, propagation, trace } from "@opentelemetry/api";
+import { uniq } from "es-toolkit";
 import { FetchError, getDocumentLoader } from "@fedify/vocab-runtime";
 import { ATTR_HTTP_REQUEST_HEADER, ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_HEADER, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_URL_FULL } from "@opentelemetry/semantic-conventions";
-import { uniq } from "es-toolkit";
 import { DataPointType, MeterProvider as MeterProvider$1, MetricReader } from "@opentelemetry/sdk-metrics";
 import { domainToASCII } from "node:url";
 //#region src/compat/transformers.ts
-const logger$1 = getLogger([
+const logger$2 = getLogger([
 	"fedify",
 	"compat",
 	"transformers"
@@ -59,7 +61,7 @@ const logger$1 = getLogger([
 function autoIdAssigner(activity, context) {
 	if (activity.id != null) return activity;
 	const id = new URL(`/#${activity.constructor.name}/${crypto.randomUUID()}`, context.origin);
-	logger$1.warn("As the activity to send does not have an id, a new id {id} has been generated for it.  However, it is recommended to explicitly set the id for the activity.", { id: id.href });
+	logger$2.warn("As the activity to send does not have an id, a new id {id} has been generated for it.  However, it is recommended to explicitly set the id for the activity.", { id: id.href });
 	return activity.clone({ id });
 }
 /**
@@ -2093,6 +2095,199 @@ const MIN_COMPONENTS = [
 	"@authority"
 ];
 //#endregion
+//#region src/federation/tasks/enqueue.ts
+/**
+* The enqueue pipeline for custom background tasks.  `ContextImpl.enqueueTask`
+* and `ContextImpl.enqueueTaskMany` delegate to {@link enqueueTasks} so the
+* handle validation, deduplication planning, payload encoding, and queue
+* dispatch live in one cohesive place instead of one oversized method.
+*
+* @module
+*/
+/**
+* Validates the task handle, plans deduplication, encodes every payload, then
+* dispatches the resulting messages to the queue.  A single item flows through
+* the same pipeline as a batch, so {@link Context.enqueueTask} and
+* {@link Context.enqueueTaskMany} share one implementation.
+* @template TContextData The context data to pass to the {@link Context}.
+* @template TData The type of the task payload, inferred from the task's schema.
+* @param ctx The enqueueing dependencies assembled by `ContextImpl`.
+* @param task The handle returned by `defineTask()`.
+* @param items The payloads to enqueue, in order.
+* @param options The enqueue options governing delay, ordering, and dedup.
+* @internal
+*/
+const enqueueTasks = (ctx) => async function(task, items, options) {
+	const def = ctx.federation.taskDefinitions.get(task.name);
+	if (def == null || def.handle !== task) throw new TypeError(`Task ${JSON.stringify(task.name)} is not defined on this federation; pass a handle returned by its defineTask().`);
+	const queue = ctx.federation.resolveTaskQueue(task.name);
+	if (queue == null) throw new TypeError("No message queue is configured for tasks; pass `queue` to createFederation() or to defineTask().");
+	if (items.length < 1) return;
+	const plan = planDeduplication(ctx, queue, task.name, options, items.length);
+	const messages = await Promise.all(items.map(encodeTaskMessage(ctx.codec, ctx.origin, task, options)));
+	const claim = await claimDeduplication(ctx, plan, task.name);
+	if (!claim.proceed) return;
+	if (!ctx.federation.manuallyStartQueue) ctx.federation._startQueueInternal(ctx.data);
+	try {
+		await dispatch(queue, messages, {
+			delay: getDurationIfDefined(options.delay),
+			orderingKey: options.orderingKey,
+			deduplicationKey: claim.forwardedDeduplicationKey
+		}, {
+			meterProvider: ctx.federation.meterProvider,
+			taskName: task.name
+		});
+	} catch (error) {
+		if (claim.rollback != null) try {
+			await claim.rollback();
+		} catch (rollbackError) {
+			logger$1.warn("Failed to roll back the deduplication marker for task {taskName} after a failed enqueue; it will expire by TTL. {rollbackError}", {
+				taskName: task.name,
+				rollbackError
+			});
+		}
+		throw error;
+	}
+};
+const getDurationIfDefined = (item) => item == null ? void 0 : Temporal.Duration.from(item);
+/**
+* Decides how a `deduplicationKey` (if any) is honored: forwarded to a native
+* queue, claimed via `cas`, or—when neither is available—dropped or rejected
+* per the federation's `taskDeduplicationFallback`.  Throws the fail-fast
+* `TypeError`s so they precede the encode.
+*/
+function planDeduplication(ctx, queue, taskName, options, itemCount) {
+	if (options.deduplicationKey == null) return { kind: "none" };
+	const key = options.deduplicationKey;
+	const native = queue.nativeDeduplication === true;
+	const canCas = ctx.federation.kv.cas != null;
+	const canBatchAtomically = queue.enqueueMany != null && !(queue instanceof ParallelMessageQueue && queue.queue.enqueueMany == null);
+	if (itemCount > 1 && !canBatchAtomically && (native || canCas)) throw new TypeError(`Task ${JSON.stringify(taskName)} was enqueued as a batch with a deduplicationKey, but its message queue does not implement enqueueMany; a multi-item batch cannot be deduplicated atomically without it.  Implement enqueueMany on the queue, or enqueue the tasks individually with enqueueTask().`);
+	if (native) return {
+		kind: "native",
+		key
+	};
+	if (canCas) return {
+		kind: "cas",
+		key
+	};
+	if (ctx.federation.taskDeduplicationFallback === "closed") throw new TypeError("deduplicationKey was set but the message queue does not declare nativeDeduplication and the key-value store exposes no conditional write (cas); set taskDeduplicationFallback to \"open\" to proceed without deduplication, or use a backend that supports it.");
+	return {
+		kind: "open",
+		key
+	};
+}
+/**
+* Executes the planned deduplication once the payloads are encoded.  A native
+* plan forwards its key to the queue; a `cas` plan claims the marker and stops
+* the enqueue when it loses the race; an `open` plan logs and proceeds.
+* @returns Whether to proceed, and the key (if any) to forward to the queue.
+*/
+async function claimDeduplication(ctx, plan, taskName) {
+	switch (plan.kind) {
+		case "native": return {
+			proceed: true,
+			forwardedDeduplicationKey: plan.key
+		};
+		case "cas": {
+			const cacheKey = [...ctx.federation.kvPrefixes.taskDeduplication, plan.key];
+			const token = crypto.randomUUID();
+			if (!await ctx.federation.kv.cas(cacheKey, void 0, token, { ttl: ctx.federation.taskDeduplicationTtl })) return { proceed: false };
+			return {
+				proceed: true,
+				rollback: async () => {
+					await ctx.federation.kv.cas(cacheKey, token, void 0);
+				}
+			};
+		}
+		case "open": logger$1.debug("deduplicationKey {deduplicationKey} for task {taskName} ignored: the message queue declares no nativeDeduplication and the key-value store has no cas; proceeding (taskDeduplicationFallback is \"open\").", {
+			deduplicationKey: plan.key,
+			taskName
+		});
+		case "none": return { proceed: true };
+		default: throw new TypeError(`Unknown deduplication plan: ${JSON.stringify(plan)}`);
+	}
+}
+/**
+* Sends the encoded messages to the queue, picking the bulk path when the
+* queue implements `enqueueMany` and otherwise fanning out parallel single
+* enqueues.  The fan-out drops `deduplicationKey`, which is only ever set for a
+* native plan that the bulk paths already cover.
+*/
+async function dispatch(queue, messages, options, { meterProvider, taskName }) {
+	const metrics = getFederationMetrics(meterProvider);
+	if (messages.length === 1) await queue.enqueue(messages[0], options);
+	else if (queue.enqueueMany != null) await queue.enqueueMany(messages, options);
+	else {
+		const fanoutOptions = {
+			delay: options.delay,
+			orderingKey: options.orderingKey
+		};
+		const rejected = (await Promise.allSettled(messages.map(async (message) => {
+			await queue.enqueue(message, fanoutOptions);
+			metrics.recordQueueTaskEnqueued({
+				role: "task",
+				queue,
+				taskName
+			}, message.attempt);
+		}))).find((result) => result.status === "rejected");
+		if (rejected != null) throw rejected.reason;
+		return;
+	}
+	metrics.recordQueueTaskEnqueued({
+		role: "task",
+		queue,
+		taskName
+	}, messages[0].attempt, messages.length);
+}
+/**
+* Builds the per-payload encoder: validates and serializes the payload, then
+* stamps the message envelope with a fresh id, the context's origin, and the
+* active trace context.  Curried so the batch encode reuses one bound encoder.
+*/
+const encodeTaskMessage = (codec, origin, task, options) => async (data) => {
+	const encoded = await codec.encode(task.schema, data);
+	const carrier = {};
+	propagation.inject(context.active(), carrier);
+	return {
+		type: "task",
+		id: crypto.randomUUID(),
+		baseUrl: origin,
+		taskName: task.name,
+		data: encoded,
+		started: Temporal.Now.instant().toString(),
+		attempt: 0,
+		orderingKey: options.orderingKey,
+		traceContext: carrier
+	};
+};
+const logger$1 = getLogger([
+	"fedify",
+	"federation",
+	"task"
+]);
+//#endregion
+//#region src/federation/tasks/error.ts
+var TaskRetryEnqueueError = class extends Error {
+	constructor(cause) {
+		super("Failed to re-enqueue a custom task for a retry.", { cause });
+		this.name = "TaskRetryEnqueueError";
+	}
+};
+const classifyAbortableError = (error) => isAbortError(error) ? { outcome: "aborted" } : {
+	outcome: "failed",
+	error
+};
+const classifyTaskError = (error) => isAbortError(error) ? { outcome: "aborted" } : error instanceof TaskRetryEnqueueError ? {
+	outcome: "failed",
+	failureReason: "retry_enqueue",
+	error: error.cause ?? error
+} : {
+	outcome: "failed",
+	failureReason: "handler",
+	error
+};
+//#endregion
 //#region src/federation/webfinger.ts
 const logger = getLogger([
 	"fedify",
@@ -2418,9 +2613,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 	inboxQueue;
 	outboxQueue;
 	fanoutQueue;
-	inboxQueueStarted;
-	outboxQueueStarted;
-	fanoutQueueStarted;
+	taskQueue;
+	startedQueues;
 	manuallyStartQueue;
 	origin;
 	documentLoaderFactory;
@@ -2434,6 +2628,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 	skipSignatureVerification;
 	outboxRetryPolicy;
 	inboxRetryPolicy;
+	taskRetryPolicy;
+	taskQueueResolution;
+	taskDeduplicationTtl;
+	taskDeduplicationFallback;
 	circuitBreaker;
 	activityTransformers;
 	_tracerProvider;
@@ -2473,20 +2671,24 @@ var FederationImpl = class extends FederationBuilderImpl {
 			httpMessageSignaturesSpec: ["_fedify", "httpMessageSignaturesSpec"],
 			acceptSignatureNonce: ["_fedify", "acceptSignatureNonce"],
 			circuitBreaker: ["_fedify", "circuit"],
+			taskDeduplication: ["_fedify", "taskDeduplication"],
 			...options.kvPrefixes ?? {}
 		};
 		if (options.queue == null) {
 			this.inboxQueue = void 0;
 			this.outboxQueue = void 0;
 			this.fanoutQueue = void 0;
+			this.taskQueue = void 0;
 		} else if ("enqueue" in options.queue && "listen" in options.queue) {
 			this.inboxQueue = options.queue;
 			this.outboxQueue = options.queue;
 			this.fanoutQueue = options.queue;
+			this.taskQueue = void 0;
 		} else {
 			this.inboxQueue = options.queue.inbox;
 			this.outboxQueue = options.queue.outbox;
 			this.fanoutQueue = options.queue.fanout;
+			this.taskQueue = options.queue.task;
 		}
 		if (options.circuitBreaker !== false && this.outboxQueue != null) {
 			this.circuitBreaker = new CircuitBreaker({
@@ -2507,9 +2709,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				]).warn("The configured key-value store does not support CAS; outbound delivery circuit breaker updates may race under concurrent workers.");
 			}
 		}
-		this.inboxQueueStarted = false;
-		this.outboxQueueStarted = false;
-		this.fanoutQueueStarted = false;
+		this.startedQueues = /* @__PURE__ */ new Set();
 		this.manuallyStartQueue = options.manuallyStartQueue ?? false;
 		if (options.origin != null) if (typeof options.origin === "string") {
 			if (!URL.canParse(options.origin) || !options.origin.match(/^https?:\/\//)) throw new TypeError(`Invalid origin: ${JSON.stringify(options.origin)}`);
@@ -2591,6 +2791,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 		this.inboxChallengePolicy = options.inboxChallengePolicy;
 		this.outboxRetryPolicy = options.outboxRetryPolicy ?? createExponentialBackoffPolicy();
 		this.inboxRetryPolicy = options.inboxRetryPolicy ?? createExponentialBackoffPolicy();
+		this.taskRetryPolicy = options.taskRetryPolicy ?? createExponentialBackoffPolicy();
+		this.taskQueueResolution = options.taskQueueResolution ?? "fallback";
+		this.taskDeduplicationTtl = Temporal.Duration.from(options.taskDeduplicationTtl ?? { hours: 1 });
+		this.taskDeduplicationFallback = options.taskDeduplicationFallback ?? "open";
 		this.activityTransformers = options.activityTransformers ?? getDefaultActivityTransformers();
 		this._tracerProvider = options.tracerProvider;
 		if (benchmarkMode) {
@@ -2623,6 +2827,9 @@ var FederationImpl = class extends FederationBuilderImpl {
 		this.#registerQueueDepthGauge(meterProvider);
 		return meterProvider;
 	}
+	get metrics() {
+		return getFederationMetrics(this.meterProvider);
+	}
 	#registerQueueDepthGauge(meterProvider) {
 		if (meterProvider === this.#queueDepthGaugeMeterProvider) return;
 		registerQueueDepthGauge(meterProvider, this.#queueDepthGaugeEntries, { sourceId: this.#queueDepthGaugeSourceId });
@@ -2639,35 +2846,38 @@ var FederationImpl = class extends FederationBuilderImpl {
 	_getTracer() {
 		return this.tracerProvider.getTracer(name, version);
 	}
+	resolveTaskQueue(taskName) {
+		const resolved = this.taskDefinitions.get(taskName)?.queue ?? this.taskQueue;
+		if (resolved != null) return resolved;
+		return this.taskQueueResolution === "strict" ? void 0 : this.outboxQueue;
+	}
 	async _startQueueInternal(ctxData, signal, queue) {
-		if (this.inboxQueue == null && this.outboxQueue == null) return;
+		const taskQueue = this.taskQueue ?? (this.taskQueueResolution === "fallback" ? this.outboxQueue : void 0);
+		const customQueues = this.taskDefinitions.values().map((def) => ["task", def.queue]);
+		const targets = [
+			["inbox", this.inboxQueue],
+			["outbox", this.outboxQueue],
+			["fanout", this.fanoutQueue],
+			["task", taskQueue],
+			...customQueues
+		];
 		const logger = getLogger([
 			"fedify",
 			"federation",
 			"queue"
 		]);
 		const promises = [];
-		if (this.inboxQueue != null && (queue == null || queue === "inbox") && !this.inboxQueueStarted) {
-			logger.debug("Starting an inbox task worker.");
-			this.inboxQueueStarted = true;
-			promises.push(this.inboxQueue.listen((msg) => this.processQueuedTask(ctxData, msg), { signal }));
-		}
-		if (this.outboxQueue != null && this.outboxQueue !== this.inboxQueue && (queue == null || queue === "outbox") && !this.outboxQueueStarted) {
-			logger.debug("Starting an outbox task worker.");
-			this.outboxQueueStarted = true;
-			promises.push(this.outboxQueue.listen((msg) => this.processQueuedTask(ctxData, msg), { signal }));
-		}
-		if (this.fanoutQueue != null && this.fanoutQueue !== this.inboxQueue && this.fanoutQueue !== this.outboxQueue && (queue == null || queue === "fanout") && !this.fanoutQueueStarted) {
-			logger.debug("Starting a fanout task worker.");
-			this.fanoutQueueStarted = true;
-			promises.push(this.fanoutQueue.listen((msg) => this.processQueuedTask(ctxData, msg), { signal }));
+		for (const [role, target] of targets) {
+			if (target == null || !(queue == null || queue === role)) continue;
+			if (this.startedQueues.has(target)) continue;
+			this.startedQueues.add(target);
+			logger.debug("Starting a {role} queue worker.", { role });
+			promises.push(target.listen((msg) => this.processQueuedTask(ctxData, msg), { signal }));
 		}
 		await Promise.all(promises);
 	}
 	processQueuedTask(contextData, message) {
-		const tracer = this._getTracer();
 		const extractedContext = propagation.extract(context.active(), message.traceContext);
-		const meter = getFederationMetrics(this.meterProvider);
 		return withContext({ messageId: message.id }, async () => {
 			if (message.type === "fanout") {
 				const common = {
@@ -2675,117 +2885,108 @@ var FederationImpl = class extends FederationBuilderImpl {
 					queue: this.fanoutQueue,
 					activityType: message.activityType
 				};
-				await tracer.startActiveSpan("activitypub.fanout", {
-					kind: SpanKind.CONSUMER,
-					attributes: { "activitypub.activity.type": message.activityType }
-				}, extractedContext, async (span) => {
-					const spanCtx = span.spanContext();
-					return await withContext({
-						traceId: spanCtx.traceId,
-						spanId: spanCtx.spanId
-					}, async () => {
-						if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-						meter.recordQueueTaskStarted(common);
-						meter.incrementQueueTaskInFlight(common);
-						const startedAt = performance.now();
-						let outcome = "completed";
-						try {
-							await this.#listenFanoutMessage(contextData, message);
-						} catch (e) {
-							const aborted = isAbortError(e);
-							outcome = aborted ? "aborted" : "failed";
-							if (!aborted) span.setStatus({
-								code: SpanStatusCode.ERROR,
-								message: String(e)
-							});
-							throw e;
-						} finally {
-							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
-							meter.decrementQueueTaskInFlight(common);
-							span.end();
-						}
-					});
-				});
+				await this.#runWorkerSpan("activitypub.fanout", { "activitypub.activity.type": message.activityType }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
+					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
+					await this.#listenFanoutMessage(contextData, message);
+					return { outcome: "completed" };
+				}, classifyAbortableError));
 			} else if (message.type === "outbox") {
 				const common = {
 					role: "outbox",
 					queue: this.outboxQueue,
 					activityType: message.activityType
 				};
-				await tracer.startActiveSpan("activitypub.outbox", {
-					kind: SpanKind.CONSUMER,
-					attributes: {
-						"activitypub.activity.type": message.activityType,
-						"activitypub.activity.retries": message.attempt
-					}
-				}, extractedContext, async (span) => {
-					const spanCtx = span.spanContext();
-					return await withContext({
-						traceId: spanCtx.traceId,
-						spanId: spanCtx.spanId
-					}, async () => {
-						if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-						meter.recordQueueTaskStarted(common);
-						meter.incrementQueueTaskInFlight(common);
-						const startedAt = performance.now();
-						let outcome = "completed";
-						try {
-							await this.#listenOutboxMessage(contextData, message, span);
-						} catch (e) {
-							const aborted = isAbortError(e);
-							outcome = aborted ? "aborted" : "failed";
-							if (!aborted) span.setStatus({
-								code: SpanStatusCode.ERROR,
-								message: String(e)
-							});
-							throw e;
-						} finally {
-							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
-							meter.decrementQueueTaskInFlight(common);
-							span.end();
-						}
-					});
-				});
+				await this.#runWorkerSpan("activitypub.outbox", {
+					"activitypub.activity.type": message.activityType,
+					"activitypub.activity.retries": message.attempt
+				}, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
+					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
+					await this.#listenOutboxMessage(contextData, message, span);
+					return { outcome: "completed" };
+				}, classifyAbortableError));
 			} else if (message.type === "inbox") {
 				const common = {
 					role: "inbox",
 					queue: this.inboxQueue
 				};
-				await tracer.startActiveSpan("activitypub.inbox", {
-					kind: SpanKind.CONSUMER,
-					attributes: { "activitypub.shared_inbox": message.identifier == null }
-				}, extractedContext, async (span) => {
-					const spanCtx = span.spanContext();
-					return await withContext({
-						traceId: spanCtx.traceId,
-						spanId: spanCtx.spanId
-					}, async () => {
-						meter.recordQueueTaskStarted(common);
-						meter.incrementQueueTaskInFlight(common);
-						const startedAt = performance.now();
-						let outcome = "completed";
-						try {
-							await this.#listenInboxMessage(contextData, message, span, (activityType) => {
-								common.activityType = activityType;
-							});
-						} catch (e) {
-							const aborted = isAbortError(e);
-							outcome = aborted ? "aborted" : "failed";
-							if (!aborted) span.setStatus({
-								code: SpanStatusCode.ERROR,
-								message: String(e)
-							});
-							throw e;
-						} finally {
-							meter.recordQueueTaskOutcome(common, outcome, getDurationMs(startedAt));
-							meter.decrementQueueTaskInFlight(common);
-							span.end();
-						}
+				await this.#runWorkerSpan("activitypub.inbox", { "activitypub.shared_inbox": message.identifier == null }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
+					await this.#listenInboxMessage(contextData, message, span, (activityType) => {
+						common.activityType = activityType;
 					});
-				});
+					return { outcome: "completed" };
+				}, classifyAbortableError));
+			} else if (message.type === "task") {
+				const registered = this.taskDefinitions.get(message.taskName) != null;
+				const common = {
+					role: "task",
+					queue: this.resolveTaskQueue(message.taskName),
+					taskName: registered ? message.taskName : void 0
+				};
+				await this.#runWorkerSpan("fedify.task", {
+					"fedify.task.name": message.taskName,
+					"fedify.task.attempt": message.attempt
+				}, extractedContext, this.#instrumentWorkerBody(common, () => this.#listenTaskMessage(contextData, message), classifyTaskError));
 			}
 		});
 	}
+	/**
+	* Opens a CONSUMER span for a queue worker and scopes the extracted trace
+	* context around its body.  The body—typically built by
+	* {@link FederationImpl.instrumentWorkerBody}—receives the span and returns
+	* the context-scoped worker callback.
+	* @param spanName The consumer span name (for example, `"fedify.task"`).
+	* @param spanAttributes The initial span attributes.
+	* @param extractedContext The trace context extracted from the message.
+	* @param instrumentedBody Builds the context-scoped worker callback for the
+	*                         opened span.
+	*/
+	async #runWorkerSpan(spanName, spanAttributes, extractedContext, instrumentedBody) {
+		await this._getTracer().startActiveSpan(spanName, {
+			kind: SpanKind.CONSUMER,
+			attributes: spanAttributes
+		}, extractedContext, async (span) => {
+			const spanCtx = span.spanContext();
+			return await withContext({
+				traceId: spanCtx.traceId,
+				spanId: spanCtx.spanId
+			}, instrumentedBody(span));
+		});
+	}
+	/**
+	* Wraps a worker body with the shared `fedify.queue.task.*` boundary
+	* telemetry, returning the instrumented body keyed by its span.  The
+	* instrumentation records the started, outcome, and duration metrics, pairs
+	* the in-flight increment with its decrement, sets the span's error status
+	* from the terminal outcome, and ends the span.  The fanout, outbox, inbox,
+	* and task workers differ only in `common`, the body, and how a thrown error
+	* maps to an outcome.
+	* @param common The shared queue-task metric attributes for this worker.
+	* @param run Runs the worker body and resolves to its terminal outcome.
+	* @param classifyError Maps a thrown error to a terminal outcome.
+	*/
+	#instrumentWorkerBody = (common, run, classifyError) => (span) => async () => {
+		this.metrics.recordQueueTaskStarted(common);
+		this.metrics.incrementQueueTaskInFlight(common);
+		const startedAt = performance.now();
+		let result = { outcome: "completed" };
+		try {
+			result = await run(span);
+		} catch (e) {
+			result = classifyError(e);
+			throw e;
+		} finally {
+			if (result.outcome === "failed") {
+				if (result.failureReason != null) span.setAttribute("fedify.task.failure_reason", result.failureReason);
+				span.setStatus({
+					code: SpanStatusCode.ERROR,
+					...result.error == null ? {} : { message: String(result.error) }
+				});
+			}
+			this.metrics.recordQueueTaskOutcome(common, result.outcome, getDurationMs(startedAt), result.outcome === "failed" ? result.failureReason : void 0);
+			this.metrics.decrementQueueTaskInFlight(common);
+			span.end();
+		}
+	};
 	async #listenFanoutMessage(data, message) {
 		getLogger([
 			"fedify",
@@ -2875,7 +3076,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				delay: clampNegativeDelay(delay),
 				orderingKey: message.orderingKey
 			});
-			getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+			this.metrics.recordQueueTaskEnqueued({
 				role: "outbox",
 				queue: outboxQueue,
 				activityType: heldMessage.activityType
@@ -3054,7 +3255,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				return;
 			}
 			if (isPermanentFailure) {
-				getFederationMetrics(this.meterProvider).recordPermanentFailure(error.inbox, error.statusCode);
+				this.metrics.recordPermanentFailure(error.inbox, error.statusCode);
 				logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not retrying.", {
 					...logData,
 					status: error.statusCode
@@ -3114,7 +3315,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						delay: clampNegativeDelay(delay),
 						orderingKey: message.orderingKey
 					});
-					getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+					this.metrics.recordQueueTaskEnqueued({
 						role: "outbox",
 						queue: outboxQueue,
 						activityType: retryMessage.activityType
@@ -3204,7 +3405,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 					};
 					await this.inboxQueue.enqueue(retryMessage, { delay: clampNegativeDelay(delay) });
 					if (activityType != null) {
-						getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued({
+						this.metrics.recordQueueTaskEnqueued({
 							role: "inbox",
 							queue: this.inboxQueue,
 							activityType
@@ -3331,7 +3532,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				try {
 					await listener(context.toInboxContext(message.identifier, message.activity, activity.id?.href, activityType), activity);
 				} finally {
-					getFederationMetrics(this.meterProvider).recordInboxProcessingDuration(activityType, getDurationMs(started));
+					this.metrics.recordInboxProcessingDuration(activityType, getDurationMs(started));
 				}
 				recordInboxActivity(this.meterProvider, "processed", activityType);
 			} catch (error) {
@@ -3347,6 +3548,105 @@ var FederationImpl = class extends FederationBuilderImpl {
 			listenerSpan.end();
 		});
 	}
+	async #listenTaskMessage(contextData, message) {
+		const logger = getLogger([
+			"fedify",
+			"federation",
+			"task"
+		]);
+		const def = this.taskDefinitions.get(message.taskName);
+		if (def == null) {
+			logger.warn("Received a custom task {taskName} with no registered handler; dropping.", { taskName: message.taskName });
+			return {
+				outcome: "failed",
+				failureReason: "unknown_task"
+			};
+		}
+		const context = this.#createContext(new URL(message.baseUrl), contextData);
+		const data = await context.codec.decode(def.schema, message.data);
+		if (!data.ok) {
+			if (data.phase === "deserialization") logger.error("Custom task {taskName} payload could not be deserialized; dropping:\n{error}", {
+				taskName: message.taskName,
+				error: data.error
+			});
+			else logger.error("Custom task {taskName} payload failed schema validation; dropping:\n{error}", {
+				taskName: message.taskName,
+				error: data.error
+			});
+			return {
+				outcome: "failed",
+				failureReason: data.phase
+			};
+		}
+		try {
+			await def.handler(context, data.value);
+			return { outcome: "completed" };
+		} catch (error) {
+			if (def.onError != null) try {
+				await def.onError(context, error, data.value);
+			} catch (onErrorError) {
+				logger.error("onError for custom task {taskName} threw:\n{error}", {
+					taskName: message.taskName,
+					error: onErrorError
+				});
+			}
+			const queue = this.resolveTaskQueue(def.name);
+			if (queue?.nativeRetrial) throw error;
+			const retryPolicy = def.retryPolicy ?? this.taskRetryPolicy;
+			let elapsedTime = Temporal.Duration.from({ seconds: 0 });
+			try {
+				elapsedTime = Temporal.Instant.from(message.started).until(Temporal.Now.instant());
+			} catch (parseError) {
+				logger.error("Custom task {taskName} has an unparsable started time {started}; treating elapsedTime as zero:\n{error}", {
+					taskName: message.taskName,
+					started: message.started,
+					error: parseError
+				});
+			}
+			const delay = retryPolicy({
+				elapsedTime,
+				attempts: message.attempt
+			});
+			if (delay != null && queue != null) {
+				logger.error("Custom task {taskName} failed (attempt #{attempt}); retry...:\n{error}", {
+					taskName: message.taskName,
+					attempt: message.attempt,
+					error
+				});
+				const retryMessage = {
+					...message,
+					attempt: message.attempt + 1
+				};
+				try {
+					await queue.enqueue(retryMessage, {
+						delay: clampNegativeDelay(delay),
+						orderingKey: message.orderingKey
+					});
+				} catch (enqueueError) {
+					logger.error("Custom task {taskName} could not be re-enqueued for a retry (attempt #{attempt}):\n{error}", {
+						taskName: message.taskName,
+						attempt: retryMessage.attempt,
+						error: enqueueError
+					});
+					throw new TaskRetryEnqueueError(error);
+				}
+				this.metrics.recordQueueTaskEnqueued({
+					role: "task",
+					queue,
+					taskName: message.taskName
+				}, retryMessage.attempt);
+				return { outcome: "completed" };
+			} else logger.error("Custom task {taskName} failed after {attempt} attempts; giving up:\n{error}", {
+				taskName: message.taskName,
+				attempt: message.attempt,
+				error
+			});
+			return isAbortError(error) ? { outcome: "aborted" } : {
+				outcome: "failed",
+				failureReason: "handler"
+			};
+		}
+	}
 	startQueue(contextData, options = {}) {
 		return this._startQueueInternal(contextData, options.signal, options.queue);
 	}
@@ -3580,7 +3880,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						});
 						if (acceptsJsonLd(request)) response.headers.set("Vary", "Accept");
 					} catch (error) {
-						getFederationMetrics(this.meterProvider).recordHttpServerRequest(request.method, metricState.endpoint ?? "error", getDurationMs(metricStart), { routeTemplate: metricState.routeTemplate });
+						this.metrics.recordHttpServerRequest(request.method, metricState.endpoint ?? "error", getDurationMs(metricStart), { routeTemplate: metricState.routeTemplate });
 						span.setStatus({
 							code: SpanStatusCode.ERROR,
 							message: `${error}`
@@ -3593,7 +3893,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						});
 						throw error;
 					}
-					getFederationMetrics(this.meterProvider).recordHttpServerRequest(request.method, metricState.endpoint ?? "error", getDurationMs(metricStart), {
+					this.metrics.recordHttpServerRequest(request.method, metricState.endpoint ?? "error", getDurationMs(metricStart), {
 						statusCode: response.status,
 						routeTemplate: metricState.routeTemplate
 					});
@@ -3931,6 +4231,7 @@ var ContextImpl = class ContextImpl {
 	documentLoader;
 	contextLoader;
 	invokedFromActorKeyPairsDispatcher;
+	#codec;
 	constructor({ url, federation, data, documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher }) {
 		this.url = url;
 		this.federation = federation;
@@ -3939,6 +4240,18 @@ var ContextImpl = class ContextImpl {
 		this.contextLoader = contextLoader;
 		this.invokedFromActorKeyPairsDispatcher = invokedFromActorKeyPairsDispatcher;
 	}
+	/**
+	* A {@link TaskCodec} bound to this context's loaders, used to encode
+	* and decode custom task payloads.  Lazily created and cached so a context
+	* that never enqueues or dispatches a task pays nothing.
+	* @internal
+	*/
+	get codec() {
+		return this.#codec ??= new TaskCodec(this);
+	}
+	get #enqueueTasks() {
+		return enqueueTasks(this);
+	}
 	clone(data) {
 		return new ContextImpl({
 			url: this.url,
@@ -4257,6 +4570,12 @@ var ContextImpl = class ContextImpl {
 			allowPrivateAddress: this.federation.allowPrivateAddress
 		});
 	}
+	async enqueueTask(task, data, options = {}) {
+		await this.#enqueueTasks(task, [data], options);
+	}
+	async enqueueTaskMany(task, payloads, options = {}) {
+		await this.#enqueueTasks(task, payloads, options);
+	}
 	sendActivity(sender, recipients, activity, options = {}) {
 		return this.tracerProvider.getTracer(name, version).startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
 			kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,