@fedify/fedify 2.5.0-dev.2597 → 2.5.0-dev.2614

package/dist/{middleware-D02eC0j7.mjs → middleware-BkRqMGPU.mjs}

@@ -5,48 +5,832 @@ import { d as isPortableUri, f as isSameObjectId, i as getPortableDid, l as isPo
 import { n as isPubliclyAddressedNode, t as isInAudience } from "./audience-CHQ-FlLD.mjs";
 import { i as getEmptyAuthenticationEvidence, n as deserializeAuthentication, s as toProofPrincipal, t as createAuthenticationEvidence } from "./inbox-authentication-CgonVRkg.mjs";
 import { a as verificationObservation, r as snapshotKey, t as observeAttempt } from "./verification-Dpfcw8ml.mjs";
-import { n as version, t as name } from "./deno-4cGKg59R.mjs";
-import { a as instrumentDocumentLoader, b as recordWebFingerHandle, c as recordCircuitBreakerStateChange, d as recordCollectionRequest, f as recordCollectionTotalItems, g as recordInboxActivity, h as recordFanoutRecipients, i as getRemoteHost, l as recordCollectionDispatchDuration, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionPageItems, v as recordOutboxActivity, x as registerQueueDepthGauge, y as recordOutboxEnqueue } from "./metrics-BvV60TwE.mjs";
+import { n as version, t as name } from "./deno-CRXki9-m.mjs";
+import { a as instrumentDocumentLoader, b as recordWebFingerHandle, c as recordCircuitBreakerStateChange, d as recordCollectionRequest, f as recordCollectionTotalItems, g as recordInboxActivity, h as recordFanoutRecipients, i as getRemoteHost, l as recordCollectionDispatchDuration, n as getDurationMs, o as isAbortError, r as getFederationMetrics, u as recordCollectionPageItems, v as recordOutboxActivity, x as registerQueueDepthGauge, y as recordOutboxEnqueue } from "./metrics-CTteRnEr.mjs";
 import { t as formatAcceptSignature } from "./accept-CPVqki78.mjs";
 import { r as getVerifiedKeyOwnerEvidence } from "./key-owner-evidence-APB6NpSk.mjs";
-import { D as validateCryptoKey, E as importJwk, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, g as verifyCompactJsonLdDetailed, l as hasSignature, m as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike, x as exportJwk, y as wrapContextLoaderForJsonLd } from "./ld-CmAQYDWE.mjs";
+import { D as validateCryptoKey, E as importJwk, a as compactJsonLd, c as getNormalizationContextLoader, d as isClearlyMalformedContextReference, f as isInvalidUrlTypeError, g as verifyCompactJsonLdDetailed, l as hasSignature, m as signJsonLd, r as assertSafeJsonLd, s as detachSignature, t as InvalidContextReferenceError, u as hasSignatureLike, x as exportJwk, y as wrapContextLoaderForJsonLd } from "./ld-C5JOBBFj.mjs";
 import { r as normalizeOutgoingActivityJsonLd } from "./outgoing-jsonld-4IJqPYos.mjs";
-import { a as hasProofLike, d as verifyPortableObjectProof, i as getProofSigner, l as verifyObject, p as verifyPortableObjectProofWithRoot, s as signObject, t as InvalidPortableObjectIdError, u as verifyObjectDetailed } from "./proof-CzWCRhnC.mjs";
-import { c as parseRfc9421SignatureInput, f as validateMaxSignatures, l as selectRequestSignature, m as verifyRequestDetailed, o as listRequestSignatures, p as verifyRequest } from "./http-D5n4S1oL.mjs";
-import { n as getAuthenticatedDocumentLoader } from "./docloader-BiHD6NSO.mjs";
-import { n as kvCache } from "./kv-cache-BV55jAE6.mjs";
+import { a as hasProofLike, d as verifyPortableObjectProof, i as getProofSigner, l as verifyObject, p as verifyPortableObjectProofWithRoot, s as signObject, t as InvalidPortableObjectIdError, u as verifyObjectDetailed } from "./proof-tAb5_Dq4.mjs";
+import { c as parseRfc9421SignatureInput, f as validateMaxSignatures, l as selectRequestSignature, m as verifyRequestDetailed, o as listRequestSignatures, p as verifyRequest } from "./http-DqOXkYbX.mjs";
+import { n as getAuthenticatedDocumentLoader } from "./docloader-Ca6i6ddp.mjs";
+import { n as kvCache } from "./kv-cache-DJsxu9nI.mjs";
+import { a as isActorlessActivityAuthenticated, c as createInboxRequestSource, d as invocationProvenance, f as outboundProvenance, g as sourceLogContext, h as snapshotLocalContext, i as getInboxIdempotencyKey, l as emptyProvenance, m as serializeProvenance, n as checkActorlessActivity, o as isInboxActivityProcessed, p as restoreProvenance, s as routeActivity, t as actorsHaveSameOrigin, u as getOutboundProvenance } from "./inbox-C4iW1g0F.mjs";
+import { t as projectSenderPayload } from "./outbound-document-BMgwLXTY.mjs";
+import { t as captureResponseBody } from "./outbound-capture-DVgnRzlf.mjs";
+import { a as snapshotHeaders, n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-B2H8GsyI.mjs";
 import { d as MAX_BODY_SIZE, f as readBoundedText, t as getNodeInfo, u as BodyTooLargeError } from "./client-5SIFzW3q.mjs";
 import { t as nodeInfoToJson } from "./types-DE_LV1JB.mjs";
-import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-C1-csjTo.mjs";
-import { t as verifyPortableObject } from "./portable-collection-C0MNEEED.mjs";
-import { a as sendActivityThroughGateways, i as sendActivity, n as extractInboxes, o as getOrderingDestination, t as SendActivityError } from "./send-C9-dwpEH.mjs";
-import { n as FederationBuilderImpl, t as ACTOR_ALIAS_PREFIX } from "./builder-Da_V_LDj.mjs";
+import { n as getKeyOwner, t as doesActorOwnKey } from "./owner-CKlRIkeF.mjs";
+import { t as verifyPortableObject } from "./portable-collection-B7RBHVGl.mjs";
+import { a as sendActivityThroughGateways, i as sendActivity, n as extractInboxes, o as getOrderingDestination, t as SendActivityError } from "./send-DzfFREML.mjs";
 import { t as CircuitBreaker } from "./circuit-breaker-XJ53md4S.mjs";
 import { t as buildCollectionSynchronizationHeader } from "./collection-CnnP1UAv.mjs";
-import { a as inspectCompoundPortableObjectApplicability, c as verifyServedPortableObjects, i as findUnsupportedCompoundProofShape, o as verifyCompoundPortableObjectProofs, r as findEmbeddedProofWithoutContext, t as containsCompoundPortableObject } from "./compound-proof-BqqP38Sf.mjs";
-import { a as isActorlessActivityAuthenticated, i as getInboxIdempotencyKey, n as checkActorlessActivity, o as isInboxActivityProcessed, s as routeActivity, t as actorsHaveSameOrigin } from "./inbox-CElAglrC.mjs";
-import { c as expandDocument, d as parseSerializedVerifiedActivityView, f as restoreVerifiedActivityView, i as createReleasableLoader, l as getActorPresence, n as buildVerifiedActivityView, o as deepFreeze, r as createContextSnapshot, t as ContextSnapshotMissError } from "./inbox-view-BTUuIIf6.mjs";
+import { a as inspectCompoundPortableObjectApplicability, c as verifyServedPortableObjects, i as findUnsupportedCompoundProofShape, o as verifyCompoundPortableObjectProofs, r as findEmbeddedProofWithoutContext, t as containsCompoundPortableObject } from "./compound-proof-BavXW-ow.mjs";
+import { c as expandDocument, d as parseSerializedVerifiedActivityView, f as restoreVerifiedActivityView, i as createReleasableLoader, l as getActorPresence, n as buildVerifiedActivityView, o as deepFreeze, r as createContextSnapshot, t as ContextSnapshotMissError } from "./inbox-view-Bu_aSwFJ.mjs";
 import { t as KvKeyCache } from "./keycache-N2opu4-3.mjs";
 import { t as acceptsJsonLd } from "./negotiation-CCocWT_N.mjs";
-import { t as hasMalformedKnownTemporalLiteral } from "./temporal-lDCl8Zxn.mjs";
+import { t as hasMalformedKnownTemporalLiteral } from "./temporal-D9Nt8Fm4.mjs";
 import { t as createExponentialBackoffPolicy } from "./retry-zX5xw5e0.mjs";
 import { t as TaskCodec } from "./codec-C6rTrpIp.mjs";
 import { getLogger, withContext } from "@logtape/logtape";
 import { RouterError } from "@fedify/uri-template";
 import { Activity, Collection, CollectionPage, CryptographicKey, Link, Multikey, Object as Object$1, OrderedCollection, OrderedCollectionPage, Tombstone, getTypeId, isActor, lookupObject, lookupObserver, traverseCollection } from "@fedify/vocab";
 import { lookupWebFinger } from "@fedify/webfinger";
-import { SpanKind, SpanStatusCode, context, metrics, propagation, trace } from "@opentelemetry/api";
+import { SpanKind, SpanStatusCode, context, isSpanContextValid, metrics, propagation, trace } from "@opentelemetry/api";
 import { FetchError, canonicalizePortableUri, decodeMultibase, formatIri, fromCompatibleEf61Id, getDocumentLoader, getFe34Origin, isGatewayUrl, parseDigestMultibase, parseHashlink, parseIri, resolveDocumentLoaderTimeout, toCompatibleEf61Id } from "@fedify/vocab-runtime";
 import { isCompatibleEf61Iri } from "@fedify/vocab-runtime/internal/portable-dereference";
 import { uniq } from "es-toolkit";
 import { ATTR_HTTP_REQUEST_HEADER, ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_HEADER, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_URL_FULL } from "@opentelemetry/semantic-conventions";
+import { encodeHex } from "byte-encodings/hex";
 import jsonld from "@fedify/vocab-runtime/jsonld";
 import { DataPointType, MeterProvider as MeterProvider$1, MetricReader } from "@opentelemetry/sdk-metrics";
 import { domainToASCII } from "node:url";
+//#region src/federation/outbound-observation.ts
+function snapshotDeliveryIdentity(identity) {
+	if (identity == null || typeof identity.deliveryId !== "string" || identity.deliveryId.length < 1 || typeof identity.destinationInbox !== "string" || identity.destinationInbox.length < 1 || identity.broadcast == null || !(identity.broadcast.type === "unknown" || identity.broadcast.type === "known" && typeof identity.broadcast.id === "string" && identity.broadcast.id.length > 0) || !(identity.portableInbox == null || typeof identity.portableInbox === "string")) throw new TypeError("Invalid outbound delivery identity in queue metadata.");
+	return Object.freeze({
+		broadcast: Object.freeze(identity.broadcast.type === "known" ? {
+			type: "known",
+			id: identity.broadcast.id
+		} : { type: "unknown" }),
+		deliveryId: identity.deliveryId,
+		destinationInbox: identity.destinationInbox,
+		...identity.portableInbox == null ? {} : { portableInbox: identity.portableInbox }
+	});
+}
+async function legacyFanoutIdentity(taskId, inbox, portableInbox) {
+	const bytes = new TextEncoder().encode(JSON.stringify([
+		"fedify-legacy-fanout-delivery",
+		taskId,
+		inbox
+	]));
+	const digest = await crypto.subtle.digest("SHA-256", bytes);
+	return {
+		broadcast: { type: "unknown" },
+		deliveryId: `legacy-fanout:${encodeHex(new Uint8Array(digest))}`,
+		destinationInbox: inbox,
+		...portableInbox == null ? {} : { portableInbox }
+	};
+}
+async function notifyObserver(context, callback, event, unit) {
+	if (callback == null) return;
+	try {
+		await withContext(sourceLogContext(event.provenance), () => callback(context, event));
+	} catch (error) {
+		logObserverFailure(unit, error);
+	}
+}
+function logObserverFailure(unit, error) {
+	try {
+		getLogger([
+			"fedify",
+			"federation",
+			"outbox"
+		]).error("An unexpected error occurred in outbound {unit} observer:\n{error}", {
+			unit,
+			error
+		});
+	} catch {}
+}
+function warnInvalidDeliveryIdentity(messageId, error) {
+	try {
+		getLogger([
+			"fedify",
+			"federation",
+			"outbox"
+		]).warn("Ignoring invalid outbound delivery identity in queue message {messageId}: {error}", {
+			messageId,
+			error
+		});
+	} catch {}
+}
+function snapshotTrace(span, parentSpanContext) {
+	const spanContext = span.spanContext();
+	if (!isSpanContextValid(spanContext) || spanContext.isRemote || spanContext.traceId === parentSpanContext?.traceId && spanContext.spanId === parentSpanContext.spanId) return void 0;
+	return Object.freeze({
+		traceId: spanContext.traceId,
+		spanId: spanContext.spanId
+	});
+}
+/** State owned by one concrete send, never persisted in queue metadata. */
+var OutboundAttemptObservation = class {
+	inboxUrl;
+	report;
+	reportExchange;
+	headerPolicies;
+	bodyCapture;
+	senderPayload;
+	#attemptId = crypto.randomUUID();
+	#failures = [];
+	#response = Object.freeze({ type: "not-observed" });
+	#finalResponse;
+	#responses = /* @__PURE__ */ new WeakMap();
+	#trace;
+	#finished = false;
+	#exchangeCount = 0;
+	#senderPayload;
+	constructor(inboxUrl, report, reportExchange = void 0, headerPolicies = {}, bodyCapture = void 0, senderPayload = false) {
+		this.inboxUrl = inboxUrl;
+		this.report = report;
+		this.reportExchange = reportExchange;
+		this.headerPolicies = headerPolicies;
+		this.bodyCapture = bodyCapture;
+		this.senderPayload = senderPayload;
+		this.#senderPayload = Object.freeze({
+			stage: "sender-payload",
+			type: "unavailable",
+			reason: senderPayload ? "not-materialized" : "disabled"
+		});
+	}
+	snapshotSenderPayload(input) {
+		if (this.senderPayload) this.#senderPayload = projectSenderPayload(input);
+	}
+	setSpan(span, parentSpanContext) {
+		this.#trace = snapshotTrace(span, parentSpanContext);
+	}
+	async fetch(request, submit) {
+		const exchangeIndex = this.#exchangeCount++;
+		const snapshot = Object.freeze({
+			url: request.url,
+			method: request.method,
+			...this.headerPolicies.requestHeaders == null ? {} : { headers: snapshotHeaders(request.headers, this.headerPolicies.requestHeaders) }
+		});
+		let response;
+		try {
+			response = await submit();
+		} catch (error) {
+			await this.#exchange(exchangeIndex, snapshot, {
+				type: "failed",
+				error
+			});
+			throw error;
+		}
+		const metadata = Object.freeze({
+			status: response.status,
+			statusText: response.statusText,
+			...response.url === "" ? {} : { url: response.url }
+		});
+		const observed = Object.freeze({
+			exchangeIndex,
+			metadata
+		});
+		this.#responses.set(response, observed);
+		this.#response = Object.freeze({
+			type: "undetermined",
+			lastObserved: observed
+		});
+		const responseSnapshot = Object.freeze({
+			...metadata,
+			...this.headerPolicies.responseHeaders == null ? {} : { headers: snapshotHeaders(response.headers, this.headerPolicies.responseHeaders) }
+		});
+		const bodyCapture = this.bodyCapture == null ? void 0 : await captureResponseBody(response, this.bodyCapture, (body, signal) => this.bodyCapture.onBody({
+			attemptId: this.#attemptId,
+			inboxUrl: this.inboxUrl,
+			exchangeIndex,
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			request: snapshot,
+			response: responseSnapshot,
+			body,
+			signal
+		}));
+		if (bodyCapture?.type === "failed" && bodyCapture.phase === "observer") logObserverFailure("response-body", bodyCapture.error);
+		await this.#exchange(exchangeIndex, snapshot, {
+			type: "response",
+			response: responseSnapshot
+		}, bodyCapture);
+		return response;
+	}
+	#exchange(exchangeIndex, request, outcome, bodyCapture) {
+		return this.reportExchange?.({
+			attemptId: this.#attemptId,
+			inboxUrl: this.inboxUrl,
+			exchangeIndex,
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			request,
+			outcome: Object.freeze(outcome),
+			...bodyCapture == null ? {} : { bodyCapture }
+		}) ?? Promise.resolve();
+	}
+	response(response, disposition) {
+		const observed = this.#responses.get(response);
+		if (observed == null) return;
+		if (disposition === "final") {
+			this.#finalResponse = Object.freeze({
+				type: "final",
+				...observed
+			});
+			this.#response = this.#finalResponse;
+		} else this.#response = Object.freeze({
+			type: "intermediate-only",
+			lastObserved: observed
+		});
+	}
+	failure(failure) {
+		this.#failures.push(Object.freeze(failure));
+	}
+	unknown(error) {
+		if (!this.#failures.some((failure) => failure.error === error || failure.phase !== "cleanup")) this.failure({
+			kind: "unknown",
+			phase: "unknown",
+			error
+		});
+	}
+	succeeded() {
+		if (this.#finished) return Promise.resolve();
+		if (this.#finalResponse == null) throw new Error("A successful outbound attempt must have an observed final response.");
+		return this.#finish({
+			type: "succeeded",
+			response: this.#finalResponse
+		});
+	}
+	failed(error) {
+		let failure;
+		let diagnostics;
+		if (this.#response.type === "final" && (this.#response.metadata.status < 200 || this.#response.metadata.status >= 300)) {
+			failure = this.#failures.find((failure) => failure.kind === "http") ?? Object.freeze({
+				kind: "http",
+				phase: "response"
+			});
+			diagnostics = this.#failures.filter((failure) => failure.kind !== "http");
+		} else {
+			failure = this.#failures.find((failure) => failure.phase !== "cleanup") ?? this.#failures[0] ?? Object.freeze({
+				kind: "unknown",
+				phase: "unknown",
+				error
+			});
+			diagnostics = this.#failures.filter((item) => item !== failure);
+		}
+		return this.#finish({
+			type: "failed",
+			failure,
+			error,
+			response: this.#response,
+			...diagnostics.length === 0 ? {} : { diagnostics: Object.freeze(diagnostics) }
+		});
+	}
+	#finish(outcome) {
+		if (this.#finished) return Promise.resolve();
+		this.#finished = true;
+		return this.report({
+			attemptId: this.#attemptId,
+			inboxUrl: this.inboxUrl,
+			httpExchangeCount: this.#exchangeCount,
+			senderPayload: this.#senderPayload,
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			...this.#trace == null ? {} : { trace: this.#trace },
+			outcome: Object.freeze(outcome)
+		});
+	}
+};
+function snapshotOutboxIdentity(message) {
+	if (message.outbound != null) try {
+		return snapshotDeliveryIdentity(message.outbound);
+	} catch (error) {
+		warnInvalidDeliveryIdentity(message.id, error);
+	}
+	return Object.freeze({
+		broadcast: Object.freeze({ type: "unknown" }),
+		deliveryId: message.id,
+		...message.portableInbox == null ? {} : { portableInbox: message.portableInbox }
+	});
+}
+/** Per-execution state, independent of concrete attempt/exchange callbacks. */
+var OutboundWorkerObservation = class {
+	context;
+	callback;
+	provenance;
+	#attemptsStarted = 0;
+	#actions;
+	#trace;
+	#preparationFailure;
+	#preparationFailures = [];
+	#queue;
+	#delivery;
+	#role;
+	#currentContext;
+	constructor(context, callback, message, provenance) {
+		this.context = context;
+		this.callback = callback;
+		this.provenance = provenance;
+		this.#role = message.type;
+		this.#queue = Object.freeze({
+			queueMessageId: message.id,
+			...message.type === "outbox" ? { queueAttempt: message.attempt } : {}
+		});
+		if (message.type === "outbox") this.#delivery = snapshotOutboxIdentity(message);
+	}
+	attemptStarted = () => {
+		this.#attemptsStarted++;
+	};
+	enableActions() {
+		this.#actions ??= [];
+	}
+	directAttemptStarted(inboxUrl, identity) {
+		this.attemptStarted();
+		this.#actions?.push(Object.freeze({
+			type: "handoff",
+			action: "direct",
+			inboxUrls: Object.freeze([inboxUrl]),
+			...identity == null ? {} : { delivery: snapshotDeliveryIdentity(identity) }
+		}));
+	}
+	async enqueueChildren(messages, enqueue) {
+		if (this.#actions == null) return await enqueue();
+		const facts = messages.map((message) => ({
+			action: "enqueue",
+			queue: Object.freeze({
+				queueMessageId: message.id,
+				queueAttempt: message.attempt
+			}),
+			inboxUrls: Object.freeze(message.gatewayInboxes != null && message.gatewayInboxes.length > 1 ? [...message.gatewayInboxes] : [message.inbox]),
+			...message.outbound == null ? {} : { delivery: snapshotDeliveryIdentity(message.outbound) }
+		}));
+		try {
+			await enqueue();
+		} catch (error) {
+			for (const fact of facts) this.#actions.push(Object.freeze({
+				type: "action-failed",
+				...fact,
+				acceptance: "unknown",
+				error
+			}));
+			throw error;
+		}
+		for (const fact of facts) this.#actions.push(Object.freeze({
+			type: "scheduled",
+			...fact
+		}));
+	}
+	async enqueue(action, message, delay, enqueue, preparationFailures) {
+		if (this.#actions == null) return await enqueue();
+		const facts = {
+			action,
+			queue: Object.freeze({
+				queueMessageId: message.id,
+				queueAttempt: message.attempt
+			}),
+			inboxUrls: Object.freeze(message.gatewayInboxes != null && message.gatewayInboxes.length > 1 ? [...message.gatewayInboxes] : [message.inbox]),
+			delay: delay.toString(),
+			...preparationFailures == null || preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...preparationFailures]) },
+			...message.gatewayNotBefore == null ? {} : { notBefore: Object.freeze({ ...message.gatewayNotBefore }) }
+		};
+		try {
+			await enqueue();
+		} catch (error) {
+			this.#actions.push(Object.freeze({
+				type: "action-failed",
+				...facts,
+				acceptance: "unknown",
+				error
+			}));
+			throw error;
+		}
+		this.#actions.push(Object.freeze({
+			type: "scheduled",
+			...facts
+		}));
+	}
+	abandon(reason, inboxUrls, preparationFailures) {
+		this.#actions?.push(Object.freeze({
+			type: "abandoned",
+			reason,
+			inboxUrls: Object.freeze([...inboxUrls]),
+			...preparationFailures == null || preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...preparationFailures]) }
+		}));
+	}
+	dropFailure(inboxUrls, error) {
+		this.#actions?.push(Object.freeze({
+			type: "action-failed",
+			action: "drop",
+			inboxUrls: Object.freeze([...inboxUrls]),
+			error
+		}));
+	}
+	async drop(inboxUrls, operation) {
+		try {
+			await operation();
+		} catch (error) {
+			this.dropFailure(inboxUrls, error);
+			throw error;
+		}
+	}
+	getContext() {
+		return this.#currentContext ??= this.context();
+	}
+	useContext(context) {
+		this.#currentContext = context;
+	}
+	setSpan(span, parentSpanContext) {
+		this.#trace = snapshotTrace(span, parentSpanContext);
+	}
+	async prepare(phase, operation) {
+		try {
+			const result = await operation();
+			this.#preparationFailure = void 0;
+			return result;
+		} catch (error) {
+			this.#preparationFailure = Object.freeze({
+				kind: "local",
+				phase,
+				error
+			});
+			throw error;
+		}
+	}
+	recordPreparationFailure(error, inboxUrl) {
+		const failure = this.#preparationFailure;
+		if (failure == null || !Object.is(failure.error, error)) return [];
+		const diagnostic = Object.freeze({
+			inboxUrl,
+			failure
+		});
+		this.#preparationFailures.push(diagnostic);
+		return [diagnostic];
+	}
+	async run(operation) {
+		try {
+			await operation();
+		} catch (error) {
+			await this.#notify(this.#preparationFailure == null || !Object.is(this.#preparationFailure.error, error) ? {
+				type: "failed",
+				error
+			} : {
+				type: "preparation-failed",
+				failure: this.#preparationFailure,
+				error
+			});
+			throw error;
+		}
+		await this.#notify({ type: "completed" });
+	}
+	async #notify(result) {
+		const observedAt = Temporal.Now.instant().epochMilliseconds;
+		try {
+			const context = this.getContext();
+			await notifyObserver(context, this.callback, Object.freeze({
+				role: this.#role,
+				queue: this.#queue,
+				...this.#delivery == null ? {} : { delivery: this.#delivery },
+				attemptsStarted: this.#attemptsStarted,
+				actions: Object.freeze([...this.#actions ?? []]),
+				...this.#preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...this.#preparationFailures]) },
+				observedAt,
+				...this.#trace == null ? {} : { trace: this.#trace },
+				provenance: this.provenance,
+				local: snapshotLocalContext(context),
+				result: Object.freeze(result)
+			}), "worker");
+		} catch (error) {
+			logObserverFailure("worker", error);
+		}
+	}
+};
+/** Selected callback and restored grouping, shared by concrete gateway attempts. */
+var OutboundAttemptObserver = class OutboundAttemptObserver {
+	context;
+	callback;
+	identity;
+	provenance;
+	onHttpExchange;
+	headerPolicies;
+	bodyCapture;
+	#queue;
+	constructor(context, callback, identity, provenance, queue, onHttpExchange = void 0, headerPolicies = {}, bodyCapture = void 0) {
+		this.context = context;
+		this.callback = callback;
+		this.identity = identity;
+		this.provenance = provenance;
+		this.onHttpExchange = onHttpExchange;
+		this.headerPolicies = headerPolicies;
+		this.bodyCapture = bodyCapture;
+		this.#queue = queue == null ? void 0 : Object.freeze({ ...queue });
+	}
+	static resumeOutbox(context, callback, message, provenance, onHttpExchange, headerPolicies = {}, bodyCapture) {
+		if (callback == null && onHttpExchange == null && bodyCapture == null) return;
+		const identity = snapshotOutboxIdentity(message);
+		return new OutboundAttemptObserver(context, callback, identity, provenance, {
+			queueMessageId: message.id,
+			queueAttempt: message.attempt
+		}, onHttpExchange, headerPolicies, bodyCapture);
+	}
+	startAttempt(inboxUrl) {
+		if (this.callback == null && this.onHttpExchange == null && this.bodyCapture == null) return void 0;
+		const context = this.context();
+		return new OutboundAttemptObservation(inboxUrl, (facts) => notifyObserver(context, this.callback, Object.freeze({
+			...this.identity,
+			...facts,
+			...this.#queue == null ? {} : { queue: this.#queue },
+			provenance: this.provenance,
+			local: snapshotLocalContext(context)
+		}), "attempt"), (facts) => notifyObserver(context, this.onHttpExchange, Object.freeze({
+			...this.identity,
+			...facts,
+			...this.#queue == null ? {} : { queue: this.#queue },
+			provenance: this.provenance,
+			local: snapshotLocalContext(context)
+		}), "http-exchange"), this.onHttpExchange == null && this.bodyCapture == null ? {} : this.headerPolicies, this.bodyCapture == null ? void 0 : {
+			...this.bodyCapture,
+			onBody: (event) => withContext(sourceLogContext(this.provenance), () => this.bodyCapture.onBody(context, Object.freeze({
+				...this.identity,
+				...event,
+				...this.#queue == null ? {} : { queue: this.#queue },
+				provenance: this.provenance,
+				local: snapshotLocalContext(context)
+			})))
+		}, this.callback != null && this.headerPolicies.senderPayload === true);
+	}
+};
+/** One producer's destination, without callbacks or live inputs in queues. */
+var OutboundDeliveryObservation = class {
+	context;
+	observer;
+	provenance;
+	trace;
+	#finished = false;
+	#reference;
+	#plan;
+	#attempts;
+	constructor(context, observer, identity, destination, provenance, trace, onAttempt, onHttpExchange, queue, headerPolicies = {}, bodyCapture) {
+		this.context = context;
+		this.observer = observer;
+		this.provenance = provenance;
+		this.trace = trace;
+		this.#reference = snapshotDeliveryIdentity(identity);
+		this.#attempts = new OutboundAttemptObserver(() => this.context, onAttempt, this.#reference, this.provenance, queue, onHttpExchange, headerPolicies, bodyCapture);
+		this.#plan = Object.freeze({
+			actorIds: Object.freeze(Array.from(destination.actorIds)),
+			sharedInbox: destination.sharedInbox,
+			mode: destination.mode ?? (destination.portableInbox == null ? "ordinary" : "portable-fallback")
+		});
+	}
+	startAttempt(inboxUrl) {
+		return this.#attempts.startAttempt(inboxUrl);
+	}
+	get identity() {
+		return this.#reference;
+	}
+	planned() {
+		return notifyObserver(this.context, this.observer, Object.freeze({
+			...this.trace == null ? {} : { trace: this.trace },
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			provenance: this.provenance,
+			local: snapshotLocalContext(this.context),
+			...this.#reference,
+			...this.#plan,
+			stage: "planned"
+		}), "delivery");
+	}
+	handoff(result) {
+		if (this.#finished) return Promise.resolve();
+		this.#finished = true;
+		return notifyObserver(this.context, this.observer, Object.freeze({
+			...this.trace == null ? {} : { trace: this.trace },
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			provenance: this.provenance,
+			local: snapshotLocalContext(this.context),
+			...this.#reference,
+			stage: "handoff",
+			result: Object.freeze(result)
+		}), "delivery");
+	}
+	failed(error) {
+		if (this.#finished) return Promise.resolve();
+		this.#finished = true;
+		return notifyObserver(this.context, this.observer, Object.freeze({
+			...this.trace == null ? {} : { trace: this.trace },
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			provenance: this.provenance,
+			local: snapshotLocalContext(this.context),
+			...this.#reference,
+			stage: "preparation-failed",
+			failure: Object.freeze({
+				kind: "local",
+				phase: "request-preparation",
+				error
+			})
+		}), "delivery");
+	}
+};
+/** State owned by one outgoing invocation, never serialized into queues. */
+var OutboundBroadcastObservation = class OutboundBroadcastObservation {
+	context;
+	operation;
+	provenance;
+	queue;
+	#broadcast;
+	#finished = false;
+	#phase = "unknown";
+	#planning = "not-started";
+	#destinationCount = 0;
+	#trace;
+	#activity = Object.freeze({});
+	#onBroadcast;
+	#onDelivery;
+	#onAttempt;
+	#onHttpExchange;
+	#headerPolicies;
+	#bodyCapture;
+	#deliveries = /* @__PURE__ */ new Map();
+	constructor(context, observers, operation, provenance = getOutboundProvenance(context), broadcast = {
+		type: "known",
+		id: crypto.randomUUID()
+	}, queue = void 0) {
+		this.context = context;
+		this.operation = operation;
+		this.provenance = provenance;
+		this.queue = queue;
+		this.#broadcast = broadcast;
+		this.#onBroadcast = observers.onBroadcast;
+		this.#onDelivery = observers.onDelivery;
+		this.#onAttempt = observers.onAttempt;
+		this.#onHttpExchange = observers.onHttpExchange;
+		this.#bodyCapture = observers.responseBodyCapture;
+		this.#headerPolicies = {
+			requestHeaders: observers.requestHeaders,
+			responseHeaders: observers.responseHeaders,
+			senderPayload: observers.senderPayload
+		};
+	}
+	/** Continue an existing fanout; do not emit a new broadcast invocation. */
+	static async resumeFanout(context, observers, message, span, parentSpanContext) {
+		if (observers.onDelivery == null && observers.onAttempt == null && observers.onHttpExchange == null && observers.responseBodyCapture == null && observers.onWorker == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null)) return;
+		const observation = new OutboundBroadcastObservation(context, {
+			onDelivery: observers.onDelivery,
+			onAttempt: observers.onAttempt,
+			onHttpExchange: observers.onHttpExchange,
+			requestHeaders: observers.requestHeaders,
+			responseHeaders: observers.responseHeaders,
+			responseBodyCapture: observers.responseBodyCapture,
+			senderPayload: observers.senderPayload
+		}, "send", getOutboundProvenance(context), { type: "unknown" }, { queueMessageId: message.id });
+		if (span != null) observation.setSpan(span, parentSpanContext);
+		observation.#finished = true;
+		observation.planned(Object.keys(message.inboxes).length);
+		for (const [inbox, destination] of Object.entries(message.inboxes)) {
+			let identity;
+			if (destination.outbound != null) try {
+				const snapshot = snapshotDeliveryIdentity(destination.outbound);
+				if (snapshot.destinationInbox !== inbox) throw new TypeError("Outbound delivery identity does not match its fanout destination.");
+				identity = snapshot;
+			} catch (error) {
+				warnInvalidDeliveryIdentity(message.id, error);
+			}
+			identity ??= await legacyFanoutIdentity(message.id, inbox, destination.portableInbox);
+			observation.#deliveries.set(inbox, observation.#createDelivery(inbox, destination, identity));
+		}
+		return observation;
+	}
+	#createDelivery(inbox, destination, identity) {
+		return new OutboundDeliveryObservation(this.context, this.#onDelivery, identity ?? {
+			broadcast: this.#broadcast,
+			deliveryId: crypto.randomUUID(),
+			destinationInbox: inbox,
+			...destination.portableInbox == null ? {} : { portableInbox: destination.portableInbox }
+		}, destination, this.provenance, this.#trace, this.#onAttempt, this.#onHttpExchange, this.queue, this.#headerPolicies, this.#bodyCapture);
+	}
+	continuation(inbox) {
+		const delivery = this.delivery(inbox);
+		return delivery == null ? void 0 : { outbound: delivery.identity };
+	}
+	async planDestinations(inboxes) {
+		this.planned(Object.keys(inboxes).length);
+		for (const [inbox, destination] of Object.entries(inboxes)) if (!this.#deliveries.has(inbox)) this.#deliveries.set(inbox, this.#createDelivery(inbox, destination));
+		for (const delivery of this.#deliveries.values()) await delivery.planned();
+	}
+	delivery(inbox) {
+		return this.#deliveries.get(inbox);
+	}
+	async enqueue(entries, queue, enqueue) {
+		let result;
+		try {
+			result = await enqueue();
+		} catch (error) {
+			await Promise.all(entries.map(({ inbox }) => this.delivery(inbox)?.handoff({
+				type: "enqueue-failed",
+				acceptance: "unknown",
+				error
+			})));
+			throw error;
+		}
+		await Promise.all(entries.map(({ inbox, queueMessageId }) => this.delivery(inbox)?.handoff({
+			type: "enqueued",
+			queue,
+			queueMessageId
+		})));
+		return result;
+	}
+	/** Attach only the span supplied by this execution, never an ambient fallback. */
+	setSpan(span, parentSpanContext) {
+		this.#trace = snapshotTrace(span, parentSpanContext);
+	}
+	setPhase(phase) {
+		this.#phase = phase;
+		if (phase === "destination-planning") this.#planning = "incomplete";
+	}
+	/** Replace the Activity facts captured by the producer path. */
+	setActivity(reference) {
+		this.#activity = Object.freeze({ ...reference });
+	}
+	planned(destinationCount) {
+		this.#planning = "complete";
+		this.#destinationCount = destinationCount;
+	}
+	async planDestination(inbox, destination) {
+		this.#destinationCount++;
+		const delivery = this.#createDelivery(inbox, destination);
+		this.#deliveries.set(inbox, delivery);
+		await delivery.planned();
+	}
+	prepared(preparedFor) {
+		return this.#finish(Object.freeze({
+			type: "prepared",
+			preparedFor,
+			destinationCount: this.#destinationCount
+		}));
+	}
+	skipped(reason = "no-destinations") {
+		return this.#finish(Object.freeze({
+			type: "skipped",
+			reason
+		}));
+	}
+	async run(send) {
+		if (this.#broadcast.type === "known") await this.#notify(Object.freeze({
+			broadcastId: this.#broadcast.id,
+			operation: this.operation,
+			stage: "started",
+			activity: this.#activity
+		}));
+		try {
+			return await send(this, this.provenance);
+		} catch (error) {
+			await this.failed(error);
+			throw error;
+		}
+	}
+	failed(error) {
+		const failure = {
+			kind: this.#phase === "unknown" ? "unknown" : "local",
+			phase: this.#phase,
+			error
+		};
+		return this.#finish(Object.freeze({
+			type: "failed",
+			failure: Object.freeze(failure),
+			planning: this.#planning,
+			plannedDestinationCount: this.#destinationCount
+		}));
+	}
+	async #finish(result) {
+		if (this.#finished) return;
+		this.#finished = true;
+		if (this.#broadcast.type === "known") await this.#notify(Object.freeze({
+			broadcastId: this.#broadcast.id,
+			operation: this.operation,
+			stage: "preparation-finished",
+			result,
+			activity: this.#activity
+		}));
+	}
+	#notify(event) {
+		return notifyObserver(this.context, this.#onBroadcast, Object.freeze({
+			...event,
+			...this.#trace == null ? {} : { trace: this.#trace },
+			observedAt: Temporal.Now.instant().epochMilliseconds,
+			provenance: this.provenance,
+			local: snapshotLocalContext(this.context)
+		}), "broadcast");
+	}
+};
+/** Observe preparation without changing the unobserved send's execution path. */
+function observeOutboundBroadcast(context, observers, send, operation = "send", dispatchId) {
+	let provenance;
+	let invalid;
+	try {
+		provenance = invocationProvenance(context, dispatchId);
+	} catch (error) {
+		provenance = invocationProvenance(context, void 0);
+		invalid = { error };
+	}
+	const execute = async (observation) => {
+		if (invalid != null) {
+			observation?.setPhase("validation");
+			throw invalid.error;
+		}
+		return await send(observation, provenance);
+	};
+	return withContext(sourceLogContext(provenance), () => {
+		if (observers.onBroadcast == null && observers.onDelivery == null && observers.onAttempt == null && observers.onHttpExchange == null && observers.responseBodyCapture == null) return execute();
+		return new OutboundBroadcastObservation(context, observers, operation, provenance).run(execute);
+	});
+}
+//#endregion
 //#region src/federation/inbox-observation.ts
 /** Internal state owned by the ingress boundary, never serialized into queues. */
 var InboxObservation = class {
 	inbox;
+	source;
 	payload = { status: "unavailable" };
 	activity = null;
 	authentication = { status: "notDetermined" };
@@ -69,8 +853,9 @@ var InboxObservation = class {
 			}
 		}
 	};
-	constructor(inbox) {
+	constructor(inbox, source = void 0) {
 		this.inbox = inbox;
+		this.source = source;
 	}
 	/** Run an ingress operation and project telemetry; fetch owns completion. */
 	async run(operation, span) {
@@ -126,7 +911,18 @@ var InboxObservation = class {
 	}
 	async finish(context, handler, outcome) {
 		if (handler == null) return;
+		const signal = context.request.signal;
+		const requestCancellation = signal.aborted ? Object.freeze({
+			type: "aborted",
+			reason: signal.reason
+		}) : Object.freeze({ type: "not-aborted" });
 		const report = {
+			...this.source == null ? {} : {
+				inboxRequestId: this.source.inboxRequestId,
+				requestId: this.source.requestId,
+				local: Object.freeze({ ...this.source.context })
+			},
+			requestCancellation,
 			inbox: this.inbox,
 			payload: this.payload,
 			activity: this.activity,
@@ -135,7 +931,7 @@ var InboxObservation = class {
 			outcome
 		};
 		try {
-			await handler(context, report);
+			await withContext(this.source == null ? {} : { inboxRequestId: this.source.inboxRequestId }, () => handler(context, report));
 		} catch (error) {
 			try {
 				getLogger([
@@ -949,9 +1745,10 @@ function coversDelivery(signature) {
 * other gateways, that gateway has the activity already, so it is claimed
 * without forwarding the activity to it.  See {@link identifySendingGateway}.
 * @param parameters The parameters.
+* @param observation The in-process producer observation, if registered.
 * @returns The target inbox URLs that the activity was handed off to.
 */
-async function forwardPortableInboxActivity(parameters) {
+async function forwardPortableInboxActivity(parameters, observation) {
 	const logger = getLogger([
 		"fedify",
 		"federation",
@@ -960,8 +1757,16 @@ async function forwardPortableInboxActivity(parameters) {
 	const { recipient, activityId, excludedOrigins, kv, kvPrefix, outboxQueue } = parameters;
 	const options = parameters.options ?? resolvePortableInboxForwardingOptions();
 	const { maxTargets, ttl } = options;
-	if (kv.cas == null || maxTargets < 1) return [];
+	if (kv.cas == null) {
+		if (observation != null) await observation.skipped("forwarding-unavailable");
+		return [];
+	}
+	if (maxTargets < 1) {
+		if (observation != null) await observation.skipped("forwarding-disabled");
+		return [];
+	}
 	const deadline = Date.now() + options.deadline.total("millisecond");
+	observation?.setPhase("destination-planning");
 	const canonicalActivityId = getCanonicalPortableId(activityId) ?? activityId.href;
 	const claimKey = (gateway) => [
 		...kvPrefix,
@@ -989,15 +1794,27 @@ async function forwardPortableInboxActivity(parameters) {
 		targets.length = maxTargets;
 	}
 	const claimed = [];
-	for (const { gateway, inbox } of targets) if (await kv.cas(claimKey(gateway), void 0, true, { ttl })) claimed.push(inbox);
-	if (claimed.length < 1) return [];
+	for (const { gateway, inbox } of targets) if (await kv.cas(claimKey(gateway), void 0, true, { ttl })) {
+		claimed.push(inbox);
+		if (observation != null) await observation.planDestination(inbox.href, {
+			actorIds: [recipient.actorId.href],
+			sharedInbox: false,
+			portableInbox: recipient.canonicalInboxId,
+			mode: "portable-replication"
+		});
+	}
+	observation?.planned(claimed.length);
+	if (claimed.length < 1) {
+		if (observation != null) await observation.skipped("no-forwarding-targets");
+		return [];
+	}
 	logger.debug("Forwarding activity {activityId} from the portable inbox {inbox} to other gateways:\n{targets}", {
 		activityId: activityId.href,
 		inbox: recipient.canonicalInboxId,
 		targets: claimed.map((t) => t.href)
 	});
-	if (outboxQueue == null) await forwardImmediately(parameters, claimed, deadline);
-	else await enqueueForwarding(parameters, outboxQueue, claimed);
+	if (outboxQueue == null) await forwardImmediately(parameters, claimed, deadline, observation);
+	else await enqueueForwarding(parameters, outboxQueue, claimed, observation);
 	return claimed;
 }
 /**
@@ -1140,7 +1957,7 @@ async function resolveForwardingKey({ getKeys, activityId }) {
 	}
 	return key;
 }
-async function forwardImmediately(parameters, inboxes, deadlineAt) {
+async function forwardImmediately(parameters, inboxes, deadlineAt, observation) {
 	const { activity, activityId, activityType, allowPrivateAddress, specDeterminer, meterProvider, tracerProvider } = parameters;
 	const logger = getLogger([
 		"fedify",
@@ -1148,23 +1965,50 @@ async function forwardImmediately(parameters, inboxes, deadlineAt) {
 		"inbox"
 	]);
 	const deadline = Math.max(0, deadlineAt - Date.now());
-	const forwarding = resolveForwardingKey(parameters).then((key) => Promise.all(inboxes.map((inbox) => sendActivity({
-		activity,
-		activityId: activityId.href,
-		activityType,
-		keys: key == null ? [] : [key],
-		inbox,
-		allowPrivateAddress,
-		specDeterminer,
-		meterProvider,
-		tracerProvider
-	}).catch((error) => {
-		logger.error("Failed to forward activity {activityId} to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
-			activityId: activityId.href,
-			inbox: inbox.href,
-			error
-		});
-	}))));
+	observation?.setPhase("key-resolution");
+	const forwarding = (async () => {
+		try {
+			const key = await resolveForwardingKey(parameters);
+			observation?.setPhase("request-preparation");
+			const sending = [];
+			for (const inbox of inboxes) {
+				const delivery = observation?.delivery(inbox.href);
+				let pending;
+				try {
+					pending = sendActivity({
+						observation: delivery,
+						activity,
+						activityId: activityId.href,
+						activityType,
+						keys: key == null ? [] : [key],
+						inbox,
+						allowPrivateAddress,
+						specDeterminer,
+						meterProvider,
+						tracerProvider
+					});
+				} catch (error) {
+					if (delivery != null) await delivery.failed(error);
+					throw error;
+				}
+				const settled = pending.catch((error) => {
+					logger.error("Failed to forward activity {activityId} to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
+						activityId: activityId.href,
+						inbox: inbox.href,
+						error
+					});
+				});
+				if (observation != null) settled.catch(() => {});
+				sending.push(settled);
+				if (delivery != null) await delivery.handoff({ type: "direct" });
+			}
+			if (observation != null) await observation.prepared("delivery");
+			await Promise.all(sending);
+		} catch (error) {
+			if (observation != null) await observation.failed(error);
+			throw error;
+		}
+	})();
 	let timer;
 	const timedOut = await Promise.race([forwarding.then(() => false), new Promise((resolve) => {
 		timer = setTimeout(() => resolve(true), deadline);
@@ -1175,13 +2019,14 @@ async function forwardImmediately(parameters, inboxes, deadlineAt) {
 		deadline
 	});
 }
-async function enqueueForwarding(parameters, outboxQueue, inboxes) {
+async function enqueueForwarding(parameters, outboxQueue, inboxes, observation) {
 	const { activity, activityId, activityType, baseUrl, startQueue, meterProvider } = parameters;
 	const logger = getLogger([
 		"fedify",
 		"federation",
 		"inbox"
 	]);
+	observation?.setPhase("key-resolution");
 	const key = await resolveForwardingKey(parameters);
 	const keys = [];
 	if (key != null) try {
@@ -1196,28 +2041,43 @@ async function enqueueForwarding(parameters, outboxQueue, inboxes) {
 			error
 		});
 	}
+	observation?.setPhase("request-preparation");
 	startQueue?.();
 	const started = (/* @__PURE__ */ new Date()).toISOString();
 	const traceContext = {};
 	propagation.inject(context.active(), traceContext);
+	let preparedCount = 0;
 	await Promise.all(inboxes.map(async (inbox) => {
-		const message = {
-			type: "outbox",
-			id: crypto.randomUUID(),
-			baseUrl,
-			keys,
-			activity,
-			activityId: activityId.href,
-			activityType,
-			inbox: inbox.href,
-			sharedInbox: false,
-			started,
-			attempt: 0,
-			headers: {},
-			traceContext
-		};
+		const delivery = observation?.delivery(inbox.href);
+		let message;
+		try {
+			message = {
+				type: "outbox",
+				...serializeProvenance(parameters.provenance),
+				id: crypto.randomUUID(),
+				baseUrl,
+				keys,
+				activity,
+				activityId: activityId.href,
+				activityType,
+				inbox: inbox.href,
+				...observation?.continuation(inbox.href),
+				sharedInbox: false,
+				started,
+				attempt: 0,
+				headers: {},
+				traceContext
+			};
+		} catch (error) {
+			if (delivery != null) await delivery.failed(error);
+			throw error;
+		}
+		if (observation != null && ++preparedCount === inboxes.length) await observation.prepared("delivery");
 		try {
-			await outboxQueue.enqueue(message);
+			await (observation == null ? outboxQueue.enqueue(message) : observation.enqueue([{
+				inbox: inbox.href,
+				queueMessageId: message.id
+			}], "outbox", () => outboxQueue.enqueue(message)));
 		} catch (error) {
 			logger.error("Failed to enqueue activity {activityId} to forward to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
 				activityId: activityId.href,
@@ -5016,41 +5876,70 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}
 		await Promise.all(promises);
 	}
-	processQueuedTask(contextData, message) {
+	async processQueuedTask(contextData, message) {
 		const extractedContext = propagation.extract(context.active(), message.traceContext);
-		return withContext({ messageId: message.id }, async () => {
+		let provenance = emptyProvenance;
+		if (message.type !== "task") try {
+			provenance = restoreProvenance(message.provenance);
+		} catch (error) {
+			try {
+				getLogger([
+					"fedify",
+					"federation",
+					"queue"
+				]).warn("Ignoring invalid provenance in queue message {messageId}: {error}", {
+					messageId: message.id,
+					error
+				});
+			} catch {}
+		}
+		if (message.type !== "task" && message.provenance !== void 0) message = {
+			...message,
+			provenance
+		};
+		return await withContext({
+			messageId: message.id,
+			inboxRequestId: void 0,
+			...sourceLogContext(provenance)
+		}, async () => {
 			if (message.type === "fanout") {
+				const observers = this.outboundObservers;
+				const worker = this.#createOutboundWorkerObservation(contextData, message, provenance, observers.onWorker);
 				const common = {
 					role: "fanout",
 					queue: this.fanoutQueue,
 					activityType: message.activityType
 				};
-				await this.#runWorkerSpan("activitypub.fanout", { "activitypub.activity.type": message.activityType }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
+				const execute = () => this.#runWorkerSpan("activitypub.fanout", { "activitypub.activity.type": message.activityType }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
 					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-					await this.#listenFanoutMessage(contextData, message);
+					await this.#listenFanoutMessage(contextData, message, provenance, span, trace.getSpanContext(extractedContext), observers, worker);
 					return { outcome: "completed" };
-				}, classifyAbortableError));
+				}, classifyAbortableError), worker);
+				await (worker == null ? execute() : worker.run(execute));
 			} else if (message.type === "outbox") {
+				const observers = this.outboundObservers;
+				const worker = this.#createOutboundWorkerObservation(contextData, message, provenance, observers.onWorker);
 				const common = {
 					role: "outbox",
 					queue: this.outboxQueue,
 					activityType: message.activityType
 				};
-				await this.#runWorkerSpan("activitypub.outbox", {
+				const execute = () => this.#runWorkerSpan("activitypub.outbox", {
 					"activitypub.activity.type": message.activityType,
 					"activitypub.activity.retries": message.attempt
 				}, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
 					if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
-					await this.#listenOutboxMessage(contextData, message, span);
+					await this.#listenOutboxMessage(contextData, message, span, provenance, observers, worker);
 					return { outcome: "completed" };
-				}, classifyAbortableError));
+				}, classifyAbortableError), worker);
+				await (worker == null ? execute() : worker.run(execute));
 			} else if (message.type === "inbox") {
 				const common = {
 					role: "inbox",
 					queue: this.inboxQueue
 				};
 				await this.#runWorkerSpan("activitypub.inbox", { "activitypub.shared_inbox": message.identifier == null }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
-					await this.#listenInboxMessage(contextData, message, span, (activityType) => {
+					await this.#listenInboxMessage(contextData, message, span, provenance, (activityType) => {
 						common.activityType = activityType;
 					});
 					return { outcome: "completed" };
@@ -5069,6 +5958,22 @@ var FederationImpl = class extends FederationBuilderImpl {
 			}
 		});
 	}
+	#createOutboundWorkerObservation(contextData, message, provenance, callback) {
+		if (callback == null) return void 0;
+		let documentLoader;
+		let contextLoader;
+		return new OutboundWorkerObservation(() => {
+			const loaderOptions = this.#getLoaderOptions(message.baseUrl);
+			return new ContextImpl({
+				[outboundProvenance]: provenance,
+				federation: this,
+				url: new URL("/", message.baseUrl),
+				data: contextData,
+				documentLoader: (...args) => (documentLoader ??= this.documentLoaderFactory(loaderOptions))(...args),
+				contextLoader: (...args) => (contextLoader ??= this.contextLoaderFactory(loaderOptions))(...args)
+			});
+		}, callback, message, provenance);
+	}
 	/**
 	* Opens a CONSUMER span for a queue worker and scopes the extracted trace
 	* context around its body.  The body—typically built by
@@ -5080,11 +5985,12 @@ var FederationImpl = class extends FederationBuilderImpl {
 	* @param instrumentedBody Builds the context-scoped worker callback for the
 	*                         opened span.
 	*/
-	async #runWorkerSpan(spanName, spanAttributes, extractedContext, instrumentedBody) {
+	async #runWorkerSpan(spanName, spanAttributes, extractedContext, instrumentedBody, worker) {
 		await this._getTracer().startActiveSpan(spanName, {
 			kind: SpanKind.CONSUMER,
 			attributes: spanAttributes
 		}, extractedContext, async (span) => {
+			worker?.setSpan(span, trace.getSpanContext(extractedContext));
 			const spanCtx = span.spanContext();
 			return await withContext({
 				traceId: spanCtx.traceId,
@@ -5127,7 +6033,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 			span.end();
 		}
 	};
-	async #listenFanoutMessage(data, message) {
+	async #listenFanoutMessage(data, message, provenance, span, parentSpanContext, observers, worker) {
+		worker?.enableActions();
 		getLogger([
 			"fedify",
 			"federation",
@@ -5136,11 +6043,12 @@ var FederationImpl = class extends FederationBuilderImpl {
 			activityId: message.activityId,
 			inboxes: globalThis.Object.keys(message.inboxes).length
 		});
-		const keys = await Promise.all(message.keys.map(async ({ keyId, privateKey }) => ({
+		const importKeys = () => Promise.all(message.keys.map(async ({ keyId, privateKey }) => ({
 			keyId: new URL(keyId),
 			privateKey: await importJwk(privateKey, "private")
 		})));
-		const activity = await Activity.fromJsonLd(message.activity, {
+		const keys = worker == null ? await importKeys() : await worker.prepare("key-import", importKeys);
+		const parseActivity = () => Activity.fromJsonLd(message.activity, {
 			contextLoader: this.contextLoaderFactory({
 				allowPrivateAddress: this.allowPrivateAddress,
 				userAgent: this.userAgent
@@ -5151,19 +6059,30 @@ var FederationImpl = class extends FederationBuilderImpl {
 			}),
 			tracerProvider: this.tracerProvider
 		});
-		const context = this.#createContext(new URL(message.baseUrl), data, { documentLoader: this.documentLoaderFactory({
-			allowPrivateAddress: this.allowPrivateAddress,
-			userAgent: this.userAgent
-		}) });
+		const activity = worker == null ? await parseActivity() : await worker.prepare("activity-parsing", parseActivity);
+		const createContext = () => this.#createContext(new URL(message.baseUrl), data, {
+			[outboundProvenance]: provenance,
+			documentLoader: this.documentLoaderFactory({
+				allowPrivateAddress: this.allowPrivateAddress,
+				userAgent: this.userAgent
+			})
+		});
+		const context = worker == null ? createContext() : await worker.prepare("context-preparation", createContext);
+		worker?.useContext(context);
+		const observation = await OutboundBroadcastObservation.resumeFanout(context, observers, message, span, parentSpanContext);
 		await this.sendActivity(keys, message.inboxes, activity, {
 			collectionSync: message.collectionSync,
 			orderingKey: message.orderingKey,
 			normalizeExistingProofs: message.normalizeExistingProofs,
 			activityJsonLd: message.activity,
+			observation,
+			worker,
 			context
 		});
 	}
-	async #listenOutboxMessage(_, message, span) {
+	async #listenOutboxMessage(contextData, message, span, provenance, observers, worker) {
+		const { onAttempt, onHttpExchange } = observers;
+		worker?.enableActions();
 		const logger = getLogger([
 			"fedify",
 			"federation",
@@ -5180,14 +6099,26 @@ var FederationImpl = class extends FederationBuilderImpl {
 		const keys = [];
 		let rsaKeyPair = null;
 		for (const { keyId, privateKey } of message.keys) {
-			const pair = {
+			const importPair = async () => ({
 				keyId: new URL(keyId),
 				privateKey: await importJwk(privateKey, "private")
-			};
+			});
+			const pair = worker == null ? await importPair() : await worker.prepare("key-import", importPair);
 			if (rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") rsaKeyPair = pair;
 			keys.push(pair);
 		}
 		const loaderOptions = this.#getLoaderOptions(message.baseUrl);
+		let observationContext;
+		let observationDocumentLoader;
+		let observationContextLoader;
+		const observation = OutboundAttemptObserver.resumeOutbox(() => worker?.getContext() ?? (observationContext ??= new ContextImpl({
+			[outboundProvenance]: provenance,
+			federation: this,
+			url: new URL("/", message.baseUrl),
+			data: contextData,
+			documentLoader: (...args) => (observationDocumentLoader ??= this.documentLoaderFactory(loaderOptions))(...args),
+			contextLoader: (...args) => (observationContextLoader ??= this.contextLoaderFactory(loaderOptions))(...args)
+		})), onAttempt, message, provenance, onHttpExchange, observers, observers.responseBodyCapture);
 		let parsedActorIds;
 		const getActorIds = () => {
 			parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => {
@@ -5210,7 +6141,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				verifyPortableObject: createPortableObjectVerifier(documentLoader, contextLoader, this)
 			});
 		};
-		const enqueueHeldOutboxMessage = async (delay, heldSince) => {
+		const enqueueHeldOutboxMessage = async (delay, heldSince, preparationFailures) => {
 			const { outboxQueue } = this;
 			if (outboxQueue == null) return;
 			const heldMessage = {
@@ -5218,10 +6149,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 				circuitHeld: true,
 				circuitHeldSince: heldSince.toString()
 			};
-			await outboxQueue.enqueue(heldMessage, {
+			const enqueueOptions = {
 				delay: clampNegativeDelay(delay),
 				orderingKey: message.orderingKey
-			});
+			};
+			await (worker == null ? outboxQueue.enqueue(heldMessage, enqueueOptions) : worker.enqueue("defer", heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), preparationFailures));
 			this.metrics.recordQueueTaskEnqueued({
 				role: "outbox",
 				queue: outboxQueue,
@@ -5238,7 +6170,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 				heldSince
 			});
 			if (this.outboxPermanentFailureHandler != null) {
-				const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
+				const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
+					[outboundProvenance]: provenance,
+					documentLoader: this.documentLoaderFactory(loaderOptions)
+				});
 				try {
 					await this.outboxPermanentFailureHandler(ctx, {
 						reason: "circuit-breaker-ttl",
@@ -5250,6 +6185,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 						actorIds: getActorIds()
 					});
 				} catch (handlerError) {
+					worker?.dropFailure([inbox.href], handlerError);
 					logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
 						...logData,
 						error: handlerError
@@ -5260,10 +6196,13 @@ var FederationImpl = class extends FederationBuilderImpl {
 		};
 		const { gatewayInboxes } = message;
 		if (gatewayInboxes != null && gatewayInboxes.length > 1) {
-			await this.#listenGatewayOutboxMessage(_, {
+			await this.#listenGatewayOutboxMessage(contextData, {
 				...message,
 				gatewayInboxes
 			}, span, {
+				observation,
+				worker,
+				provenance,
 				keys,
 				logData,
 				getActorIds,
@@ -5273,7 +6212,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 			return;
 		}
 		try {
-			const inbox = new URL(message.inbox);
+			const parseInbox = () => new URL(message.inbox);
+			const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
 			const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
 			const remoteHost = getRemoteHost(inbox);
 			let decision;
@@ -5297,13 +6237,19 @@ var FederationImpl = class extends FederationBuilderImpl {
 					return;
 				}
 				if (decision.type === "drop") {
-					const activity = await parseActivity();
-					await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
+					worker?.abandon("held-ttl-expired", [inbox.href]);
+					const drop = async () => {
+						const activity = await parseActivity();
+						await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
+					};
+					await (worker == null ? drop() : worker.drop([inbox.href], drop));
 					return;
 				}
 				if (decision.stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, decision.stateChange);
 			}
 			await sendActivity({
+				attemptStarted: worker?.attemptStarted,
+				observation,
 				allowPrivateAddress: this.allowPrivateAddress,
 				keys,
 				activity: message.activity,
@@ -5311,7 +6257,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				activityType: message.activityType,
 				inbox,
 				sharedInbox: message.sharedInbox,
-				headers: new Headers(message.headers),
+				headers: worker == null ? new Headers(message.headers) : await worker.prepare("request-preparation", () => new Headers(message.headers)),
 				specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
 				meterProvider: this.meterProvider,
 				tracerProvider: this.tracerProvider
@@ -5331,6 +6277,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				});
 			}
 		} catch (error) {
+			const preparationFailures = worker?.recordPreparationFailure(error, message.inbox) ?? [];
 			span.setStatus({
 				code: SpanStatusCode.ERROR,
 				message: String(error)
@@ -5339,7 +6286,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				if (error instanceof SendActivityError) return getRemoteHost(error.inbox);
 				try {
 					return getRemoteHost(new URL(message.inbox));
-				} catch (_) {
+				} catch {
 					logger.warn("Invalid inbox URL in queued outbox message: {inbox}", logData);
 					return;
 				}
@@ -5412,32 +6359,42 @@ var FederationImpl = class extends FederationBuilderImpl {
 				});
 			}
 			if (circuitDrop != null) {
-				await dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
+				worker?.abandon("held-ttl-expired", [circuitDrop.inbox.href], preparationFailures);
+				const drop = () => dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
+				await (worker == null ? drop() : worker.drop([circuitDrop.inbox.href], drop));
 				return;
 			}
 			if (isPermanentFailure) {
+				worker?.abandon("permanent-failure", [message.inbox], preparationFailures);
 				this.metrics.recordPermanentFailure(error.inbox, error.statusCode);
 				logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not retrying.", {
 					...logData,
 					status: error.statusCode
 				});
 				if (this.outboxPermanentFailureHandler != null) {
-					const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
-					try {
-						await this.outboxPermanentFailureHandler(ctx, {
-							reason: "http",
-							inbox: new URL(message.inbox),
-							activity,
-							error,
-							statusCode: error.statusCode,
-							actorIds: getActorIds()
-						});
-					} catch (handlerError) {
-						logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
-							...logData,
-							error: handlerError
+					const drop = async () => {
+						const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
+							[outboundProvenance]: provenance,
+							documentLoader: this.documentLoaderFactory(loaderOptions)
 						});
-					}
+						try {
+							await this.outboxPermanentFailureHandler(ctx, {
+								reason: "http",
+								inbox: new URL(message.inbox),
+								activity,
+								error,
+								statusCode: error.statusCode,
+								actorIds: getActorIds()
+							});
+						} catch (handlerError) {
+							worker?.dropFailure([message.inbox], handlerError);
+							logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
+								...logData,
+								error: handlerError
+							});
+						}
+					};
+					await (worker == null ? drop() : worker.drop([message.inbox], drop));
 				}
 				recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
 				return;
@@ -5449,7 +6406,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				});
 				recordCircuitBreakerHeldSpanEvent(span, circuitHold.remoteHost, circuitHold.state);
 				const circuit = this.circuitBreaker;
-				await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince);
+				await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince, preparationFailures);
 				return;
 			}
 			if (this.outboxQueue?.nativeRetrial && retryAfterDelay == null) {
@@ -5472,18 +6429,20 @@ var FederationImpl = class extends FederationBuilderImpl {
 				};
 				const { outboxQueue } = this;
 				if (outboxQueue != null) {
-					await outboxQueue.enqueue(retryMessage, {
+					const enqueueOptions = {
 						delay: clampNegativeDelay(delay),
 						orderingKey: message.orderingKey
-					});
+					};
+					await (worker == null ? outboxQueue.enqueue(retryMessage, enqueueOptions) : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => outboxQueue.enqueue(retryMessage, enqueueOptions), preparationFailures));
 					this.metrics.recordQueueTaskEnqueued({
 						role: "outbox",
 						queue: outboxQueue,
 						activityType: retryMessage.activityType
 					}, retryMessage.attempt);
 					recordOutboxActivity(this.meterProvider, "retried", retryMessage.activityType);
-				}
+				} else worker?.abandon("queue-unavailable", [message.inbox], preparationFailures);
 			} else {
+				worker?.abandon("retry-policy-declined", [message.inbox], preparationFailures);
 				logger.error("Failed to send activity {activityId} to {inbox} after {attempt} attempts; giving up:\n{error}", {
 					...logData,
 					error
@@ -5518,25 +6477,32 @@ var FederationImpl = class extends FederationBuilderImpl {
 			"federation",
 			"circuit"
 		]);
-		const { keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = helpers;
+		const { observation, worker, provenance, keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = helpers;
 		const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
 		const specDeterminer = new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl });
 		const survivors = [];
+		const preparationFailuresByRoute = /* @__PURE__ */ new Map();
+		const permanentRoutes = /* @__PURE__ */ new Set();
 		let lastError;
 		let lastPermanent;
 		let activity;
 		const getActivity = async () => activity ??= await parseActivity();
 		const drop = async (circuit, remoteHost, inbox, heldSince) => {
-			await dropHeldOutboxMessage(circuit, remoteHost, inbox, heldSince, await getActivity());
+			const routes = message.gatewayInboxes.filter((route) => !permanentRoutes.has(route));
+			worker?.abandon("held-ttl-expired", routes, routes.flatMap((route) => preparationFailuresByRoute.get(route) ?? []));
+			const execute = async () => dropHeldOutboxMessage(circuit, remoteHost, inbox, heldSince, await getActivity());
+			await (worker == null ? execute() : worker.drop(routes, execute));
 		};
 		const heldSince = parseInstant(message.circuitHeldSince);
 		if (circuit != null && heldSince != null && Temporal.Instant.compare(heldSince.add(circuit.options.heldActivityTtl), Temporal.Now.instant()) <= 0) {
-			const inbox = new URL(message.gatewayInboxes[0]);
+			const parseInbox = () => new URL(message.gatewayInboxes[0]);
+			const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
 			await drop(circuit, getRemoteHost(inbox), inbox, heldSince);
 			return;
 		}
 		for (const inboxHref of message.gatewayInboxes) {
-			const inbox = new URL(inboxHref);
+			const parseInbox = () => new URL(inboxHref);
+			const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
 			const remoteHost = getRemoteHost(inbox);
 			const now = Temporal.Now.instant();
 			const notBefore = parseInstant(message.gatewayNotBefore?.[inboxHref]);
@@ -5574,6 +6540,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 			}
 			try {
 				await sendActivity({
+					attemptStarted: worker?.attemptStarted,
+					observation,
 					allowPrivateAddress: this.allowPrivateAddress,
 					keys,
 					activity: message.activity,
@@ -5581,12 +6549,14 @@ var FederationImpl = class extends FederationBuilderImpl {
 					activityType: message.activityType,
 					inbox,
 					sharedInbox: message.sharedInbox,
-					headers: new Headers(message.headers),
+					headers: worker == null ? new Headers(message.headers) : await worker.prepare("request-preparation", () => new Headers(message.headers)),
 					specDeterminer,
 					meterProvider: this.meterProvider,
 					tracerProvider: this.tracerProvider
 				});
 			} catch (error) {
+				const preparationFailures = worker?.recordPreparationFailure(error, inboxHref) ?? [];
+				if (preparationFailures.length > 0) preparationFailuresByRoute.set(inboxHref, preparationFailures);
 				lastError = error;
 				const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode);
 				let retryAfter = !isPermanentFailure && error instanceof SendActivityError && (error.statusCode === 429 || error.statusCode === 503) ? parseRetryAfter(error.responseHeaders) : void 0;
@@ -5631,6 +6601,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 					});
 				}
 				if (isPermanentFailure) {
+					permanentRoutes.add(inboxHref);
+					worker?.abandon("permanent-failure", [inboxHref], preparationFailures);
 					logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not trying the gateway again.", {
 						...logData,
 						inbox: inboxHref,
@@ -5695,22 +6667,29 @@ var FederationImpl = class extends FederationBuilderImpl {
 				portableInbox: message.portableInbox
 			});
 			if (this.outboxPermanentFailureHandler != null) {
-				const ctx = this.#createContext(new URL(message.baseUrl), contextData, { documentLoader: this.documentLoaderFactory(this.#getLoaderOptions(message.baseUrl)) });
-				try {
-					await this.outboxPermanentFailureHandler(ctx, {
-						reason: "http",
-						inbox,
-						activity: await getActivity(),
-						error,
-						statusCode: error.statusCode,
-						actorIds: getActorIds()
-					});
-				} catch (handlerError) {
-					logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
-						...logData,
-						error: handlerError
+				const drop = async () => {
+					const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
+						[outboundProvenance]: provenance,
+						documentLoader: this.documentLoaderFactory(this.#getLoaderOptions(message.baseUrl))
 					});
-				}
+					try {
+						await this.outboxPermanentFailureHandler(ctx, {
+							reason: "http",
+							inbox,
+							activity: await getActivity(),
+							error,
+							statusCode: error.statusCode,
+							actorIds: getActorIds()
+						});
+					} catch (handlerError) {
+						worker?.dropFailure(message.gatewayInboxes, handlerError);
+						logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
+							...logData,
+							error: handlerError
+						});
+					}
+				};
+				await (worker == null ? drop() : worker.drop(message.gatewayInboxes, drop));
 			}
 			recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
 			return;
@@ -5746,10 +6725,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 				...logData,
 				portableInbox: message.portableInbox
 			});
-			await outboxQueue.enqueue(heldMessage, {
+			const enqueueOptions = {
 				delay: clampNegativeDelay(delay),
 				orderingKey: message.orderingKey
-			});
+			};
+			await (worker == null ? outboxQueue.enqueue(heldMessage, enqueueOptions) : worker.enqueue("defer", heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? [])));
 			this.metrics.recordQueueTaskEnqueued({
 				role: "outbox",
 				queue: outboxQueue,
@@ -5767,6 +6747,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 			throw lastError;
 		}
 		if (policyDelay == null || outboxQueue == null) {
+			worker?.abandon(outboxQueue == null ? "queue-unavailable" : "retry-policy-declined", survivors.map((route) => route.inbox), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? []));
 			logger.error("Failed to send activity {activityId} to every gateway of {portableInbox} after {attempt} attempts; giving up:\n{error}", {
 				...logData,
 				portableInbox: message.portableInbox,
@@ -5785,10 +6766,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 			portableInbox: message.portableInbox,
 			error: lastError
 		});
-		await outboxQueue.enqueue(retryMessage, {
+		const enqueueOptions = {
 			delay: clampNegativeDelay(delay),
 			orderingKey: message.orderingKey
-		});
+		};
+		await (worker == null ? outboxQueue.enqueue(retryMessage, enqueueOptions) : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => outboxQueue.enqueue(retryMessage, enqueueOptions), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? [])));
 		this.metrics.recordQueueTaskEnqueued({
 			role: "outbox",
 			queue: outboxQueue,
@@ -5796,18 +6778,24 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}, retryMessage.attempt);
 		recordOutboxActivity(this.meterProvider, "retried", retryMessage.activityType);
 	}
-	async #listenInboxMessage(ctxData, message, span, onActivityType) {
+	async #listenInboxMessage(ctxData, message, span, provenance, onActivityType) {
 		const logger = getLogger([
 			"fedify",
 			"federation",
 			"inbox"
 		]);
 		const baseUrl = new URL(message.baseUrl);
-		let context = this.#createContext(baseUrl, ctxData);
-		if (message.identifier != null && this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(baseUrl, ctxData, { documentLoader: await context.getDocumentLoader({ identifier: message.identifier }) });
+		let context = this.#createContext(baseUrl, ctxData, { [outboundProvenance]: provenance });
+		if (message.identifier != null && this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(baseUrl, ctxData, {
+			[outboundProvenance]: provenance,
+			documentLoader: await context.getDocumentLoader({ identifier: message.identifier })
+		});
 		else if (message.identifier == null && this.sharedInboxKeyDispatcher != null) {
 			const identity = await this.sharedInboxKeyDispatcher(context);
-			if (identity != null) context = this.#createContext(baseUrl, ctxData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
+			if (identity != null) context = this.#createContext(baseUrl, ctxData, {
+				[outboundProvenance]: provenance,
+				documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
+			});
 		}
 		await this._getTracer().startActiveSpan("activitypub.dispatch_inbox_listener", { kind: SpanKind.INTERNAL }, async (listenerSpan) => {
 			let activity = null;
@@ -6230,6 +7218,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}
 		const loaderOptions = this.#getLoaderOptions(url.origin);
 		const ctxOptions = {
+			[outboundProvenance]: opts[outboundProvenance],
 			url,
 			federation: this,
 			data: contextData,
@@ -6261,22 +7250,37 @@ var FederationImpl = class extends FederationBuilderImpl {
 			"federation",
 			"outbox"
 		]);
-		const { immediate, collectionSync, orderingKey, context: ctx } = options;
-		if (activity.id == null) throw new TypeError("The activity to send must have an id.");
-		if (activity.actorId == null) throw new TypeError("The activity to send must have at least one actor property.");
-		else if (keys.length < 1) throw new TypeError("The keys must not be empty.");
-		const contextLoader = this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin));
-		const activityId = activity.id.href;
-		let rsaKey = null;
-		const ldSignable = !hasPortableActor(activity);
-		for (const { keyId, privateKey } of keys) {
-			validateCryptoKey(privateKey, "private");
-			if (ldSignable && rsaKey == null && privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && !isPortableKeyId(keyId)) rsaKey = {
-				keyId,
-				privateKey
+		const { immediate, collectionSync, orderingKey, context: ctx, observation, worker, provenance = getOutboundProvenance(ctx) } = options;
+		observation?.setPhase("validation");
+		const validateActivity = () => {
+			if (activity.id == null) throw new TypeError("The activity to send must have an id.");
+			if (activity.actorId == null) throw new TypeError("The activity to send must have at least one actor property.");
+			else if (keys.length < 1) throw new TypeError("The keys must not be empty.");
+			return activity.id.href;
+		};
+		const activityId = worker == null ? validateActivity() : await worker.prepare("validation", validateActivity);
+		observation?.setPhase("serialization");
+		const createContextLoader = () => this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin));
+		const contextLoader = worker == null ? createContextLoader() : await worker.prepare("serialization", createContextLoader);
+		const resolveRsaKey = () => {
+			const ldSignable = !hasPortableActor(activity);
+			let rsaKey = null;
+			for (const { keyId, privateKey } of keys) {
+				validateCryptoKey(privateKey, "private");
+				if (ldSignable && rsaKey == null && privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && !isPortableKeyId(keyId)) rsaKey = {
+					keyId,
+					privateKey
+				};
+			}
+			return {
+				rsaKey,
+				ldSignable
 			};
-		}
-		const signed = await signOutgoingActivity(activity, keys.map(({ keyId, privateKey }) => ({
+		};
+		observation?.setPhase("key-resolution");
+		const { rsaKey, ldSignable } = worker == null ? resolveRsaKey() : await worker.prepare("key-resolution", resolveRsaKey);
+		observation?.setPhase("signing");
+		const signActivity = () => signOutgoingActivity(activity, keys.map(({ keyId, privateKey }) => ({
 			verificationMethod: keyId,
 			privateKey
 		})), {
@@ -6284,14 +7288,23 @@ var FederationImpl = class extends FederationBuilderImpl {
 			tracerProvider: this.tracerProvider,
 			appendToExistingProofs: false
 		});
+		const signed = worker == null ? await signActivity() : await worker.prepare("signing", signActivity);
 		activity = signed.activity;
 		const { hasProof, proofCreated } = signed;
-		let jsonLd = !proofCreated && options.activityJsonLd != null ? options.activityJsonLd : await activity.toJsonLd({
-			format: "compact",
-			contextLoader
-		});
-		if (proofCreated || !hasProof || options.normalizeExistingProofs) jsonLd = await normalizeOutgoingActivityJsonLd(jsonLd, contextLoader, { preserveNestedSecuredDocuments: true });
-		assertSupportedCompoundProofShape(jsonLd, activityId);
+		observation?.setPhase("serialization");
+		const serializeActivity = async () => {
+			let result = !proofCreated && options.activityJsonLd != null ? options.activityJsonLd : await activity.toJsonLd({
+				format: "compact",
+				contextLoader
+			});
+			if (proofCreated || !hasProof || options.normalizeExistingProofs) result = await normalizeOutgoingActivityJsonLd(result, contextLoader, { preserveNestedSecuredDocuments: true });
+			return result;
+		};
+		let jsonLd = worker == null ? await serializeActivity() : await worker.prepare("serialization", serializeActivity);
+		observation?.setPhase("validation");
+		const validateProofShape = () => assertSupportedCompoundProofShape(jsonLd, activityId);
+		if (worker == null) validateProofShape();
+		else await worker.prepare("validation", validateProofShape);
 		if (rsaKey == null && !ldSignable) logger.debug("The activity {activityId} is performed by a portable actor, so it is sent without a Linked Data signature.", { activityId });
 		else if (rsaKey == null) logger.warn("No supported key found to create a Linked Data signature for the activity {activityId}.  The activity will be sent without a Linked Data signature.  In order to create a Linked Data signature, at least one RSASSA-PKCS1-v1_5 key must be provided.", {
 			activityId,
@@ -6300,17 +7313,24 @@ var FederationImpl = class extends FederationBuilderImpl {
 				privateKey: pair.privateKey
 			}))
 		});
-		else try {
-			jsonLd = await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, {
-				contextLoader,
-				tracerProvider: this.tracerProvider
-			});
-		} catch (error) {
-			if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error;
-			logger.warn("Failed to create a Linked Data signature for the activity {activityId}.  The activity will be sent without a Linked Data signature.", {
-				activityId,
-				error
-			});
+		else {
+			const signLinkedData = async () => {
+				try {
+					observation?.setPhase("signing");
+					return await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, {
+						contextLoader,
+						tracerProvider: this.tracerProvider
+					});
+				} catch (error) {
+					if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error;
+					logger.warn("Failed to create a Linked Data signature for the activity {activityId}.  The activity will be sent without a Linked Data signature.", {
+						activityId,
+						error
+					});
+					return jsonLd;
+				}
+			};
+			jsonLd = worker == null ? await signLinkedData() : await worker.prepare("signing", signLinkedData);
 		}
 		if (!hasProof) logger.warn("No supported key found to create a proof for the activity {activityId}.  The activity will be sent without a proof.  In order to create a proof, at least one Ed25519 key must be provided.", {
 			activityId,
@@ -6319,6 +7339,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				privateKey: pair.privateKey
 			}))
 		});
+		observation?.setPhase("request-preparation");
 		if (immediate || this.outboxQueue == null) {
 			if (immediate) logger.debug("Sending activity immediately without queue since immediate option is set.", {
 				activityId: activity.id.href,
@@ -6329,18 +7350,40 @@ var FederationImpl = class extends FederationBuilderImpl {
 				activity: jsonLd
 			});
 			const promises = [];
-			for (const inbox in inboxes) promises.push(sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
-				allowPrivateAddress: this.allowPrivateAddress,
-				keys,
-				activity: jsonLd,
-				activityId: activity.id?.href,
-				activityType: getTypeId(activity).href,
-				sharedInbox: inboxes[inbox].sharedInbox,
-				headers: collectionSync == null ? void 0 : new Headers({ "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) }),
-				specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
-				meterProvider: this.meterProvider,
-				tracerProvider: this.tracerProvider
-			}));
+			for (const inbox in inboxes) {
+				const delivery = observation?.delivery(inbox);
+				let sending;
+				try {
+					const createHeaders = collectionSync == null ? void 0 : async () => new Headers({ "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) });
+					const sendActivityId = activity.id?.href;
+					const activityType = getTypeId(activity).href;
+					observation?.setActivity({
+						...sendActivityId == null ? {} : { id: sendActivityId },
+						type: activityType
+					});
+					sending = sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
+						attemptStarted: worker == null ? void 0 : (target) => worker.directAttemptStarted(target, observation?.continuation(inbox)?.outbound),
+						observation: delivery,
+						allowPrivateAddress: this.allowPrivateAddress,
+						keys,
+						activity: jsonLd,
+						activityId: sendActivityId,
+						activityType,
+						sharedInbox: inboxes[inbox].sharedInbox,
+						headers: createHeaders == null ? void 0 : await (worker == null ? createHeaders() : worker.prepare("request-preparation", createHeaders)),
+						specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
+						meterProvider: this.meterProvider,
+						tracerProvider: this.tracerProvider
+					});
+				} catch (error) {
+					if (delivery != null) await delivery.failed(error);
+					throw error;
+				}
+				if (observation != null) sending.catch(() => {});
+				if (delivery != null) await delivery.handoff({ type: "direct" });
+				promises.push(sending);
+			}
+			if (observation != null) await observation.prepared("delivery");
 			await Promise.all(promises);
 			return;
 		}
@@ -6348,48 +7391,71 @@ var FederationImpl = class extends FederationBuilderImpl {
 			activityId: activity.id.href,
 			activity: jsonLd
 		});
+		observation?.setPhase("key-resolution");
 		const keyJwkPairs = [];
 		for (const { keyId, privateKey } of keys) {
-			const privateKeyJwk = await exportJwk(privateKey);
+			const exportPrivateKey = () => exportJwk(privateKey);
+			const privateKeyJwk = worker == null ? await exportPrivateKey() : await worker.prepare("key-export", exportPrivateKey);
 			keyJwkPairs.push({
 				keyId: keyId.href,
 				privateKey: privateKeyJwk
 			});
 		}
 		if (!this.manuallyStartQueue) this._startQueueInternal(ctx.data);
+		observation?.setPhase("request-preparation");
 		const carrier = {};
-		propagation.inject(context.active(), carrier);
+		const injectTraceContext = () => propagation.inject(context.active(), carrier);
+		if (worker == null) injectTraceContext();
+		else await worker.prepare("request-preparation", injectTraceContext);
 		const messages = [];
-		for (const inbox in inboxes) {
-			const { portableInbox, gatewayInboxes } = inboxes[inbox];
-			const messageOrderingKey = orderingKey == null ? void 0 : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`;
-			const message = {
-				type: "outbox",
-				id: crypto.randomUUID(),
-				baseUrl: ctx.origin,
-				keys: keyJwkPairs,
-				activity: jsonLd,
-				activityId: activity.id?.href,
-				activityType: getTypeId(activity).href,
-				inbox,
-				sharedInbox: inboxes[inbox].sharedInbox,
-				actorIds: [...inboxes[inbox].actorIds],
-				started: (/* @__PURE__ */ new Date()).toISOString(),
-				attempt: 0,
-				headers: collectionSync == null ? {} : { "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) },
-				orderingKey: messageOrderingKey,
-				...getPortableMessageFields(portableInbox, gatewayInboxes),
-				traceContext: carrier
+		for (const inbox in inboxes) try {
+			const prepareMessage = async () => {
+				const { portableInbox, gatewayInboxes } = inboxes[inbox];
+				const messageOrderingKey = orderingKey == null ? void 0 : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`;
+				const messageActivityId = activity.id?.href;
+				const messageActivityType = getTypeId(activity).href;
+				observation?.setActivity({
+					...messageActivityId == null ? {} : { id: messageActivityId },
+					type: messageActivityType
+				});
+				return {
+					message: {
+						type: "outbox",
+						...serializeProvenance(provenance),
+						id: crypto.randomUUID(),
+						baseUrl: ctx.origin,
+						keys: keyJwkPairs,
+						activity: jsonLd,
+						activityId: messageActivityId,
+						activityType: messageActivityType,
+						inbox,
+						...observation?.continuation(inbox),
+						sharedInbox: inboxes[inbox].sharedInbox,
+						actorIds: [...inboxes[inbox].actorIds],
+						started: (/* @__PURE__ */ new Date()).toISOString(),
+						attempt: 0,
+						headers: collectionSync == null ? {} : { "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) },
+						orderingKey: messageOrderingKey,
+						...getPortableMessageFields(portableInbox, gatewayInboxes),
+						traceContext: carrier
+					},
+					orderingKey: messageOrderingKey
+				};
 			};
-			messages.push({
-				message,
-				orderingKey: messageOrderingKey
-			});
+			messages.push(await (worker == null ? prepareMessage() : worker.prepare("request-preparation", prepareMessage)));
+		} catch (error) {
+			if (observation != null) await observation.delivery(inbox)?.failed(error);
+			throw error;
 		}
+		if (observation != null) await observation.prepared("delivery");
 		const { outboxQueue } = this;
 		if (outboxQueue.enqueueMany == null || orderingKey != null) {
 			const errors = (await Promise.allSettled(messages.map(async (m) => {
-				await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
+				const enqueue = () => worker == null ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) : worker.enqueueChildren([m.message], () => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }));
+				await (observation == null ? enqueue() : observation.enqueue([{
+					inbox: m.message.inbox,
+					queueMessageId: m.message.id
+				}], "outbox", enqueue));
 				recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message);
 			}))).filter((r) => r.status === "rejected").map((r) => r.reason);
 			if (errors.length > 0) {
@@ -6402,7 +7468,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 			}
 		} else {
 			try {
-				await outboxQueue.enqueueMany(messages.map((m) => m.message));
+				const enqueue = () => worker == null ? outboxQueue.enqueueMany(messages.map((m) => m.message)) : worker.enqueueChildren(messages.map((m) => m.message), () => outboxQueue.enqueueMany(messages.map((m) => m.message)));
+				await (observation == null ? enqueue() : observation.enqueue(messages.map(({ message }) => ({
+					inbox: message.inbox,
+					queueMessageId: message.id
+				})), "outbox", enqueue));
 			} catch (error) {
 				logger.error("Failed to enqueue activity {activityId} to send later: {error}", {
 					activityId: activity.id.href,
@@ -6414,9 +7484,16 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}
 	}
 	fetch(request, options) {
-		return withContext({ requestId: getRequestId(request) }, async () => {
+		const requestId = getRequestId(request);
+		return withContext({
+			requestId,
+			inboxRequestId: void 0,
+			dispatchId: void 0,
+			sourceInboxRequestId: void 0,
+			sourceRequestId: void 0
+		}, async () => {
 			const tracer = this._getTracer();
-			const metricState = {};
+			const metricState = { requestId };
 			const metricStart = performance.now();
 			return await tracer.startActiveSpan(request.method, {
 				kind: SpanKind.SERVER,
@@ -6461,7 +7538,8 @@ var FederationImpl = class extends FederationBuilderImpl {
 						logger.error("An error occurred while serving request {method} {url}: {error}", {
 							method: request.method,
 							url: request.url,
-							error
+							error,
+							...metricState.inboxRequest == null ? {} : { inboxRequestId: metricState.inboxRequest.inboxRequestId }
 						});
 						throw error;
 					}
@@ -6481,6 +7559,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 					const url = new URL(request.url);
 					const logTpl = "{method} {path}: {status}";
 					const values = {
+						...metricState.inboxRequest == null ? {} : { inboxRequestId: metricState.inboxRequest.inboxRequestId },
 						method: request.method,
 						path: `${url.pathname}${url.search}`,
 						url: request.url,
@@ -6533,9 +7612,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 		metricState.endpoint = getEndpointCategory(route.name);
 		span.updateName(`${request.method} ${route.template}`);
 		const routeName = route.name.replace(/:.*$/, "");
+		const source = request.method === "POST" && (routeName === "inbox" || routeName === "sharedInbox") ? this.#createInboxRequestSource(request, metricState) : void 0;
+		const provenance = globalThis.Object.freeze(source == null ? { originatingRequestId: metricState.requestId } : { inboxRequest: source });
 		let context;
 		try {
-			context = this.#createContext(request, contextData);
+			context = withContext(source == null ? {} : { inboxRequestId: source.inboxRequestId }, () => this.#createContext(request, contextData, { [outboundProvenance]: provenance }));
 		} catch (error) {
 			if (request.method !== "POST" || routeName !== "inbox" && routeName !== "sharedInbox") throw error;
 			return await this.#reportInboxPreparationError(request, contextData, {
@@ -6606,13 +7687,19 @@ var FederationImpl = class extends FederationBuilderImpl {
 			const observation = new InboxObservation({
 				kind: routeName === "inbox" ? "personal" : "shared",
 				recipient
-			});
+			}, source);
 			this.#setInboxCompletion(metricState, observation, () => context);
-			return await observation.run(async () => {
-				if (routeName === "inbox") context = this.#createContext(request, contextData, { documentLoader: await context.getDocumentLoader({ identifier: recipient }) });
+			return await withContext({ inboxRequestId: source.inboxRequestId }, () => observation.run(async () => {
+				if (routeName === "inbox") context = this.#createContext(request, contextData, {
+					[outboundProvenance]: getOutboundProvenance(context),
+					documentLoader: await context.getDocumentLoader({ identifier: recipient })
+				});
 				else if (this.sharedInboxKeyDispatcher != null) {
 					const identity = await this.sharedInboxKeyDispatcher(context);
-					if (identity != null) context = this.#createContext(request, contextData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
+					if (identity != null) context = this.#createContext(request, contextData, {
+						[outboundProvenance]: getOutboundProvenance(context),
+						documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
+					});
 				}
 				return await this.#handleInbox(request, {
 					recipient,
@@ -6621,13 +7708,16 @@ var FederationImpl = class extends FederationBuilderImpl {
 					onNotFound,
 					observation
 				});
-			}, span);
+			}, span));
 		}
 		switch (routeName) {
 			case "actor":
 			case "actorAlias": {
 				const identifier = route.name.startsWith("actorAlias:") ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier;
-				context = this.#createContext(request, contextData, { invokedFromActorDispatcher: { identifier } });
+				context = this.#createContext(request, contextData, {
+					[outboundProvenance]: getOutboundProvenance(context),
+					invokedFromActorDispatcher: { identifier }
+				});
 				return await handleActor(request, {
 					identifier,
 					context,
@@ -6641,10 +7731,13 @@ var FederationImpl = class extends FederationBuilderImpl {
 				const typeId = route.name.replace(/^object:/, "");
 				const callbacks = this.objectCallbacks[typeId];
 				const cls = this.objectTypeIds[typeId];
-				context = this.#createContext(request, contextData, { invokedFromObjectDispatcher: {
-					cls,
-					values: route.values
-				} });
+				context = this.#createContext(request, contextData, {
+					[outboundProvenance]: getOutboundProvenance(context),
+					invokedFromObjectDispatcher: {
+						cls,
+						values: route.values
+					}
+				});
 				return await handleObject(request, {
 					values: route.values,
 					context,
@@ -6702,7 +7795,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 			case "sharedInbox":
 				if (this.sharedInboxKeyDispatcher != null) {
 					const identity = await this.sharedInboxKeyDispatcher(context);
-					if (identity != null) context = this.#createContext(request, contextData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
+					if (identity != null) context = this.#createContext(request, contextData, {
+						[outboundProvenance]: getOutboundProvenance(context),
+						documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
+					});
 				}
 				return await this.#handleInbox(request, {
 					recipient: route.values.identifier ?? null,
@@ -6846,14 +7942,17 @@ var FederationImpl = class extends FederationBuilderImpl {
 			});
 			return response;
 		}
+		const provenance = globalThis.Object.freeze({ originatingRequestId: metricState.requestId });
 		if (collectionRoute != null) return await this.#fetchPortableCollection(request, portable, route, {
 			onNotFound,
 			onUnauthorized,
-			contextData
+			contextData,
+			provenance
 		});
 		if (isActor) {
 			const identifier = route.name.startsWith("actorAlias:") ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier;
 			const context = this.#createContext(request, contextData, {
+				[outboundProvenance]: provenance,
 				invokedFromActorDispatcher: { identifier },
 				portableRequest: portable.portableRequest
 			});
@@ -6874,6 +7973,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 		const callbacks = this.objectCallbacks[typeId];
 		const cls = this.objectTypeIds[typeId];
 		const context = this.#createContext(request, contextData, {
+			[outboundProvenance]: provenance,
 			invokedFromObjectDispatcher: {
 				cls,
 				values: route.values
@@ -6891,7 +7991,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 			onNotFound
 		});
 	}
-	async #fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData }) {
+	async #fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData, provenance }) {
 		const logger = getLogger([
 			"fedify",
 			"federation",
@@ -6899,8 +7999,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 		]);
 		const url = new URL(request.url);
 		const { authority } = portable.portableRequest;
-		const ownerContext = this.#createContext(request, contextData);
-		const context = this.#createContext(request, contextData, { portableRequest: portable.portableRequest });
+		const ownerContext = this.#createContext(request, contextData, { [outboundProvenance]: provenance });
+		const context = this.#createContext(request, contextData, {
+			[outboundProvenance]: provenance,
+			portableRequest: portable.portableRequest
+		});
 		const reject = async (reason, identifier) => {
 			logger.debug("Not serving the portable collection {collection} of the actor {identifier}: {reason}.", {
 				collection: portable.canonicalId,
@@ -6993,6 +8096,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 		}
 		return await onNotFound(request);
 	}
+	#createInboxRequestSource(request, metricState) {
+		const source = createInboxRequestSource(request, metricState.requestId ?? getRequestId(request), this.origin?.webOrigin);
+		metricState.inboxRequest = source;
+		return source;
+	}
 	#setInboxCompletion(metricState, observation, getContext) {
 		const handler = this.inboxRequestFinishedHandler;
 		metricState.inboxCompletion = {
@@ -7024,6 +8132,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 	async #reportInboxPreparationError(request, contextData, inbox, error, span, metricState) {
 		const unavailableLoader = () => Promise.reject(error);
 		const context = new RequestContextImpl({
+			[outboundProvenance]: globalThis.Object.freeze({ inboxRequest: metricState.inboxRequest }),
 			url: new URL(request.url),
 			request,
 			federation: this,
@@ -7031,7 +8140,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 			documentLoader: unavailableLoader,
 			contextLoader: unavailableLoader
 		});
-		const observation = new InboxObservation(inbox);
+		const observation = new InboxObservation(inbox, metricState.inboxRequest);
 		this.#setInboxCompletion(metricState, observation, () => context);
 		return await observation.run(() => Promise.reject(error), span);
 	}
@@ -7089,9 +8198,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 		metricState.endpoint = "inbox";
 		span.updateName(`${request.method} ${metricState.routeTemplate}`);
 		const identifier = route.values.identifier;
+		const source = this.#createInboxRequestSource(request, metricState);
 		let context;
 		try {
-			context = this.#createContext(request, contextData);
+			context = withContext({ inboxRequestId: source.inboxRequestId }, () => this.#createContext(request, contextData, { [outboundProvenance]: globalThis.Object.freeze({ inboxRequest: source }) }));
 		} catch (error) {
 			return await this.#reportInboxPreparationError(request, contextData, {
 				kind: "portable",
@@ -7101,9 +8211,9 @@ var FederationImpl = class extends FederationBuilderImpl {
 		const observation = new InboxObservation({
 			kind: "portable",
 			recipient: identifier
-		});
+		}, source);
 		this.#setInboxCompletion(metricState, observation, () => context);
-		return await observation.run(async () => {
+		return await withContext({ inboxRequestId: source.inboxRequestId }, () => observation.run(async () => {
 			const actor = this.actorCallbacks?.dispatcher == null ? null : await this.actorCallbacks.dispatcher(context, identifier);
 			const localOrigin = context.canonicalOrigin;
 			const resolution = resolvePortableInboxRecipient(actor == null || actor instanceof Tombstone ? null : actor, {
@@ -7120,7 +8230,10 @@ var FederationImpl = class extends FederationBuilderImpl {
 				return await onNotFound(request);
 			}
 			const { recipient } = resolution;
-			if (this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(request, contextData, { documentLoader: await context.getDocumentLoader({ identifier }) });
+			if (this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(request, contextData, {
+				[outboundProvenance]: getOutboundProvenance(context),
+				documentLoader: await context.getDocumentLoader({ identifier })
+			});
 			const excludedOrigins = [localOrigin, new URL(request.url).origin];
 			return await this.#handleInbox(request, {
 				recipient: identifier,
@@ -7130,16 +8243,25 @@ var FederationImpl = class extends FederationBuilderImpl {
 				onNotFound,
 				portableInbox: {
 					recipient,
-					forward: async (activity, activityId, activityType, signatures) => {
-						if (this.portableInboxForwarding.maxTargets < 1) return;
+					forward: (activity, activityId, activityType, signatures) => observeOutboundBroadcast(context, this.outboundObservers, async (observation) => {
+						observation?.setActivity({
+							...activityId == null ? {} : { id: activityId.href },
+							type: activityType
+						});
+						if (this.portableInboxForwarding.maxTargets < 1) {
+							if (observation != null) await observation.skipped("forwarding-disabled");
+							return;
+						}
 						if (this.kv.cas == null) {
 							if (!this.#portableInboxForwardingWarned) {
 								this.#portableInboxForwardingWarned = true;
 								logger.warn("Activities delivered to FEP-ef61 portable inboxes are not forwarded to the other gateways, as the key–value store does not support compare-and-swap (KvStore.cas()), which is needed to forward each activity at most once.");
 							}
+							if (observation != null) await observation.skipped("forwarding-unavailable");
 							return;
 						}
 						await forwardPortableInboxActivity({
+							provenance: getOutboundProvenance(context),
 							recipient,
 							activity,
 							activityId,
@@ -7157,11 +8279,11 @@ var FederationImpl = class extends FederationBuilderImpl {
 							options: this.portableInboxForwarding,
 							meterProvider: this.meterProvider,
 							tracerProvider: this.tracerProvider
-						});
-					}
+						}, observation);
+					}, "portable-replication")
 				}
 			});
-		}, span);
+		}, span));
 	}
 	/**
 	* Gets this server's gateway key pairs for the portable actor that owns
@@ -7231,7 +8353,7 @@ var FederationImpl = class extends FederationBuilderImpl {
 				headers: { "Content-Type": "text/plain; charset=utf-8" }
 			});
 		}
-		const context = this.#createContext(request, contextData);
+		const context = this.#createContext(request, contextData, { [outboundProvenance]: globalThis.Object.freeze({ originatingRequestId: metricState.requestId }) });
 		const response = await this.hashlinkMediaDispatcher(context, media.media);
 		if (response == null) return await onNotFound(request);
 		return await discardBodyForHead(request, response, false);
@@ -7373,8 +8495,9 @@ var ContextImpl = class ContextImpl {
 	verifyPortableObject;
 	invokedFromActorKeyPairsDispatcher;
 	#codec;
-	constructor({ url, federation, data, documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher }) {
+	constructor({ url, federation, data, documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher, [outboundProvenance]: provenance = emptyProvenance }) {
 		this.url = url;
+		globalThis.Object.assign(this, { [outboundProvenance]: provenance });
 		this.federation = federation;
 		this.data = data;
 		this.documentLoader = documentLoader;
@@ -7396,6 +8519,7 @@ var ContextImpl = class ContextImpl {
 	}
 	clone(data) {
 		return new ContextImpl({
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data,
@@ -7406,6 +8530,7 @@ var ContextImpl = class ContextImpl {
 	}
 	toInboxContext(recipient, activity, activityId, activityType, metadata) {
 		return new InboxContextImpl(recipient, activity, activityId, activityType, {
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data: this.data,
@@ -7416,6 +8541,7 @@ var ContextImpl = class ContextImpl {
 	}
 	toOutboxContext(identifier, activity, activityId, activityType) {
 		return new OutboxContextImpl(identifier, activity, activityId, activityType, {
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data: this.data,
@@ -7805,36 +8931,49 @@ var ContextImpl = class ContextImpl {
 		await this.#enqueueTasks(task, payloads, options);
 	}
 	sendActivity(sender, recipients, activity, options = {}) {
-		return this.tracerProvider.getTracer(name, version).startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
-			kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
-			attributes: {
-				"activitypub.activity.type": getTypeId(activity).href,
-				"activitypub.activity.to": activity.toIds.map((to) => to.href),
-				"activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
-				"activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
-				"activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
-			}
-		}, async (span) => {
-			try {
-				if (activity.id != null) span.setAttribute("activitypub.activity.id", activity.id.href);
-				await this.sendActivityInternal(sender, recipients, activity, options, span);
-			} catch (e) {
-				span.setStatus({
-					code: SpanStatusCode.ERROR,
-					message: String(e)
-				});
-				throw e;
-			} finally {
-				span.end();
-			}
-		});
+		return observeOutboundBroadcast(this, this.federation.outboundObservers, (observation, provenance) => {
+			const tracer = this.tracerProvider.getTracer(name, version);
+			const parentSpanContext = trace.getSpanContext(context.active());
+			const activityType = getTypeId(activity).href;
+			observation?.setActivity({ type: activityType });
+			return tracer.startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
+				kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
+				attributes: {
+					"activitypub.activity.type": activityType,
+					"activitypub.activity.to": activity.toIds.map((to) => to.href),
+					"activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
+					"activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
+					"activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
+				}
+			}, async (span) => {
+				observation?.setSpan(span, parentSpanContext);
+				try {
+					const activityId = activity.id?.href;
+					if (activityId != null) span.setAttribute("activitypub.activity.id", activityId);
+					observation?.setActivity({
+						type: activityType,
+						...activityId == null ? {} : { id: activityId }
+					});
+					await this.sendActivityInternal(sender, recipients, activity, options, span, observation, provenance);
+				} catch (e) {
+					span.setStatus({
+						code: SpanStatusCode.ERROR,
+						message: String(e)
+					});
+					throw e;
+				} finally {
+					span.end();
+				}
+			});
+		}, "send", options.dispatchId);
 	}
-	async sendActivityInternal(sender, recipients, activity, options, span) {
+	async sendActivityInternal(sender, recipients, activity, options, span, observation, provenance = getOutboundProvenance(this)) {
 		const logger = getLogger([
 			"fedify",
 			"federation",
 			"outbox"
 		]);
+		observation?.setPhase("sender-resolution");
 		let keys;
 		let identifier = null;
 		let actorKeyPairs = null;
@@ -7850,6 +8989,7 @@ var ContextImpl = class ContextImpl {
 				}
 			}
 			span.setAttribute("fedify.actor.identifier", identifier);
+			observation?.setPhase("key-resolution");
 			if (this.federation.actorCallbacks?.keyPairsDispatcher == null) throw new Error("No actor key pairs dispatcher registered.");
 			actorKeyPairs = await this.getActorKeyPairs(identifier);
 			if (actorKeyPairs.length < 1) throw new Error(`No key pair found for actor ${JSON.stringify(identifier)}.`);
@@ -7858,11 +8998,14 @@ var ContextImpl = class ContextImpl {
 				privateKey: kp.privateKey
 			}));
 		} else if (Array.isArray(sender)) {
+			observation?.setPhase("key-resolution");
 			if (sender.length < 1) throw new Error("The sender's key pairs are empty.");
 			keys = sender;
 		} else keys = [sender];
 		if (keys.length < 1) throw new TypeError("The sender's keys must not be empty.");
+		observation?.setPhase("key-resolution");
 		for (const { privateKey } of keys) validateCryptoKey(privateKey, "private");
+		observation?.setPhase("recipient-resolution");
 		let expandedRecipients;
 		let collectionSync;
 		if (Array.isArray(recipients)) expandedRecipients = recipients;
@@ -7878,14 +9021,34 @@ var ContextImpl = class ContextImpl {
 		} else expandedRecipients = [recipients];
 		const opts = {
 			context: this,
+			observation,
+			provenance,
 			orderingKey: options.orderingKey,
 			collectionSync,
 			immediate: options.immediate,
 			normalizeExistingProofs: options.normalizeExistingProofs
 		};
 		span.setAttribute("activitypub.inboxes", expandedRecipients.length);
+		observation?.setPhase("transformation");
+		if (this.federation.activityTransformers.length > 0) observation?.setActivity({});
 		for (const activityTransformer of this.federation.activityTransformers) activity = activityTransformer(activity, this);
-		span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? "");
+		let transformedActivityId;
+		try {
+			transformedActivityId = activity?.id?.href;
+		} finally {
+			if (observation != null) {
+				let activityType;
+				try {
+					activityType = getTypeId(activity)?.href;
+				} catch {}
+				observation.setActivity({
+					...transformedActivityId == null ? {} : { id: transformedActivityId },
+					...activityType == null ? {} : { type: activityType }
+				});
+			}
+		}
+		span?.setAttribute("activitypub.activity.id", transformedActivityId ?? "");
+		observation?.setPhase("validation");
 		if (activity.actorId == null) {
 			logger.error("Activity {activityId} to send does not have an actor.", {
 				activity,
@@ -7893,6 +9056,7 @@ var ContextImpl = class ContextImpl {
 			});
 			throw new TypeError("The activity to send must have at least one actor property.");
 		}
+		observation?.setPhase("signing");
 		const { activity: signedActivity, proofCreated } = await signOutgoingActivity(activity, actorKeyPairs == null ? keys.map(({ keyId, privateKey }) => ({
 			verificationMethod: keyId,
 			privateKey
@@ -7905,16 +9069,19 @@ var ContextImpl = class ContextImpl {
 			appendToExistingProofs: actorKeyPairs != null
 		});
 		activity = signedActivity;
+		observation?.setPhase("destination-planning");
 		const inboxes = extractInboxes({
 			recipients: expandedRecipients,
 			preferSharedInbox: options.preferSharedInbox,
 			excludeBaseUris: options.excludeBaseUris
 		});
+		if (observation != null) await observation.planDestinations(inboxes);
 		if (globalThis.Object.keys(inboxes).length < 1) {
 			logger.debug("No inboxes found for activity {activityId}.", {
 				activityId: activity.id?.href,
 				activity
 			});
+			if (observation != null) await observation.skipped();
 			return false;
 		}
 		logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", {
@@ -7922,6 +9089,7 @@ var ContextImpl = class ContextImpl {
 			activityId: activity.id?.href,
 			activity
 		});
+		observation?.setPhase("serialization");
 		const activityJsonLd = await activity.toJsonLd({
 			format: "compact",
 			contextLoader: this.contextLoader
@@ -7942,33 +9110,48 @@ var ContextImpl = class ContextImpl {
 			});
 			return true;
 		}
+		observation?.setPhase("validation");
 		assertSupportedCompoundProofShape(activityJsonLd, activity.id?.href);
+		observation?.setPhase("key-resolution");
 		const keyJwkPairs = await Promise.all(keys.map(async ({ keyId, privateKey }) => ({
 			keyId: keyId.href,
 			privateKey: await exportJwk(privateKey)
 		})));
+		observation?.setPhase("request-preparation");
 		const carrier = {};
 		propagation.inject(context.active(), carrier);
+		const activityId = activity.id?.href;
+		const activityType = getTypeId(activity).href;
+		observation?.setActivity({
+			...activityId == null ? {} : { id: activityId },
+			type: activityType
+		});
 		const message = {
 			type: "fanout",
+			...serializeProvenance(provenance),
 			id: crypto.randomUUID(),
 			baseUrl: this.origin,
 			keys: keyJwkPairs,
 			inboxes: globalThis.Object.fromEntries(globalThis.Object.entries(inboxes).map(([k, { actorIds, sharedInbox, portableInbox, gatewayInboxes }]) => [k, {
 				actorIds: [...actorIds],
+				...observation?.continuation(k),
 				sharedInbox,
 				...getPortableMessageFields(portableInbox, gatewayInboxes)
 			}])),
 			activity: activityJsonLd,
-			activityId: activity.id?.href,
-			activityType: getTypeId(activity).href,
+			activityId,
+			activityType,
 			collectionSync: opts.collectionSync,
 			orderingKey: options.orderingKey,
 			normalizeExistingProofs: proofCreated || options.normalizeExistingProofs,
 			traceContext: carrier
 		};
 		if (!this.federation.manuallyStartQueue) this.federation._startQueueInternal(this.data);
-		await this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey });
+		if (observation != null) await observation.prepared("fanout");
+		await (observation == null ? this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey }) : observation.enqueue(globalThis.Object.keys(inboxes).map((inbox) => ({
+			inbox,
+			queueMessageId: message.id
+		})), "fanout", () => this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey })));
 		getFederationMetrics(this.federation.meterProvider).recordQueueTaskEnqueued({
 			role: "fanout",
 			queue: this.federation.fanoutQueue,
@@ -8229,6 +9412,7 @@ var RequestContextImpl = class RequestContextImpl extends ContextImpl {
 	}
 	clone(data) {
 		return new RequestContextImpl({
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data,
@@ -8400,30 +9584,40 @@ var RequestContextImpl = class RequestContextImpl extends ContextImpl {
 	}
 };
 function forwardActivity(ctx, loggerCategory, forwarder, recipients, options) {
-	return ctx.tracerProvider.getTracer(name, version).startActiveSpan(ctx.federation.outboxQueue == null || options?.immediate ? `activitypub.${loggerCategory}` : "activitypub.fanout", {
-		kind: ctx.federation.outboxQueue == null || options?.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
-		attributes: { "activitypub.activity.type": ctx.activityType }
-	}, async (span) => {
-		try {
-			if (ctx.activityId != null) span.setAttribute("activitypub.activity.id", ctx.activityId);
-			return await forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options);
-		} catch (e) {
-			span.setStatus({
-				code: SpanStatusCode.ERROR,
-				message: String(e)
-			});
-			throw e;
-		} finally {
-			span.end();
-		}
-	});
+	return observeOutboundBroadcast(ctx, ctx.federation.outboundObservers, (observation, provenance) => {
+		observation?.setActivity({
+			...ctx.activityId == null ? {} : { id: ctx.activityId },
+			type: ctx.activityType
+		});
+		const tracer = ctx.tracerProvider.getTracer(name, version);
+		const parentSpanContext = trace.getSpanContext(context.active());
+		return tracer.startActiveSpan(ctx.federation.outboxQueue == null || options?.immediate ? `activitypub.${loggerCategory}` : "activitypub.fanout", {
+			kind: ctx.federation.outboxQueue == null || options?.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
+			attributes: { "activitypub.activity.type": ctx.activityType }
+		}, async (span) => {
+			observation?.setSpan(span, parentSpanContext);
+			try {
+				if (ctx.activityId != null) span.setAttribute("activitypub.activity.id", ctx.activityId);
+				return await forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options, observation, provenance);
+			} catch (e) {
+				span.setStatus({
+					code: SpanStatusCode.ERROR,
+					message: String(e)
+				});
+				throw e;
+			} finally {
+				span.end();
+			}
+		});
+	}, "forward", options?.dispatchId);
 }
-async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options) {
+async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options, observation, provenance = getOutboundProvenance(ctx)) {
 	const logger = getLogger([
 		"fedify",
 		"federation",
 		loggerCategory
 	]);
+	observation?.setPhase("sender-resolution");
 	let keys;
 	let identifier = null;
 	if ("identifier" in forwarder || "username" in forwarder) {
@@ -8437,6 +9631,7 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 				identifier = mapped;
 			}
 		}
+		observation?.setPhase("key-resolution");
 		const actorKeyPairs = await ctx.getActorKeyPairs(identifier);
 		if (actorKeyPairs.length < 1) throw new Error(`No key pair found for actor ${JSON.stringify(identifier)}.`);
 		keys = actorKeyPairs.map((kp) => ({
@@ -8444,12 +9639,17 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			privateKey: kp.privateKey
 		}));
 	} else if (Array.isArray(forwarder)) {
+		observation?.setPhase("key-resolution");
 		if (forwarder.length < 1) throw new Error("The forwarder's key pairs are empty.");
 		keys = forwarder;
 	} else keys = [forwarder];
+	observation?.setPhase("validation");
 	if (!hasSignatureLike(ctx.activity)) {
 		if (!hasProofLike(ctx.activity)) {
-			if (options?.skipIfUnsigned) return false;
+			if (options?.skipIfUnsigned) {
+				if (observation != null) await observation.skipped("unsigned");
+				return false;
+			}
 			logger.warn("The activity {activityId} is not signed; even if it is forwarded to other servers as is, it may not be accepted by them due to the lack of a signature/proof.", {
 				activityId: ctx.activityId,
 				activityType: ctx.activityType,
@@ -8457,23 +9657,27 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			});
 		}
 	}
+	observation?.setPhase("recipient-resolution");
 	if (recipients === "followers") {
 		if (identifier == null) throw new Error("If recipients is \"followers\", forwarder must be an actor identifier or username.");
 		const followers = [];
 		for await (const recipient of ctx.getFollowers(identifier)) followers.push(recipient);
 		recipients = followers;
 	}
+	observation?.setPhase("destination-planning");
 	const inboxes = extractInboxes({
 		recipients: Array.isArray(recipients) ? recipients : [recipients],
 		preferSharedInbox: options?.preferSharedInbox,
 		excludeBaseUris: options?.excludeBaseUris
 	});
+	if (observation != null) await observation.planDestinations(inboxes);
 	if (globalThis.Object.keys(inboxes).length < 1) {
 		logger.debug("No inboxes found for activity {activityId}.", {
 			activityId: ctx.activityId,
 			activityType: ctx.activityType,
 			identifier: identifier ?? void 0
 		});
+		if (observation != null) await observation.skipped();
 		return false;
 	}
 	logger.debug("Forwarding activity {activityId} to inboxes:\n{inboxes}", {
@@ -8484,18 +9688,33 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 	if (options?.immediate || ctx.federation.outboxQueue == null) {
 		if (options?.immediate) logger.debug("Forwarding activity immediately without queue since immediate option is set.");
 		else logger.debug("Forwarding activity immediately without queue since queue is not set.");
+		observation?.setPhase("request-preparation");
 		const promises = [];
-		for (const inbox in inboxes) promises.push(sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
-			allowPrivateAddress: ctx.federation.allowPrivateAddress,
-			keys,
-			activity: ctx.activity,
-			activityId: ctx.activityId,
-			activityType: ctx.activityType,
-			sharedInbox: inboxes[inbox].sharedInbox,
-			meterProvider: ctx.meterProvider,
-			tracerProvider: ctx.tracerProvider,
-			specDeterminer: new KvSpecDeterminer(ctx.federation.kv, ctx.federation.kvPrefixes.httpMessageSignaturesSpec, ctx.federation.firstKnock, { specTtl: ctx.federation.httpMessageSignaturesSpecTtl })
-		}));
+		for (const inbox in inboxes) {
+			const delivery = observation?.delivery(inbox);
+			let sending;
+			try {
+				sending = sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
+					observation: delivery,
+					allowPrivateAddress: ctx.federation.allowPrivateAddress,
+					keys,
+					activity: ctx.activity,
+					activityId: ctx.activityId,
+					activityType: ctx.activityType,
+					sharedInbox: inboxes[inbox].sharedInbox,
+					meterProvider: ctx.meterProvider,
+					tracerProvider: ctx.tracerProvider,
+					specDeterminer: new KvSpecDeterminer(ctx.federation.kv, ctx.federation.kvPrefixes.httpMessageSignaturesSpec, ctx.federation.firstKnock, { specTtl: ctx.federation.httpMessageSignaturesSpecTtl })
+				});
+			} catch (error) {
+				if (delivery != null) await delivery.failed(error);
+				throw error;
+			}
+			if (observation != null) sending.catch(() => {});
+			if (delivery != null) await delivery.handoff({ type: "direct" });
+			promises.push(sending);
+		}
+		if (observation != null) await observation.prepared("delivery");
 		await Promise.all(promises);
 		return true;
 	}
@@ -8503,7 +9722,9 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 		activityId: ctx.activityId,
 		activity: ctx.activity
 	});
+	observation?.setPhase("request-preparation");
 	if (!ctx.federation.manuallyStartQueue) ctx.federation._startQueueInternal(ctx.data);
+	observation?.setPhase("key-resolution");
 	const keyJwkPairs = [];
 	for (const { keyId, privateKey } of keys) {
 		const privateKeyJwk = await exportJwk(privateKey);
@@ -8512,15 +9733,17 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			privateKey: privateKeyJwk
 		});
 	}
+	observation?.setPhase("request-preparation");
 	const carrier = {};
 	propagation.inject(context.active(), carrier);
 	const orderingKey = options?.orderingKey;
 	const started = (/* @__PURE__ */ new Date()).toISOString();
 	const messages = [];
-	for (const inbox in inboxes) {
+	for (const inbox in inboxes) try {
 		const { portableInbox, gatewayInboxes } = inboxes[inbox];
 		const message = {
 			type: "outbox",
+			...serializeProvenance(provenance),
 			id: crypto.randomUUID(),
 			baseUrl: ctx.origin,
 			keys: keyJwkPairs,
@@ -8528,6 +9751,7 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			activityId: ctx.activityId,
 			activityType: ctx.activityType,
 			inbox,
+			...observation?.continuation(inbox),
 			sharedInbox: inboxes[inbox].sharedInbox,
 			actorIds: [...inboxes[inbox].actorIds],
 			started,
@@ -8541,11 +9765,18 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 			message,
 			orderingKey: message.orderingKey
 		});
+	} catch (error) {
+		if (observation != null) await observation.delivery(inbox)?.failed(error);
+		throw error;
 	}
+	if (observation != null) await observation.prepared("delivery");
 	const { outboxQueue } = ctx.federation;
 	if (outboxQueue.enqueueMany == null || orderingKey != null) {
 		const errors = (await Promise.allSettled(messages.map(async (m) => {
-			await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
+			await (observation == null ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) : observation.enqueue([{
+				inbox: m.message.inbox,
+				queueMessageId: m.message.id
+			}], "outbox", () => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey })));
 			recordOutboxEnqueue(ctx.federation.meterProvider, outboxQueue, m.message);
 		}))).filter((r) => r.status === "rejected").map((r) => r.reason);
 		if (errors.length > 0) {
@@ -8558,7 +9789,10 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
 		}
 	} else {
 		try {
-			await outboxQueue.enqueueMany(messages.map((m) => m.message));
+			await (observation == null ? outboxQueue.enqueueMany(messages.map((m) => m.message)) : observation.enqueue(messages.map(({ message }) => ({
+				inbox: message.inbox,
+				queueMessageId: message.id
+			})), "outbox", () => outboxQueue.enqueueMany(messages.map((m) => m.message))));
 		} catch (error) {
 			logger.error("Failed to enqueue activity {activityId} to forward later:\n{error}", {
 				activityId: ctx.activityId,
@@ -8594,6 +9828,7 @@ var InboxContextImpl = class InboxContextImpl extends ContextImpl {
 	}
 	clone(data) {
 		return new InboxContextImpl(this.recipient, this.activity, this.activityId, this.activityType, {
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data,
@@ -8624,29 +9859,41 @@ var OutboxContextImpl = class OutboxContextImpl extends ContextImpl {
 		return this.#deliveryState.delivered;
 	}
 	sendActivity(sender, recipients, activity, options = {}) {
-		return this.tracerProvider.getTracer(name, version).startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
-			kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
-			attributes: {
-				"activitypub.activity.type": getTypeId(activity).href,
-				"activitypub.activity.to": activity.toIds.map((to) => to.href),
-				"activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
-				"activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
-				"activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
-			}
-		}, async (span) => {
-			try {
-				if (activity.id != null) span.setAttribute("activitypub.activity.id", activity.id.href);
-				if (await this.sendActivityInternal(sender, recipients, activity, options, span)) this.#deliveryState.delivered = true;
-			} catch (e) {
-				span.setStatus({
-					code: SpanStatusCode.ERROR,
-					message: String(e)
-				});
-				throw e;
-			} finally {
-				span.end();
-			}
-		});
+		return observeOutboundBroadcast(this, this.federation.outboundObservers, (observation, provenance) => {
+			const tracer = this.tracerProvider.getTracer(name, version);
+			const parentSpanContext = trace.getSpanContext(context.active());
+			const activityType = getTypeId(activity).href;
+			observation?.setActivity({ type: activityType });
+			return tracer.startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
+				kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
+				attributes: {
+					"activitypub.activity.type": activityType,
+					"activitypub.activity.to": activity.toIds.map((to) => to.href),
+					"activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
+					"activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
+					"activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
+				}
+			}, async (span) => {
+				observation?.setSpan(span, parentSpanContext);
+				try {
+					const activityId = activity.id?.href;
+					if (activityId != null) span.setAttribute("activitypub.activity.id", activityId);
+					observation?.setActivity({
+						type: activityType,
+						...activityId == null ? {} : { id: activityId }
+					});
+					if (await this.sendActivityInternal(sender, recipients, activity, options, span, observation, provenance)) this.#deliveryState.delivered = true;
+				} catch (e) {
+					span.setStatus({
+						code: SpanStatusCode.ERROR,
+						message: String(e)
+					});
+					throw e;
+				} finally {
+					span.end();
+				}
+			});
+		}, "send", options.dispatchId);
 	}
 	forwardActivity(forwarder, recipients, options) {
 		return forwardActivity(this, "outbox", forwarder, recipients, options).then((delivered) => {
@@ -8655,6 +9902,7 @@ var OutboxContextImpl = class OutboxContextImpl extends ContextImpl {
 	}
 	clone(data) {
 		return new OutboxContextImpl(this.identifier, this.activity, this.activityId, this.activityType, {
+			[outboundProvenance]: getOutboundProvenance(this),
 			url: this.url,
 			federation: this.federation,
 			data,
@@ -8716,4 +9964,4 @@ function getRequestId(request) {
 	return `req_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
 }
 //#endregion
-export { getEmbeddedFirstGateway as C, handleNodeInfoJrd as D, handleNodeInfo as E, actorDehydrator as O, getCompactRootId as S, handleBenchmarkTrigger as T, coversDelivery as _, OutboxContextImpl as a, PORTABLE_OBJECT_CONTENT_TYPE as b, assertPortableActorActivity as c, handleCustomCollection as d, handleInbox as f, respondWithObjectIfAcceptable as g, respondWithObject as h, KvSpecDeterminer as i, autoIdAssigner as k, handleActor as l, handleOutbox as m, FederationImpl as n, createFederation as o, handleObject as p, InboxContextImpl as r, handleWebFinger as s, ContextImpl as t, handleCollection as u, forwardPortableInboxActivity as v, warnCompatibleIdsInJson as w, parseHashlinkGatewayRequest as x, resolvePortableInboxForwardingOptions as y };
+export { OutboundAttemptObservation as A, getEmbeddedFirstGateway as C, handleNodeInfoJrd as D, handleNodeInfo as E, observeOutboundBroadcast as M, actorDehydrator as O, getCompactRootId as S, handleBenchmarkTrigger as T, coversDelivery as _, OutboxContextImpl as a, PORTABLE_OBJECT_CONTENT_TYPE as b, assertPortableActorActivity as c, handleCustomCollection as d, handleInbox as f, respondWithObjectIfAcceptable as g, respondWithObject as h, KvSpecDeterminer as i, OutboundBroadcastObservation as j, autoIdAssigner as k, handleActor as l, handleOutbox as m, FederationImpl as n, createFederation as o, handleObject as p, InboxContextImpl as r, handleWebFinger as s, ContextImpl as t, handleCollection as u, forwardPortableInboxActivity as v, warnCompatibleIdsInJson as w, parseHashlinkGatewayRequest as x, resolvePortableInboxForwardingOptions as y };