@fedify/fedify 2.5.0-dev.2609 → 2.5.0-dev.2618

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/dist/{builder-BLCvt0zp.mjs → builder-C-zhu1PC.mjs} +82 -3
  2. package/dist/compat/mod.d.cts +1 -1
  3. package/dist/compat/mod.d.ts +1 -1
  4. package/dist/compat/transformers.test.mjs +1 -1
  5. package/dist/{compound-proof-DNhLvscb.mjs → compound-proof-B6Mz3pzt.mjs} +2 -2
  6. package/dist/{context-ByflxsY9.d.cts → context-C1NqcIct.d.cts} +1061 -443
  7. package/dist/{context-uQUY5fQT.d.ts → context-xrqq4Hrq.d.ts} +1064 -446
  8. package/dist/{deno-DYkYmKp0.mjs → deno-CaPDSXDl.mjs} +1 -1
  9. package/dist/{docloader-B9RO5nh9.mjs → docloader-kYMkiXaB.mjs} +2 -2
  10. package/dist/federation/builder.test.mjs +1 -1
  11. package/dist/federation/compound-fanout.test.mjs +3 -3
  12. package/dist/federation/handler-compound-proof.test.mjs +3 -3
  13. package/dist/federation/handler.test.mjs +4 -4
  14. package/dist/federation/hashlink-media.test.mjs +2 -2
  15. package/dist/federation/idempotency.test.mjs +2 -2
  16. package/dist/federation/inbox-actorless.test.mjs +5 -5
  17. package/dist/federation/inbox-report.test.mjs +220 -5
  18. package/dist/federation/inbox-view.test.mjs +1 -1
  19. package/dist/federation/inbox.test.mjs +1 -1
  20. package/dist/federation/metrics.test.mjs +1 -1
  21. package/dist/federation/middleware.test.mjs +7 -7
  22. package/dist/federation/mod.cjs +1 -1
  23. package/dist/federation/mod.d.cts +3 -3
  24. package/dist/federation/mod.d.ts +4 -4
  25. package/dist/federation/mod.js +1 -1
  26. package/dist/federation/mq.test.mjs +1 -1
  27. package/dist/federation/outbound-body.test.d.mts +2 -0
  28. package/dist/federation/outbound-body.test.mjs +149 -0
  29. package/dist/federation/outbound-capture.test.d.mts +2 -0
  30. package/dist/federation/outbound-capture.test.mjs +394 -0
  31. package/dist/federation/outbound-dispatch.test.d.mts +2 -0
  32. package/dist/federation/outbound-dispatch.test.mjs +368 -0
  33. package/dist/federation/outbound-document.test.d.mts +2 -0
  34. package/dist/federation/outbound-document.test.mjs +171 -0
  35. package/dist/federation/outbound-provenance.test.d.mts +2 -0
  36. package/dist/federation/outbound-provenance.test.mjs +543 -0
  37. package/dist/federation/outbound-report.test.d.mts +2 -0
  38. package/dist/federation/outbound-report.test.mjs +5475 -0
  39. package/dist/federation/outbound-workers.test.d.mts +2 -0
  40. package/dist/federation/outbound-workers.test.mjs +64 -0
  41. package/dist/federation/outbound-workflow.test.d.mts +2 -0
  42. package/dist/federation/outbound-workflow.test.mjs +154 -0
  43. package/dist/federation/outgoing-proof.test.mjs +4 -4
  44. package/dist/federation/portable-actor.test.mjs +3 -3
  45. package/dist/federation/portable-chain.test.mjs +3 -3
  46. package/dist/federation/portable-collection.test.mjs +4 -4
  47. package/dist/federation/portable-delivery.test.mjs +106 -7
  48. package/dist/federation/portable-gateway-keys.test.mjs +5 -5
  49. package/dist/federation/portable-inbox.test.mjs +430 -5
  50. package/dist/federation/portable-parse-uri.test.mjs +1 -1
  51. package/dist/federation/portable-route-activity.test.mjs +2 -2
  52. package/dist/federation/portable.test.mjs +3 -3
  53. package/dist/federation/send.test.mjs +44 -3
  54. package/dist/federation/tasks/enqueue.test.mjs +1 -1
  55. package/dist/federation/tasks/tasks.test.mjs +2 -2
  56. package/dist/federation/temporal.test.mjs +1 -1
  57. package/dist/federation/webfinger.test.mjs +1 -1
  58. package/dist/{federation-DzMdu4zU.cjs → federation-c-ulWHXG.cjs} +2358 -405
  59. package/dist/{http-EoFzzh1s.js → http-C12QVNZt.js} +62 -32
  60. package/dist/{http-BdTJujR0.cjs → http-DwL_g670.cjs} +62 -32
  61. package/dist/{http-TgeyuTDM.mjs → http-NYqvoigN.mjs} +64 -34
  62. package/dist/{inbox-ChkbRxCN.mjs → inbox-CkL5YgHZ.mjs} +90 -3
  63. package/dist/{inbox-view-BosBZdlt.mjs → inbox-view-IJCZG0_b.mjs} +1 -1
  64. package/dist/{key-DbLNT-vA.d.ts → key-dwwAg5bA.d.ts} +1 -1
  65. package/dist/{kv-cache-CMgOYiLU.js → kv-cache-1BO1Deou.js} +1 -1
  66. package/dist/{kv-cache-CTA2i08a.mjs → kv-cache-BdFnnrOU.mjs} +1 -1
  67. package/dist/{kv-cache-B0o18_Ni.cjs → kv-cache-CcGUsFoZ.cjs} +1 -1
  68. package/dist/{ld-CZsMAfOw.mjs → ld-CEOS8Pho.mjs} +2 -2
  69. package/dist/{ld-Inagww4R.d.ts → ld-o4LnhCYY.d.ts} +2 -2
  70. package/dist/{metrics-Kc9mP1LD.mjs → metrics-CvuJeWaJ.mjs} +1 -1
  71. package/dist/{metrics-DoRtI0qU.d.ts → metrics-DVKYY6K1.d.ts} +1 -1
  72. package/dist/{middleware-C1coPz8l.mjs → middleware-BPZ9KhAd.mjs} +1 -1
  73. package/dist/{middleware-B-pjIvQ8.js → middleware-CzcR7pNr.js} +2303 -351
  74. package/dist/{middleware-CIRoIVSU.mjs → middleware-DMrpt2wD.mjs} +1573 -325
  75. package/dist/{mod-B7j30IQf.d.ts → mod-CQC9xlLZ.d.ts} +3 -3
  76. package/dist/{mod-B4kBw1-8.d.ts → mod-D22ZsiKi.d.ts} +2 -2
  77. package/dist/{mod-BN0TTfyQ.d.ts → mod-DO0epJGH.d.ts} +1 -1
  78. package/dist/{mod-DF2gbN_d.d.cts → mod-TwiQgEf3.d.cts} +1 -1
  79. package/dist/mod.cjs +4 -4
  80. package/dist/mod.d.cts +4 -4
  81. package/dist/mod.d.ts +8 -8
  82. package/dist/mod.js +4 -4
  83. package/dist/nodeinfo/handler.test.mjs +1 -1
  84. package/dist/outbound-body-CckPA5gc.mjs +83 -0
  85. package/dist/outbound-capture-DVgnRzlf.mjs +184 -0
  86. package/dist/outbound-document-BMgwLXTY.mjs +179 -0
  87. package/dist/{owner-Dwz4fGUW.mjs → owner-BOyHaAlU.mjs} +3 -3
  88. package/dist/{portable-collection-BikiijSk.js → portable-collection-B4HFXlgN.js} +2 -2
  89. package/dist/{portable-collection-Oxfavx8f.mjs → portable-collection-BBSMcaDY.mjs} +2 -2
  90. package/dist/{portable-collection-Ci6vIhbR.cjs → portable-collection-D_zZTK4C.cjs} +2 -2
  91. package/dist/{portable-key--fWpTNDN.mjs → portable-key-CF8xK_lj.mjs} +1 -1
  92. package/dist/{proof-D6Kb0Qq9.mjs → proof-VrE-GaGv.mjs} +3 -3
  93. package/dist/{send-BFti0L3I.mjs → send-CY6agn9T.mjs} +153 -40
  94. package/dist/sig/compound-proof-discovery.test.mjs +1 -1
  95. package/dist/sig/compound-proof-producer.test.mjs +1 -1
  96. package/dist/sig/compound-proof-verification.test.mjs +2 -2
  97. package/dist/sig/compound-proof.test.mjs +2 -2
  98. package/dist/sig/http.test.mjs +38 -2
  99. package/dist/sig/key.test.mjs +1 -1
  100. package/dist/sig/ld.test.mjs +1 -1
  101. package/dist/sig/mitra-receive-interop.test.mjs +4 -4
  102. package/dist/sig/mitra-send-interop.test.mjs +3 -3
  103. package/dist/sig/mod.cjs +2 -2
  104. package/dist/sig/mod.d.ts +3 -3
  105. package/dist/sig/mod.js +2 -2
  106. package/dist/sig/owner.test.mjs +1 -1
  107. package/dist/sig/portable-collection.test.mjs +1 -1
  108. package/dist/sig/portable-interop.test.mjs +3 -3
  109. package/dist/sig/portable-key.test.mjs +5 -5
  110. package/dist/sig/proof.test.mjs +3 -3
  111. package/dist/sig/tootik-request-interop.test.mjs +4 -4
  112. package/dist/sig/tootik-send-interop.test.mjs +3 -3
  113. package/dist/sig/verification.test.mjs +3 -3
  114. package/dist/{temporal-DlYL_raW.mjs → temporal-QO2pe6W4.mjs} +1 -1
  115. package/dist/testing/kv.test.mjs +1 -1
  116. package/dist/testing/mod.d.mts +1030 -412
  117. package/dist/utils/docloader.test.mjs +2 -2
  118. package/dist/utils/kv-cache.test.mjs +1 -1
  119. package/dist/utils/mod.cjs +1 -1
  120. package/dist/utils/mod.d.ts +1 -1
  121. package/dist/utils/mod.js +1 -1
  122. package/package.json +7 -7
  123. package/skills/fedify/SKILL.md +60 -1
@@ -2,15 +2,15 @@ import { Temporal } from "temporal-polyfill";
2
2
  import { URLPattern } from "urlpattern-polyfill";
3
3
  import { t as __exportAll } from "./rolldown-runtime-BC-Eo5Mj.js";
4
4
  import { r as getDefaultActivityTransformers } from "./transformers-DAtXl_Kc.js";
5
- import { $ as getVerifiedKeyOwnerEvidence, A as compactJsonLd, At as getRawCanonicalPortableId, B as verifyCompactJsonLdDetailed, C as verifyPortableObjectProofWithRoot, Ct as observeAttempt, D as InvalidContextReferenceError, Dt as getCanonicalPortableId, E as normalizeOutgoingActivityJsonLd, Et as getAuthenticationOrigin, F as hasSignatureLike, Ft as isPortableUri, I as isClearlyMalformedContextReference, It as isSameObjectId, L as isInvalidUrlTypeError, M as detachSignature, Mt as isCompatibleKeyId, N as getNormalizationContextLoader, Nt as isPortableId, O as assertSafeJsonLd, Ot as getGatewayKeyBase, P as hasSignature, Pt as isPortableKeyId, Q as MAX_PORTABLE_KEY_TTL, R as markEffectiveContextLoader, Rt as name, S as verifyPortableObjectProofPolicy, Tt as verificationObservation, U as wrapContextLoaderForJsonLd, W as exportJwk, X as validateCryptoKey, Y as importJwk, _ as signObject, _t as registerQueueDepthGauge, at as isAbortError, b as verifyObjectDetailed, c as verifyRequestDetailed, ct as recordCollectionPageItems, d as InvalidPortableObjectIdError, f as classifyFep2277CoreType, ft as recordFanoutRecipients, g as preparePortableObjectProofPolicy, gt as recordWebFingerHandle, h as hasProofLike, ht as recordOutboxEnqueue, i as selectRequestSignature, it as instrumentDocumentLoader, jt as hasPortableActor, kt as getPortableDid, lt as recordCollectionRequest, m as getProofSigner, mt as recordOutboxActivity, n as listRequestSignatures, nt as getFederationMetrics, o as validateMaxSignatures, ot as recordCircuitBreakerStateChange, pt as recordInboxActivity, r as parseRfc9421SignatureInput, rt as getRemoteHost, s as verifyRequest, st as recordCollectionDispatchDuration, t as doubleKnockScoped, tt as getDurationMs, ut as recordCollectionTotalItems, v as verifyMapLocalProof, vt as preloadedOnlyDocumentLoader, w as verifyPreparedPortableObjectProofPolicy, wt as snapshotKey, x as verifyPortableObjectProof, y as verifyObject, yt as formatAcceptSignature, z as signJsonLd, zt as version } from "./http-EoFzzh1s.js";
5
+ import { $ as getVerifiedKeyOwnerEvidence, A as compactJsonLd, At as getRawCanonicalPortableId, B as verifyCompactJsonLdDetailed, C as verifyPortableObjectProofWithRoot, Ct as observeAttempt, D as InvalidContextReferenceError, Dt as getCanonicalPortableId, E as normalizeOutgoingActivityJsonLd, Et as getAuthenticationOrigin, F as hasSignatureLike, Ft as isPortableUri, I as isClearlyMalformedContextReference, It as isSameObjectId, L as isInvalidUrlTypeError, M as detachSignature, Mt as isCompatibleKeyId, N as getNormalizationContextLoader, Nt as isPortableId, O as assertSafeJsonLd, Ot as getGatewayKeyBase, P as hasSignature, Pt as isPortableKeyId, Q as MAX_PORTABLE_KEY_TTL, R as markEffectiveContextLoader, Rt as name, S as verifyPortableObjectProofPolicy, Tt as verificationObservation, U as wrapContextLoaderForJsonLd, W as exportJwk, X as validateCryptoKey, Y as importJwk, _ as signObject, _t as registerQueueDepthGauge, at as isAbortError, b as verifyObjectDetailed, c as verifyRequestDetailed, ct as recordCollectionPageItems, d as InvalidPortableObjectIdError, f as classifyFep2277CoreType, ft as recordFanoutRecipients, g as preparePortableObjectProofPolicy, gt as recordWebFingerHandle, h as hasProofLike, ht as recordOutboxEnqueue, i as selectRequestSignature, it as instrumentDocumentLoader, jt as hasPortableActor, kt as getPortableDid, lt as recordCollectionRequest, m as getProofSigner, mt as recordOutboxActivity, n as listRequestSignatures, nt as getFederationMetrics, o as validateMaxSignatures, ot as recordCircuitBreakerStateChange, pt as recordInboxActivity, r as parseRfc9421SignatureInput, rt as getRemoteHost, s as verifyRequest, st as recordCollectionDispatchDuration, t as doubleKnockScoped, tt as getDurationMs, ut as recordCollectionTotalItems, v as verifyMapLocalProof, vt as preloadedOnlyDocumentLoader, w as verifyPreparedPortableObjectProofPolicy, wt as snapshotKey, x as verifyPortableObjectProof, y as verifyObject, yt as formatAcceptSignature, z as signJsonLd, zt as version } from "./http-C12QVNZt.js";
6
6
  import { a as MAX_BODY_SIZE, i as BodyTooLargeError, n as getNodeInfo, o as readBoundedText, t as nodeInfoToJson } from "./types-DUjZb4i6.js";
7
- import { n as doesActorOwnKey, r as getKeyOwner, t as verifyPortableObject } from "./portable-collection-BikiijSk.js";
8
- import { n as getAuthenticatedDocumentLoader, t as kvCache } from "./kv-cache-CMgOYiLU.js";
7
+ import { n as doesActorOwnKey, r as getKeyOwner, t as verifyPortableObject } from "./portable-collection-B4HFXlgN.js";
8
+ import { n as getAuthenticatedDocumentLoader, t as kvCache } from "./kv-cache-1BO1Deou.js";
9
9
  import { getLogger, withContext } from "@logtape/logtape";
10
+ import { FetchError, UrlError, canonicalizePortableUri, decodeMultibase, formatIri, fromCompatibleEf61Id, getDocumentLoader, getFe34Origin, isGatewayUrl, parseDigestMultibase, parseHashlink, parseIri, preloadedContexts, resolveDocumentLoaderTimeout, toCompatibleEf61Id } from "@fedify/vocab-runtime";
10
11
  import { Router, RouterError, assertPath } from "@fedify/uri-template";
11
12
  import { Activity, Collection, CollectionPage, CryptographicKey, Link, Multikey, Object as Object$1, OrderedCollection, OrderedCollectionPage, PUBLIC_COLLECTION, Tombstone, getTypeId, isActor, lookupObject, lookupObserver, traverseCollection } from "@fedify/vocab";
12
- import { SpanKind, SpanStatusCode, context, metrics, propagation, trace } from "@opentelemetry/api";
13
- import { FetchError, UrlError, canonicalizePortableUri, decodeMultibase, formatIri, fromCompatibleEf61Id, getDocumentLoader, getFe34Origin, isGatewayUrl, parseDigestMultibase, parseHashlink, parseIri, preloadedContexts, resolveDocumentLoaderTimeout, toCompatibleEf61Id } from "@fedify/vocab-runtime";
13
+ import { SpanKind, SpanStatusCode, context, isSpanContextValid, metrics, propagation, trace } from "@opentelemetry/api";
14
14
  import { getPortableGatewayCandidates, isCompatibleEf61Iri } from "@fedify/vocab-runtime/internal/portable-dereference";
15
15
  import { encodeHex } from "byte-encodings/hex";
16
16
  import jsonld from "@fedify/vocab-runtime/jsonld";
@@ -22,6 +22,500 @@ import { lookupWebFinger } from "@fedify/webfinger";
22
22
  import { DataPointType, MeterProvider as MeterProvider$1, MetricReader } from "@opentelemetry/sdk-metrics";
23
23
  import { parse, stringifyAsync } from "devalue";
24
24
  import { domainToASCII } from "node:url";
25
+ //#region src/federation/outbound-document.ts
26
+ const asContext = "https://www.w3.org/ns/activitystreams";
27
+ const asNamespace = `${asContext}#`;
28
+ const blind = /* @__PURE__ */ new Set([
29
+ "bto",
30
+ "bcc",
31
+ "as:bto",
32
+ "as:bcc",
33
+ `${asNamespace}bto`,
34
+ `${asNamespace}bcc`
35
+ ]);
36
+ const contextCandidates = [
37
+ asContext,
38
+ "https://w3id.org/security/v1",
39
+ "https://w3id.org/identity/v1",
40
+ "https://gotosocial.org/ns",
41
+ "https://w3id.org/security/data-integrity/v1",
42
+ "https://w3id.org/security/data-integrity/v2"
43
+ ];
44
+ function safeContext(context) {
45
+ if (Array.isArray(context)) return context.every(safeContext);
46
+ if (context == null || typeof context !== "object") return false;
47
+ for (const [name, definition] of Object.entries(context)) {
48
+ if (name === "@import") return false;
49
+ const mapping = typeof definition === "string" ? definition : definition != null && typeof definition === "object" ? definition["@id"] : void 0;
50
+ if (name === "as" && mapping !== asNamespace) return false;
51
+ if (name !== "as" && typeof mapping === "string" && mapping.startsWith(asNamespace) && (mapping.endsWith("#") || mapping.endsWith("/") || typeof definition === "object" && definition["@prefix"] === true)) return false;
52
+ if (typeof mapping === "string" && blind.has(mapping) && name !== "bto" && name !== "bcc") return false;
53
+ if ((name === "bto" || name === "bcc") && mapping !== `as:${name}` && mapping !== `${asNamespace}${name}`) return false;
54
+ if (name === "@vocab" && typeof mapping === "string" && mapping.startsWith(asNamespace)) return false;
55
+ if (definition != null && typeof definition === "object" && "@context" in definition && !safeContext(definition["@context"])) return false;
56
+ }
57
+ return true;
58
+ }
59
+ const contexts = new Set(contextCandidates.filter((url) => {
60
+ const document = preloadedContexts[url];
61
+ return document != null && typeof document === "object" && safeContext(document["@context"]);
62
+ }));
63
+ const generatedTerms = {
64
+ toot: "http://joinmastodon.org/ns#",
65
+ misskey: "https://misskey-hub.net/ns#",
66
+ fedibird: "http://fedibird.com/ns#",
67
+ sensitive: "as:sensitive",
68
+ votersCount: {
69
+ "@id": "toot:votersCount",
70
+ "@type": "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"
71
+ },
72
+ Emoji: "toot:Emoji",
73
+ Hashtag: "as:Hashtag",
74
+ ChatMessage: "http://litepub.social/ns#ChatMessage",
75
+ quote: {
76
+ "@id": "https://w3id.org/fep/044f#quote",
77
+ "@type": "@id"
78
+ },
79
+ quoteUrl: "as:quoteUrl",
80
+ _misskey_quote: "misskey:_misskey_quote",
81
+ quoteUri: "fedibird:quoteUri",
82
+ QuoteAuthorization: "https://w3id.org/fep/044f#QuoteAuthorization",
83
+ QuoteRequest: "https://w3id.org/fep/044f#QuoteRequest",
84
+ quoteAuthorization: {
85
+ "@id": "https://w3id.org/fep/044f#quoteAuthorization",
86
+ "@type": "@id"
87
+ },
88
+ emojiReactions: {
89
+ "@id": "fedibird:emojiReactions",
90
+ "@type": "@id"
91
+ }
92
+ };
93
+ function sameDefinition(value, expected) {
94
+ if (value === expected) return true;
95
+ if (value == null || expected == null || typeof value !== "object" || typeof expected !== "object" || Array.isArray(value)) return false;
96
+ const actual = value;
97
+ const reference = expected;
98
+ return Object.keys(actual).length === Object.keys(reference).length && Object.keys(actual).every((name) => Object.hasOwn(reference, name) && sameDefinition(actual[name], reference[name]));
99
+ }
100
+ function generatedContext(value) {
101
+ if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
102
+ return Object.entries(value).every(([name, definition]) => Object.hasOwn(generatedTerms, name) && sameDefinition(definition, generatedTerms[name]));
103
+ }
104
+ const keywords = /* @__PURE__ */ new Set([
105
+ "@id",
106
+ "@type",
107
+ "@value",
108
+ "@language",
109
+ "@direction",
110
+ "@index",
111
+ "@list",
112
+ "@set",
113
+ "@graph",
114
+ "@reverse",
115
+ "@included",
116
+ "@nest"
117
+ ]);
118
+ /** Validate the explicit opt-in before replacing observer registration. */
119
+ function snapshotSenderPayloadOption(value) {
120
+ if (value === void 0) return false;
121
+ if (typeof value !== "boolean") throw new TypeError("Outbound senderPayload must be a boolean.");
122
+ return value;
123
+ }
124
+ /** Project existing plain JSON only; no getters, JSON-LD processing or serializers. */
125
+ function projectSenderPayload(input) {
126
+ try {
127
+ let compact = false;
128
+ let redacted = false;
129
+ const seen = /* @__PURE__ */ new WeakSet();
130
+ const copy = (value, depth, contextData = false) => {
131
+ if (depth >= 64) throw new TypeError("Unsafe document depth.");
132
+ if (value === null || typeof value === "string" || typeof value === "boolean") return value;
133
+ if (typeof value === "number" && Number.isFinite(value)) return value;
134
+ if (typeof value !== "object" || value == null || seen.has(value)) throw new TypeError("Not independent JSON data.");
135
+ const array = Array.isArray(value);
136
+ const prototype = Object.getPrototypeOf(value);
137
+ if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) throw new TypeError("Unsupported document object.");
138
+ seen.add(value);
139
+ try {
140
+ const descriptors = Object.getOwnPropertyDescriptors(value);
141
+ if (Object.hasOwn(descriptors, "toJSON")) throw new TypeError("Custom JSON conversion is not projected.");
142
+ const names = Reflect.ownKeys(descriptors).filter((name) => descriptors[name].enumerable);
143
+ if (names.some((name) => typeof name !== "string" || !("value" in descriptors[name]))) throw new TypeError("Not plain JSON properties.");
144
+ let contextValue;
145
+ if (!contextData && !array && descriptors["@context"]?.enumerable) {
146
+ const context = copy(descriptors["@context"].value, depth + 1, true);
147
+ contextValue = context;
148
+ const urls = typeof context === "string" ? [context] : context;
149
+ if (!Array.isArray(urls) || urls.length === 0 || (depth === 0 ? !urls.includes(asContext) : !compact) || !urls.every((url) => typeof url === "string" ? contexts.has(url) : depth === 0 && generatedContext(url))) throw new TypeError("Unsupported document context.");
150
+ compact = true;
151
+ }
152
+ if (array) {
153
+ const length = descriptors.length.value;
154
+ if (names.length !== length || names.some((name, index) => name !== String(index))) throw new TypeError("Not a JSON array.");
155
+ return Object.freeze(names.map((name) => copy(descriptors[name].value, depth + 1, contextData)));
156
+ }
157
+ const result = Object.create(null);
158
+ for (const name of names) {
159
+ if (contextData) {
160
+ result[name] = copy(descriptors[name].value, depth + 1, true);
161
+ continue;
162
+ }
163
+ if (name === "@context") {
164
+ if (contextValue === void 0 || !compact) throw new TypeError("Scoped context not supported.");
165
+ result[name] = contextValue;
166
+ continue;
167
+ }
168
+ if (!compact && !keywords.has(name) && !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(name)) throw new TypeError("Unresolved document property.");
169
+ const next = copy(descriptors[name].value, depth + 1);
170
+ if (blind.has(name)) {
171
+ redacted = true;
172
+ continue;
173
+ }
174
+ result[name] = next;
175
+ }
176
+ return Object.freeze(result);
177
+ } finally {
178
+ seen.delete(value);
179
+ }
180
+ };
181
+ if (input == null || typeof input !== "object") throw new TypeError("Not an outgoing document.");
182
+ const value = copy(input, 0);
183
+ return Object.freeze({
184
+ stage: "sender-payload",
185
+ type: "available",
186
+ representation: "privacy-projection",
187
+ redacted,
188
+ value
189
+ });
190
+ } catch {
191
+ return Object.freeze({
192
+ stage: "sender-payload",
193
+ type: "unavailable",
194
+ reason: "privacy-filter-unavailable"
195
+ });
196
+ }
197
+ }
198
+ //#endregion
199
+ //#region src/federation/outbound-body.ts
200
+ /** One pull-driven response reader with an absolute lifetime and no extra queue. */
201
+ var ResponseBodyPump = class {
202
+ #reader;
203
+ #expiresAt;
204
+ #deadline;
205
+ #timer;
206
+ #expireDeadline;
207
+ #pending;
208
+ #timedOut = false;
209
+ #closed = false;
210
+ #ended = false;
211
+ constructor(body, timeoutMs) {
212
+ this.#reader = body.getReader();
213
+ this.#expiresAt = performance.now() + timeoutMs;
214
+ this.#deadline = new Promise((resolve) => {
215
+ this.#expireDeadline = () => resolve(void 0);
216
+ });
217
+ this.#timer = setTimeout(() => this.#expire(), timeoutMs);
218
+ }
219
+ /** Undefined means the absolute lifetime expired, independently of EOF. */
220
+ read() {
221
+ if (this.#closed) return Promise.resolve(this.#timedOut ? void 0 : {
222
+ done: true,
223
+ value: void 0
224
+ });
225
+ if (performance.now() >= this.#expiresAt) {
226
+ this.#expire();
227
+ return Promise.resolve(void 0);
228
+ }
229
+ this.#pending ??= Promise.race([this.#reader.read(), this.#deadline]).then((result) => {
230
+ this.#pending = void 0;
231
+ if (this.#timedOut || performance.now() >= this.#expiresAt) {
232
+ this.#expire();
233
+ return;
234
+ }
235
+ if (this.#closed) return {
236
+ done: true,
237
+ value: void 0
238
+ };
239
+ if (result?.done) {
240
+ this.#ended = true;
241
+ this.close();
242
+ }
243
+ return result;
244
+ }, (error) => {
245
+ this.#pending = void 0;
246
+ this.close();
247
+ throw error;
248
+ });
249
+ return this.#pending;
250
+ }
251
+ /** Shorten the remaining lifetime when ownership passes to diagnostic cleanup. */
252
+ limitLifetime(timeoutMs) {
253
+ if (this.#closed) return;
254
+ this.#expiresAt = Math.min(this.#expiresAt, performance.now() + Math.max(0, timeoutMs));
255
+ clearTimeout(this.#timer);
256
+ if (performance.now() >= this.#expiresAt) this.#expire();
257
+ else this.#timer = setTimeout(() => this.#expire(), this.#expiresAt - performance.now());
258
+ }
259
+ /** Stop pulling and release ownership without awaiting arbitrary cancellation. */
260
+ close() {
261
+ if (this.#closed) return;
262
+ this.#closed = true;
263
+ clearTimeout(this.#timer);
264
+ try {
265
+ if (!this.#ended) this.#reader.cancel().catch(() => {});
266
+ } finally {
267
+ this.#reader.releaseLock();
268
+ }
269
+ }
270
+ #expire() {
271
+ if (this.#timedOut) return;
272
+ this.#timedOut = true;
273
+ this.#expireDeadline();
274
+ this.close();
275
+ }
276
+ };
277
+ //#endregion
278
+ //#region src/federation/outbound-capture.ts
279
+ const diagnosticTimeoutMs = 1e3;
280
+ const diagnosticMaxBytes = 1024;
281
+ const diagnostics = /* @__PURE__ */ new WeakMap();
282
+ /** Reuse a selected capture's diagnostic preview instead of acquiring a second reader. */
283
+ function takeCapturedDiagnostic(response) {
284
+ const value = diagnostics.get(response);
285
+ diagnostics.delete(response);
286
+ return value;
287
+ }
288
+ /** Validate and copy before replacing the current observer registration. */
289
+ function snapshotBodyCapture(value) {
290
+ if (value === void 0) return void 0;
291
+ if (value == null || typeof value !== "object" || typeof value.onBody !== "function") throw new TypeError("Response body capture requires an onBody callback.");
292
+ const { maxBytes, timeoutMs, responses = "all", onBody } = value;
293
+ if (typeof maxBytes !== "number" || typeof timeoutMs !== "number") throw new TypeError("Response body capture requires numeric maxBytes and timeoutMs.");
294
+ if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) throw new RangeError("Response body capture maxBytes must be a positive safe integer.");
295
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || Math.ceil(timeoutMs) > 2147483647) throw new RangeError("Response body capture timeoutMs is outside the timer range.");
296
+ if (responses !== "all" && responses !== "non-success-http-responses") throw new TypeError("Invalid response body capture selector.");
297
+ return Object.freeze({
298
+ maxBytes,
299
+ timeoutMs: Math.ceil(timeoutMs),
300
+ responses,
301
+ onBody
302
+ });
303
+ }
304
+ /** One source reader feeds both the revocable application view and bounded diagnostics. */
305
+ async function captureResponseBody(response, options, onBody) {
306
+ if (options.responses === "non-success-http-responses" && response.ok) return Object.freeze({
307
+ type: "filtered",
308
+ bytesExposed: 0
309
+ });
310
+ if (response.body == null) return Object.freeze({
311
+ type: "no-body",
312
+ bytesExposed: 0
313
+ });
314
+ const start = performance.now();
315
+ const expires = start + options.timeoutMs;
316
+ let pump;
317
+ try {
318
+ pump = new ResponseBodyPump(response.body, Math.max(options.timeoutMs, response.ok ? 0 : diagnosticTimeoutMs));
319
+ } catch (error) {
320
+ return Object.freeze({
321
+ type: "failed",
322
+ phase: "body",
323
+ error,
324
+ bytesExposed: 0
325
+ });
326
+ }
327
+ const signal = new AbortController();
328
+ let controller;
329
+ let active = true;
330
+ let bytesExposed = 0;
331
+ let result;
332
+ let eof = false;
333
+ let diagnosticBytes = 0;
334
+ let diagnosticTruncated = false;
335
+ let diagnosticFailed = false;
336
+ const preview = [];
337
+ const decoder = new TextDecoder();
338
+ let pending;
339
+ const read = () => pending ??= pump.read().then((value) => {
340
+ pending = void 0;
341
+ if (value?.done) eof = true;
342
+ if (!response.ok && value != null && !value.done && performance.now() - start < diagnosticTimeoutMs) {
343
+ const remaining = diagnosticMaxBytes - diagnosticBytes;
344
+ const chunk = value.value;
345
+ if (chunk.length > remaining) diagnosticTruncated = true;
346
+ if (remaining > 0 && chunk.length > 0) {
347
+ const selected = chunk.subarray(0, remaining);
348
+ preview.push(decoder.decode(selected, { stream: true }));
349
+ diagnosticBytes += selected.length;
350
+ }
351
+ }
352
+ return value;
353
+ }, (error) => {
354
+ pending = void 0;
355
+ diagnosticFailed = true;
356
+ throw error;
357
+ });
358
+ const revoke = (reason) => {
359
+ active = false;
360
+ if (!signal.signal.aborted) signal.abort(reason);
361
+ try {
362
+ controller.error(reason);
363
+ } catch {}
364
+ };
365
+ const body = new ReadableStream({
366
+ start(value) {
367
+ controller = value;
368
+ },
369
+ async pull() {
370
+ try {
371
+ while (active) {
372
+ if (performance.now() >= expires) {
373
+ result = { type: "deadline" };
374
+ revoke(new DOMException("Response body capture deadline exceeded.", "TimeoutError"));
375
+ return;
376
+ }
377
+ const value = await read();
378
+ if (!active) return;
379
+ if (performance.now() >= expires || value == null) {
380
+ result = { type: "deadline" };
381
+ revoke(new DOMException("Response body capture deadline exceeded.", "TimeoutError"));
382
+ return;
383
+ }
384
+ if (value.done) {
385
+ result = { type: "complete" };
386
+ active = false;
387
+ controller.close();
388
+ return;
389
+ }
390
+ if (value.value.length === 0) continue;
391
+ const chunk = value.value.slice(0, options.maxBytes - bytesExposed);
392
+ bytesExposed += chunk.length;
393
+ controller.enqueue(chunk);
394
+ if (bytesExposed === options.maxBytes) {
395
+ result = { type: "limit-reached" };
396
+ revoke(new DOMException("Response body capture byte limit reached.", "QuotaExceededError"));
397
+ }
398
+ return;
399
+ }
400
+ } catch (error) {
401
+ if (active) {
402
+ result = {
403
+ type: "failed",
404
+ phase: "body",
405
+ error
406
+ };
407
+ revoke(error);
408
+ }
409
+ }
410
+ },
411
+ cancel(reason) {
412
+ result = {
413
+ type: "consumer-stopped",
414
+ reason
415
+ };
416
+ revoke(reason);
417
+ }
418
+ }, { highWaterMark: 0 });
419
+ let timer;
420
+ const deadline = new Promise((resolve) => {
421
+ timer = setTimeout(() => {
422
+ result = { type: "deadline" };
423
+ revoke(new DOMException("Response body capture deadline exceeded.", "TimeoutError"));
424
+ resolve();
425
+ }, Math.max(0, expires - performance.now()));
426
+ });
427
+ const callback = Promise.resolve().then(() => onBody(body, signal.signal)).then(() => {}, (error) => {
428
+ if (result == null || result.type === "complete") result = {
429
+ type: "failed",
430
+ phase: "observer",
431
+ error
432
+ };
433
+ });
434
+ try {
435
+ await Promise.race([callback, deadline]);
436
+ clearTimeout(timer);
437
+ if (performance.now() >= expires) result = { type: "deadline" };
438
+ result ??= { type: "callback-returned" };
439
+ revoke(new DOMException("Response body capture finished.", "AbortError"));
440
+ if (!response.ok) {
441
+ pump.limitLifetime(diagnosticTimeoutMs - (performance.now() - start));
442
+ try {
443
+ while (!eof && !diagnosticTruncated && performance.now() - start < diagnosticTimeoutMs) if (await read() == null) break;
444
+ } catch {}
445
+ diagnostics.set(response, preview.join("") + (diagnosticTruncated ? "… (truncated)" : !eof && !diagnosticFailed ? "… (timed out)" : ""));
446
+ }
447
+ return Object.freeze({
448
+ ...result,
449
+ bytesExposed
450
+ });
451
+ } finally {
452
+ clearTimeout(timer);
453
+ pump.close();
454
+ }
455
+ }
456
+ //#endregion
457
+ //#region src/federation/outbound-headers.ts
458
+ const protectedHeaders = /* @__PURE__ */ new Set([
459
+ "authorization",
460
+ "proxy-authorization",
461
+ "cookie",
462
+ "set-cookie",
463
+ "signature",
464
+ "signature-input",
465
+ "authentication-info",
466
+ "proxy-authentication-info",
467
+ "x-api-key",
468
+ "x-auth-token"
469
+ ]);
470
+ function snapshotNames(names) {
471
+ if (!Array.isArray(names)) throw new TypeError("Outbound header names must be an array.");
472
+ const normalized = /* @__PURE__ */ new Set();
473
+ for (const name of names) {
474
+ if (typeof name !== "string" || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name) || name.includes("*")) throw new TypeError("Outbound header names must be literal HTTP tokens.");
475
+ normalized.add(name.toLowerCase());
476
+ }
477
+ return Object.freeze([...normalized].sort());
478
+ }
479
+ /** Validate/copy policy before registration replaces the selected observers. */
480
+ function snapshotHeaderPolicy(policy) {
481
+ if (policy === void 0) return void 0;
482
+ if (policy == null || typeof policy !== "object") throw new TypeError("Outbound header policy must be an object.");
483
+ const { allow, redact, maxBytes } = policy;
484
+ if (typeof maxBytes !== "number") throw new TypeError("Outbound header maxBytes must be a number.");
485
+ if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) throw new RangeError("Outbound header maxBytes must be a positive safe integer.");
486
+ return Object.freeze({
487
+ allow: snapshotNames(allow),
488
+ ...redact === void 0 ? {} : { redact: snapshotNames(redact) },
489
+ maxBytes
490
+ });
491
+ }
492
+ /** Copy only selected/redacted entries; budgets never truncate a header value. */
493
+ function snapshotHeaders(headers, policy) {
494
+ const redact = /* @__PURE__ */ new Set([...protectedHeaders, ...policy.redact ?? []]);
495
+ const entries = [];
496
+ const encoder = new TextEncoder();
497
+ let bytes = 0;
498
+ let limitReached = false;
499
+ for (const name of policy.allow) {
500
+ if (!headers.has(name)) continue;
501
+ for (const [key, rawValue] of headers) {
502
+ if (key !== name) continue;
503
+ const value = redact.has(name) ? "[redacted]" : rawValue;
504
+ const size = encoder.encode(`${name}: ${value}\r\n`).byteLength;
505
+ if (size > policy.maxBytes - bytes) {
506
+ limitReached = true;
507
+ continue;
508
+ }
509
+ entries.push(Object.freeze([name, value]));
510
+ bytes += size;
511
+ }
512
+ }
513
+ return Object.freeze({
514
+ entries: Object.freeze(entries),
515
+ limitReached
516
+ });
517
+ }
518
+ //#endregion
25
519
  //#region src/federation/activity-listener.ts
26
520
  var ActivityListenerSet = class {
27
521
  #listeners;
@@ -107,6 +601,7 @@ var FederationBuilderImpl = class {
107
601
  inboxListeners;
108
602
  outboxListeners;
109
603
  inboxRequestFinishedHandler;
604
+ outboundObservers = globalThis.Object.freeze({});
110
605
  inboxErrorHandler;
111
606
  outboxListenerErrorHandler;
112
607
  outboxAuthorizePredicate;
@@ -169,6 +664,7 @@ var FederationBuilderImpl = class {
169
664
  f.inboxListeners = this.inboxListeners?.clone();
170
665
  f.outboxListeners = this.outboxListeners?.clone();
171
666
  f.inboxRequestFinishedHandler = this.inboxRequestFinishedHandler;
667
+ f.setOutboundObservers(this.outboundObservers);
172
668
  f.inboxErrorHandler = this.inboxErrorHandler;
173
669
  f.outboxListenerErrorHandler = this.outboxListenerErrorHandler;
174
670
  f.outboxAuthorizePredicate = this.outboxAuthorizePredicate;
@@ -316,6 +812,19 @@ var FederationBuilderImpl = class {
316
812
  };
317
813
  return setters;
318
814
  }
815
+ setOutboundObservers(observers) {
816
+ this.outboundObservers = globalThis.Object.freeze({
817
+ senderPayload: snapshotSenderPayloadOption(observers.senderPayload),
818
+ responseBodyCapture: snapshotBodyCapture(observers.responseBodyCapture),
819
+ requestHeaders: snapshotHeaderPolicy(observers.requestHeaders),
820
+ responseHeaders: snapshotHeaderPolicy(observers.responseHeaders),
821
+ onBroadcast: observers.onBroadcast,
822
+ onDelivery: observers.onDelivery,
823
+ onAttempt: observers.onAttempt,
824
+ onHttpExchange: observers.onHttpExchange,
825
+ onWorker: observers.onWorker
826
+ });
827
+ }
319
828
  setNodeInfoDispatcher(path, dispatcher) {
320
829
  if (this.router.has("nodeInfo")) throw new RouterError("NodeInfo dispatcher already set.");
321
830
  if (Router.variables(path).size !== 0) throw new RouterError("Path for NodeInfo dispatcher must have no variables.");
@@ -1462,6 +1971,7 @@ async function buildCollectionSynchronizationHeader(collectionId, actorIds) {
1462
1971
  /** Internal state owned by the ingress boundary, never serialized into queues. */
1463
1972
  var InboxObservation = class {
1464
1973
  inbox;
1974
+ source;
1465
1975
  payload = { status: "unavailable" };
1466
1976
  activity = null;
1467
1977
  authentication = { status: "notDetermined" };
@@ -1484,8 +1994,9 @@ var InboxObservation = class {
1484
1994
  }
1485
1995
  }
1486
1996
  };
1487
- constructor(inbox) {
1997
+ constructor(inbox, source = void 0) {
1488
1998
  this.inbox = inbox;
1999
+ this.source = source;
1489
2000
  }
1490
2001
  /** Run an ingress operation and project telemetry; fetch owns completion. */
1491
2002
  async run(operation, span) {
@@ -1541,7 +2052,18 @@ var InboxObservation = class {
1541
2052
  }
1542
2053
  async finish(context, handler, outcome) {
1543
2054
  if (handler == null) return;
2055
+ const signal = context.request.signal;
2056
+ const requestCancellation = signal.aborted ? Object.freeze({
2057
+ type: "aborted",
2058
+ reason: signal.reason
2059
+ }) : Object.freeze({ type: "not-aborted" });
1544
2060
  const report = {
2061
+ ...this.source == null ? {} : {
2062
+ inboxRequestId: this.source.inboxRequestId,
2063
+ requestId: this.source.requestId,
2064
+ local: Object.freeze({ ...this.source.context })
2065
+ },
2066
+ requestCancellation,
1545
2067
  inbox: this.inbox,
1546
2068
  payload: this.payload,
1547
2069
  activity: this.activity,
@@ -1550,7 +2072,7 @@ var InboxObservation = class {
1550
2072
  outcome
1551
2073
  };
1552
2074
  try {
1553
- await handler(context, report);
2075
+ await withContext(this.source == null ? {} : { inboxRequestId: this.source.inboxRequestId }, () => handler(context, report));
1554
2076
  } catch (error) {
1555
2077
  try {
1556
2078
  getLogger([
@@ -2812,6 +3334,92 @@ async function isInAudience(object, actor, options = {}) {
2812
3334
  return false;
2813
3335
  }
2814
3336
  //#endregion
3337
+ //#region src/federation/outbound-provenance.ts
3338
+ const outboundProvenance = Symbol("outboundProvenance");
3339
+ const emptyProvenance = Object.freeze({});
3340
+ function getOutboundProvenance(context) {
3341
+ return context[outboundProvenance] ?? emptyProvenance;
3342
+ }
3343
+ function snapshotLocalContext(context) {
3344
+ return Object.freeze({
3345
+ origin: context.origin,
3346
+ host: context.host,
3347
+ hostname: context.hostname,
3348
+ canonicalOrigin: context.canonicalOrigin
3349
+ });
3350
+ }
3351
+ function createInboxRequestSource(request, requestId, canonicalOrigin) {
3352
+ const url = new URL(request.url);
3353
+ return Object.freeze({
3354
+ inboxRequestId: crypto.randomUUID(),
3355
+ requestId,
3356
+ context: snapshotLocalContext({
3357
+ origin: url.origin,
3358
+ host: url.host,
3359
+ hostname: url.hostname,
3360
+ canonicalOrigin: canonicalOrigin ?? url.origin
3361
+ })
3362
+ });
3363
+ }
3364
+ /** The same exact-string budget applies to public input and queued metadata. */
3365
+ function validateDispatchId(value) {
3366
+ if (value === void 0) return;
3367
+ if (typeof value !== "string") throw new TypeError("dispatchId must be a string.");
3368
+ if (value.length > 1024 || new TextEncoder().encode(value).byteLength > 1024) throw new RangeError("dispatchId must not exceed 1,024 UTF-8 bytes.");
3369
+ }
3370
+ /** A new public invocation keeps its HTTP source, with its own dispatch ID. */
3371
+ function invocationProvenance(context, dispatchId) {
3372
+ validateDispatchId(dispatchId);
3373
+ const { dispatchId: _previous, ...source } = getOutboundProvenance(context);
3374
+ return Object.freeze({
3375
+ ...source,
3376
+ ...dispatchId === void 0 ? {} : { dispatchId }
3377
+ });
3378
+ }
3379
+ /** Restore only explicit source facts; never infer them from queue/trace IDs. */
3380
+ function restoreProvenance(value) {
3381
+ if (value === void 0) return emptyProvenance;
3382
+ const record = (v) => v != null && typeof v === "object" && !Array.isArray(v);
3383
+ const invalid = () => /* @__PURE__ */ new TypeError("Invalid provenance in queue metadata.");
3384
+ if (!record(value)) throw invalid();
3385
+ validateDispatchId(value.dispatchId);
3386
+ if (value.originatingRequestId !== void 0 && typeof value.originatingRequestId !== "string") throw invalid();
3387
+ let inboxRequest;
3388
+ if (value.inboxRequest !== void 0) {
3389
+ const source = value.inboxRequest;
3390
+ if (!record(source) || typeof source.inboxRequestId !== "string" || source.inboxRequestId.length < 1 || typeof source.requestId !== "string" || !record(source.context)) throw invalid();
3391
+ const facts = source.context;
3392
+ if (typeof facts.origin !== "string" || typeof facts.host !== "string" || typeof facts.hostname !== "string" || typeof facts.canonicalOrigin !== "string") throw invalid();
3393
+ inboxRequest = Object.freeze({
3394
+ inboxRequestId: source.inboxRequestId,
3395
+ requestId: source.requestId,
3396
+ context: Object.freeze({
3397
+ origin: facts.origin,
3398
+ host: facts.host,
3399
+ hostname: facts.hostname,
3400
+ canonicalOrigin: facts.canonicalOrigin
3401
+ })
3402
+ });
3403
+ }
3404
+ return Object.freeze({
3405
+ ...value.dispatchId === void 0 ? {} : { dispatchId: value.dispatchId },
3406
+ ...inboxRequest == null ? {} : { inboxRequest },
3407
+ ...value.originatingRequestId === void 0 ? {} : { originatingRequestId: value.originatingRequestId }
3408
+ });
3409
+ }
3410
+ /** Omit absent provenance on legacy and independent producers. */
3411
+ function serializeProvenance(value) {
3412
+ const provenance = restoreProvenance(value);
3413
+ return provenance.dispatchId === void 0 && provenance.inboxRequest == null && provenance.originatingRequestId == null ? {} : { provenance };
3414
+ }
3415
+ function sourceLogContext(source) {
3416
+ return {
3417
+ dispatchId: source.dispatchId,
3418
+ sourceInboxRequestId: source.inboxRequest?.inboxRequestId,
3419
+ sourceRequestId: source.inboxRequest?.requestId ?? source.originatingRequestId
3420
+ };
3421
+ }
3422
+ //#endregion
2815
3423
  //#region src/federation/inbox-authentication.ts
2816
3424
  /**
2817
3425
  * The evidence of an activity that has not been authenticated by Fedify.
@@ -3185,6 +3793,7 @@ async function routeActivity({ context: ctx, json, originalJson, normalizedActiv
3185
3793
  propagation.inject(context.active(), carrier);
3186
3794
  message = {
3187
3795
  type: "inbox",
3796
+ ...serializeProvenance(getOutboundProvenance(ctx)),
3188
3797
  id: crypto.randomUUID(),
3189
3798
  baseUrl: ctx.origin,
3190
3799
  activity: originalJson ?? json,
@@ -4349,11 +4958,15 @@ function extractInboxes({ recipients, preferSharedInbox, excludeBaseUris }) {
4349
4958
  * @throws {Error} If the activity fails to send.
4350
4959
  */
4351
4960
  function sendActivity(options, deps = {}) {
4961
+ options.attemptStarted?.(options.inbox.href);
4962
+ const attempt = options.observation?.startAttempt(options.inbox.href);
4963
+ const parentSpanContext = trace.getSpanContext(context.active());
4352
4964
  const tracerProvider = options.tracerProvider ?? trace.getTracerProvider();
4353
4965
  return tracerProvider.getTracer(name, version).startActiveSpan("activitypub.send_activity", {
4354
4966
  kind: SpanKind.CLIENT,
4355
4967
  attributes: { "activitypub.shared_inbox": options.sharedInbox ?? false }
4356
4968
  }, async (span) => {
4969
+ attempt?.setSpan(span, parentSpanContext);
4357
4970
  if (options.activityId != null) span.setAttribute("activitypub.activity.id", options.activityId);
4358
4971
  if (options.activityType != null) span.setAttribute("activitypub.activity.type", options.activityType);
4359
4972
  const spanContext = span.spanContext();
@@ -4364,12 +4977,14 @@ function sendActivity(options, deps = {}) {
4364
4977
  }, () => sendActivityInternal({
4365
4978
  ...options,
4366
4979
  tracerProvider
4367
- }, span, deps));
4980
+ }, span, deps, attempt));
4981
+ await attempt?.succeeded();
4368
4982
  } catch (e) {
4369
4983
  span.setStatus({
4370
4984
  code: SpanStatusCode.ERROR,
4371
4985
  message: String(e)
4372
4986
  });
4987
+ await attempt?.failed(e);
4373
4988
  throw e;
4374
4989
  } finally {
4375
4990
  span.end();
@@ -4409,6 +5024,7 @@ async function sendActivityThroughGateways(parameters, inboxes) {
4409
5024
  throw lastError;
4410
5025
  }
4411
5026
  const MAX_ERROR_RESPONSE_BODY_BYTES = 1024;
5027
+ const ERROR_RESPONSE_BODY_TIMEOUT_MS = 1e3;
4412
5028
  function getActivityActorId(activity) {
4413
5029
  if (!isRecord(activity)) return void 0;
4414
5030
  return getIdValue(activity.actor);
@@ -4429,16 +5045,25 @@ function isRecord(value) {
4429
5045
  return typeof value === "object" && value != null;
4430
5046
  }
4431
5047
  async function readLimitedResponseBody(response, maxBytes) {
5048
+ const captured = takeCapturedDiagnostic(response);
5049
+ if (captured !== void 0) return captured;
4432
5050
  if (response.body == null) return "";
4433
- const reader = response.body.getReader();
5051
+ const pump = new ResponseBodyPump(response.body, ERROR_RESPONSE_BODY_TIMEOUT_MS);
4434
5052
  const decoder = new TextDecoder();
4435
5053
  const chunks = [];
4436
5054
  let totalBytes = 0;
4437
5055
  let truncated = false;
5056
+ let timedOut = false;
4438
5057
  try {
4439
5058
  while (true) {
4440
- const { done, value } = await reader.read();
5059
+ const result = await pump.read();
5060
+ if (result == null) {
5061
+ timedOut = true;
5062
+ break;
5063
+ }
5064
+ const { done, value } = result;
4441
5065
  if (done) break;
5066
+ if (value.length === 0) continue;
4442
5067
  if (totalBytes + value.length > maxBytes) {
4443
5068
  const remaining = maxBytes - totalBytes;
4444
5069
  if (remaining > 0) chunks.push(decoder.decode(value.slice(0, remaining), { stream: true }));
@@ -4449,14 +5074,14 @@ async function readLimitedResponseBody(response, maxBytes) {
4449
5074
  totalBytes += value.length;
4450
5075
  }
4451
5076
  } finally {
4452
- if (truncated) reader.cancel().catch(() => {});
4453
- reader.releaseLock();
5077
+ pump.close();
4454
5078
  }
4455
5079
  let result = chunks.join("");
4456
5080
  if (truncated) result += "… (truncated)";
5081
+ else if (timedOut) result += "… (timed out)";
4457
5082
  return result;
4458
5083
  }
4459
- async function sendActivityInternal({ activity, activityId, activityType, keys, inbox, allowPrivateAddress, headers, specDeterminer, meterProvider, tracerProvider }, span, deps) {
5084
+ async function sendActivityInternal({ activity, activityId, activityType, keys, inbox, allowPrivateAddress, headers, specDeterminer, meterProvider, tracerProvider }, span, deps, attempt) {
4460
5085
  const logger = getLogger([
4461
5086
  "fedify",
4462
5087
  "federation",
@@ -4472,6 +5097,11 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
4472
5097
  session = await openUnvalidatedHop(url);
4473
5098
  } catch (error) {
4474
5099
  if (error instanceof UrlError && error.reason === "dns") {
5100
+ attempt?.failure({
5101
+ kind: "transport",
5102
+ phase: "validation",
5103
+ error
5104
+ });
4475
5105
  logger.error("DNS resolution failed for URL: {url}", {
4476
5106
  url,
4477
5107
  error
@@ -4480,13 +5110,47 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
4480
5110
  failure.cause = error;
4481
5111
  throw failure;
4482
5112
  }
4483
- if (error instanceof UrlError) logger.error("Disallowed private URL: {url}", {
4484
- url,
4485
- error
4486
- });
5113
+ if (error instanceof UrlError) {
5114
+ if (error.reason === "disallowed") attempt?.failure({
5115
+ kind: "policy",
5116
+ phase: "validation",
5117
+ error
5118
+ });
5119
+ logger.error("Disallowed private URL: {url}", {
5120
+ url,
5121
+ error
5122
+ });
5123
+ }
4487
5124
  throw error;
4488
5125
  }
4489
- return session;
5126
+ if (attempt == null) return session;
5127
+ return {
5128
+ url: session.url,
5129
+ async fetch(input, init) {
5130
+ try {
5131
+ return await session.fetch(input, init);
5132
+ } catch (error) {
5133
+ attempt.failure({
5134
+ kind: "transport",
5135
+ phase: "request",
5136
+ error
5137
+ });
5138
+ throw error;
5139
+ }
5140
+ },
5141
+ async close() {
5142
+ try {
5143
+ await session.close();
5144
+ } catch (error) {
5145
+ attempt.failure({
5146
+ kind: "local",
5147
+ phase: "cleanup",
5148
+ error
5149
+ });
5150
+ throw error;
5151
+ }
5152
+ }
5153
+ };
4490
5154
  }
4491
5155
  let initialSession;
4492
5156
  try {
@@ -4504,14 +5168,45 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
4504
5168
  return openHop(url);
4505
5169
  };
4506
5170
  try {
4507
- headers = new Headers(headers);
4508
- headers.set("Content-Type", "application/activity+json");
4509
- const request = new Request(inbox, {
4510
- method: "POST",
4511
- headers,
4512
- body: JSON.stringify(activity),
4513
- redirect: "manual"
4514
- });
5171
+ let request;
5172
+ let body;
5173
+ try {
5174
+ headers = new Headers(headers);
5175
+ headers.set("Content-Type", "application/activity+json");
5176
+ } catch (error) {
5177
+ attempt?.failure({
5178
+ kind: "local",
5179
+ phase: "request-preparation",
5180
+ error
5181
+ });
5182
+ throw error;
5183
+ }
5184
+ try {
5185
+ attempt?.snapshotSenderPayload(activity);
5186
+ body = JSON.stringify(activity);
5187
+ } catch (error) {
5188
+ attempt?.failure({
5189
+ kind: "local",
5190
+ phase: "serialization",
5191
+ error
5192
+ });
5193
+ throw error;
5194
+ }
5195
+ try {
5196
+ request = new Request(inbox, {
5197
+ method: "POST",
5198
+ headers,
5199
+ body,
5200
+ redirect: "manual"
5201
+ });
5202
+ } catch (error) {
5203
+ attempt?.failure({
5204
+ kind: "local",
5205
+ phase: "request-preparation",
5206
+ error
5207
+ });
5208
+ throw error;
5209
+ }
4515
5210
  let rsaKey = null;
4516
5211
  for (const key of keys) if (key.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") {
4517
5212
  rsaKey = key;
@@ -4528,13 +5223,25 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
4528
5223
  privateKey: pair.privateKey
4529
5224
  }))
4530
5225
  });
5226
+ const consume = (response) => {
5227
+ return consumeSendActivityResponse(response, inbox);
5228
+ };
4531
5229
  let outcome;
4532
5230
  try {
4533
- outcome = rsaKey == null ? await fetchWithValidatedRedirects(request, openDeliveryHop, (response) => consumeSendActivityResponse(response, inbox)) : await doubleKnockScoped(request, rsaKey, {
5231
+ outcome = rsaKey == null ? await fetchWithValidatedRedirects(request, openDeliveryHop, consume, attempt) : await doubleKnockScoped(request, rsaKey, {
4534
5232
  tracerProvider,
4535
5233
  specDeterminer
4536
- }, openDeliveryHop, (response) => consumeSendActivityResponse(response, inbox));
5234
+ }, openDeliveryHop, consume, attempt == null ? void 0 : {
5235
+ fetch: (request, submit) => attempt.fetch(request, submit),
5236
+ response: (response, disposition) => attempt.response(response, disposition),
5237
+ error: (phase, error) => attempt.failure({
5238
+ kind: "local",
5239
+ phase,
5240
+ error
5241
+ })
5242
+ });
4537
5243
  } catch (error) {
5244
+ attempt?.unknown(error);
4538
5245
  const failure = error instanceof FetchError || error instanceof UrlError ? error : createFetchError(inbox.href, error);
4539
5246
  logger.error("Failed to send activity {activityId} to {inbox}:\n{error}", {
4540
5247
  activityId,
@@ -4554,7 +5261,13 @@ async function sendActivityInternal({ activity, activityId, activityType, keys,
4554
5261
  statusText,
4555
5262
  error
4556
5263
  });
4557
- throw new SendActivityError(inbox, status, `Failed to send activity ${activityId} to ${inbox.href} (${status} ${statusText}):\n${error}`, error, responseHeaders);
5264
+ const failure = new SendActivityError(inbox, status, `Failed to send activity ${activityId} to ${inbox.href} (${status} ${statusText}):\n${error}`, error, responseHeaders);
5265
+ attempt?.failure({
5266
+ kind: "http",
5267
+ phase: "response",
5268
+ error: failure
5269
+ });
5270
+ throw failure;
4558
5271
  }
4559
5272
  deliverySuccess = true;
4560
5273
  const eventAttributes = {
@@ -4598,7 +5311,7 @@ function cancelResponseBody(response, url) {
4598
5311
  /**
4599
5312
  * Reads what is needed from the final response of a delivery while its
4600
5313
  * session is open: the body of a successful response is cancelled, and at
4601
- * most 1 KiB of an error response body is read.
5314
+ * most 1 KiB of an error response body is read within its diagnostic budget.
4602
5315
  */
4603
5316
  async function consumeSendActivityResponse(response, inbox) {
4604
5317
  if (response.ok) {
@@ -4628,9 +5341,10 @@ async function consumeSendActivityResponse(response, inbox) {
4628
5341
  * @param request The request to send.
4629
5342
  * @param openHop The function to open a session for each hop.
4630
5343
  * @param consume The function to consume the final response.
5344
+ * @param attempt The in-process attempt observation, if registered.
4631
5345
  * @returns The value returned by `consume`.
4632
5346
  */
4633
- async function fetchWithValidatedRedirects(request, openHop, consume) {
5347
+ async function fetchWithValidatedRedirects(request, openHop, consume, attempt) {
4634
5348
  const logger = getLogger([
4635
5349
  "fedify",
4636
5350
  "federation",
@@ -4641,7 +5355,7 @@ async function fetchWithValidatedRedirects(request, openHop, consume) {
4641
5355
  const session = await openHop(request.url);
4642
5356
  let response;
4643
5357
  try {
4644
- response = await session.fetch(request, { redirect: "manual" });
5358
+ response = await (attempt == null ? session.fetch(request, { redirect: "manual" }) : attempt.fetch(request, () => session.fetch(request, { redirect: "manual" })));
4645
5359
  } catch (error) {
4646
5360
  await closeAfterError(session, logger);
4647
5361
  throw error;
@@ -4653,17 +5367,21 @@ async function fetchWithValidatedRedirects(request, openHop, consume) {
4653
5367
  303,
4654
5368
  307,
4655
5369
  308
4656
- ].includes(response.status) || location == null) try {
4657
- return await consume(response);
4658
- } catch (error) {
4659
- logger.debug("Failed to consume the response from {url}; closing the session: {error}", {
4660
- url: request.url,
4661
- error
4662
- });
4663
- throw error;
4664
- } finally {
4665
- await session.close();
5370
+ ].includes(response.status) || location == null) {
5371
+ attempt?.response(response, "final");
5372
+ try {
5373
+ return await consume(response);
5374
+ } catch (error) {
5375
+ logger.debug("Failed to consume the response from {url}; closing the session: {error}", {
5376
+ url: request.url,
5377
+ error
5378
+ });
5379
+ throw error;
5380
+ } finally {
5381
+ await session.close();
5382
+ }
4666
5383
  }
5384
+ attempt?.response(response, "intermediate");
4667
5385
  cancelResponseBody(response, request.url);
4668
5386
  await session.close();
4669
5387
  if (redirects >= 20) throw new TypeError("Too many redirects");
@@ -4724,7 +5442,9 @@ var SendActivityError = class extends Error {
4724
5442
  * The response body from the inbox, if any. Note that this may be
4725
5443
  * truncated to a maximum of 1 KiB to prevent excessive memory consumption
4726
5444
  * when remote servers return large error pages (e.g., Cloudflare error pages).
4727
- * If truncated, the string will end with `"… (truncated)"`.
5445
+ * If truncated, the string will end with `"… (truncated)"`. Diagnostic reading
5446
+ * has a separate one-second budget; a deadline retains any partial preview and
5447
+ * appends `"… (timed out)"`. This is independent of application body capture.
4728
5448
  */
4729
5449
  responseBody;
4730
5450
  /**
@@ -4878,9 +5598,10 @@ function coversDelivery(signature) {
4878
5598
  * other gateways, that gateway has the activity already, so it is claimed
4879
5599
  * without forwarding the activity to it. See {@link identifySendingGateway}.
4880
5600
  * @param parameters The parameters.
5601
+ * @param observation The in-process producer observation, if registered.
4881
5602
  * @returns The target inbox URLs that the activity was handed off to.
4882
5603
  */
4883
- async function forwardPortableInboxActivity(parameters) {
5604
+ async function forwardPortableInboxActivity(parameters, observation) {
4884
5605
  const logger = getLogger([
4885
5606
  "fedify",
4886
5607
  "federation",
@@ -4889,8 +5610,16 @@ async function forwardPortableInboxActivity(parameters) {
4889
5610
  const { recipient, activityId, excludedOrigins, kv, kvPrefix, outboxQueue } = parameters;
4890
5611
  const options = parameters.options ?? resolvePortableInboxForwardingOptions();
4891
5612
  const { maxTargets, ttl } = options;
4892
- if (kv.cas == null || maxTargets < 1) return [];
5613
+ if (kv.cas == null) {
5614
+ if (observation != null) await observation.skipped("forwarding-unavailable");
5615
+ return [];
5616
+ }
5617
+ if (maxTargets < 1) {
5618
+ if (observation != null) await observation.skipped("forwarding-disabled");
5619
+ return [];
5620
+ }
4893
5621
  const deadline = Date.now() + options.deadline.total("millisecond");
5622
+ observation?.setPhase("destination-planning");
4894
5623
  const canonicalActivityId = getCanonicalPortableId(activityId) ?? activityId.href;
4895
5624
  const claimKey = (gateway) => [
4896
5625
  ...kvPrefix,
@@ -4918,15 +5647,27 @@ async function forwardPortableInboxActivity(parameters) {
4918
5647
  targets.length = maxTargets;
4919
5648
  }
4920
5649
  const claimed = [];
4921
- for (const { gateway, inbox } of targets) if (await kv.cas(claimKey(gateway), void 0, true, { ttl })) claimed.push(inbox);
4922
- if (claimed.length < 1) return [];
5650
+ for (const { gateway, inbox } of targets) if (await kv.cas(claimKey(gateway), void 0, true, { ttl })) {
5651
+ claimed.push(inbox);
5652
+ if (observation != null) await observation.planDestination(inbox.href, {
5653
+ actorIds: [recipient.actorId.href],
5654
+ sharedInbox: false,
5655
+ portableInbox: recipient.canonicalInboxId,
5656
+ mode: "portable-replication"
5657
+ });
5658
+ }
5659
+ observation?.planned(claimed.length);
5660
+ if (claimed.length < 1) {
5661
+ if (observation != null) await observation.skipped("no-forwarding-targets");
5662
+ return [];
5663
+ }
4923
5664
  logger.debug("Forwarding activity {activityId} from the portable inbox {inbox} to other gateways:\n{targets}", {
4924
5665
  activityId: activityId.href,
4925
5666
  inbox: recipient.canonicalInboxId,
4926
5667
  targets: claimed.map((t) => t.href)
4927
5668
  });
4928
- if (outboxQueue == null) await forwardImmediately(parameters, claimed, deadline);
4929
- else await enqueueForwarding(parameters, outboxQueue, claimed);
5669
+ if (outboxQueue == null) await forwardImmediately(parameters, claimed, deadline, observation);
5670
+ else await enqueueForwarding(parameters, outboxQueue, claimed, observation);
4930
5671
  return claimed;
4931
5672
  }
4932
5673
  /**
@@ -5069,7 +5810,7 @@ async function resolveForwardingKey({ getKeys, activityId }) {
5069
5810
  }
5070
5811
  return key;
5071
5812
  }
5072
- async function forwardImmediately(parameters, inboxes, deadlineAt) {
5813
+ async function forwardImmediately(parameters, inboxes, deadlineAt, observation) {
5073
5814
  const { activity, activityId, activityType, allowPrivateAddress, specDeterminer, meterProvider, tracerProvider } = parameters;
5074
5815
  const logger = getLogger([
5075
5816
  "fedify",
@@ -5077,23 +5818,50 @@ async function forwardImmediately(parameters, inboxes, deadlineAt) {
5077
5818
  "inbox"
5078
5819
  ]);
5079
5820
  const deadline = Math.max(0, deadlineAt - Date.now());
5080
- const forwarding = resolveForwardingKey(parameters).then((key) => Promise.all(inboxes.map((inbox) => sendActivity({
5081
- activity,
5082
- activityId: activityId.href,
5083
- activityType,
5084
- keys: key == null ? [] : [key],
5085
- inbox,
5086
- allowPrivateAddress,
5087
- specDeterminer,
5088
- meterProvider,
5089
- tracerProvider
5090
- }).catch((error) => {
5091
- logger.error("Failed to forward activity {activityId} to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
5092
- activityId: activityId.href,
5093
- inbox: inbox.href,
5094
- error
5095
- });
5096
- }))));
5821
+ observation?.setPhase("key-resolution");
5822
+ const forwarding = (async () => {
5823
+ try {
5824
+ const key = await resolveForwardingKey(parameters);
5825
+ observation?.setPhase("request-preparation");
5826
+ const sending = [];
5827
+ for (const inbox of inboxes) {
5828
+ const delivery = observation?.delivery(inbox.href);
5829
+ let pending;
5830
+ try {
5831
+ pending = sendActivity({
5832
+ observation: delivery,
5833
+ activity,
5834
+ activityId: activityId.href,
5835
+ activityType,
5836
+ keys: key == null ? [] : [key],
5837
+ inbox,
5838
+ allowPrivateAddress,
5839
+ specDeterminer,
5840
+ meterProvider,
5841
+ tracerProvider
5842
+ });
5843
+ } catch (error) {
5844
+ if (delivery != null) await delivery.failed(error);
5845
+ throw error;
5846
+ }
5847
+ const settled = pending.catch((error) => {
5848
+ logger.error("Failed to forward activity {activityId} to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
5849
+ activityId: activityId.href,
5850
+ inbox: inbox.href,
5851
+ error
5852
+ });
5853
+ });
5854
+ if (observation != null) settled.catch(() => {});
5855
+ sending.push(settled);
5856
+ if (delivery != null) await delivery.handoff({ type: "direct" });
5857
+ }
5858
+ if (observation != null) await observation.prepared("delivery");
5859
+ await Promise.all(sending);
5860
+ } catch (error) {
5861
+ if (observation != null) await observation.failed(error);
5862
+ throw error;
5863
+ }
5864
+ })();
5097
5865
  let timer;
5098
5866
  const timedOut = await Promise.race([forwarding.then(() => false), new Promise((resolve) => {
5099
5867
  timer = setTimeout(() => resolve(true), deadline);
@@ -5104,13 +5872,14 @@ async function forwardImmediately(parameters, inboxes, deadlineAt) {
5104
5872
  deadline
5105
5873
  });
5106
5874
  }
5107
- async function enqueueForwarding(parameters, outboxQueue, inboxes) {
5875
+ async function enqueueForwarding(parameters, outboxQueue, inboxes, observation) {
5108
5876
  const { activity, activityId, activityType, baseUrl, startQueue, meterProvider } = parameters;
5109
5877
  const logger = getLogger([
5110
5878
  "fedify",
5111
5879
  "federation",
5112
5880
  "inbox"
5113
5881
  ]);
5882
+ observation?.setPhase("key-resolution");
5114
5883
  const key = await resolveForwardingKey(parameters);
5115
5884
  const keys = [];
5116
5885
  if (key != null) try {
@@ -5125,30 +5894,45 @@ async function enqueueForwarding(parameters, outboxQueue, inboxes) {
5125
5894
  error
5126
5895
  });
5127
5896
  }
5897
+ observation?.setPhase("request-preparation");
5128
5898
  startQueue?.();
5129
5899
  const started = (/* @__PURE__ */ new Date()).toISOString();
5130
5900
  const traceContext = {};
5131
5901
  propagation.inject(context.active(), traceContext);
5902
+ let preparedCount = 0;
5132
5903
  await Promise.all(inboxes.map(async (inbox) => {
5133
- const message = {
5134
- type: "outbox",
5135
- id: crypto.randomUUID(),
5136
- baseUrl,
5137
- keys,
5138
- activity,
5139
- activityId: activityId.href,
5140
- activityType,
5141
- inbox: inbox.href,
5142
- sharedInbox: false,
5143
- started,
5144
- attempt: 0,
5145
- headers: {},
5146
- traceContext
5147
- };
5904
+ const delivery = observation?.delivery(inbox.href);
5905
+ let message;
5148
5906
  try {
5149
- await outboxQueue.enqueue(message);
5150
- } catch (error) {
5151
- logger.error("Failed to enqueue activity {activityId} to forward to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
5907
+ message = {
5908
+ type: "outbox",
5909
+ ...serializeProvenance(parameters.provenance),
5910
+ id: crypto.randomUUID(),
5911
+ baseUrl,
5912
+ keys,
5913
+ activity,
5914
+ activityId: activityId.href,
5915
+ activityType,
5916
+ inbox: inbox.href,
5917
+ ...observation?.continuation(inbox.href),
5918
+ sharedInbox: false,
5919
+ started,
5920
+ attempt: 0,
5921
+ headers: {},
5922
+ traceContext
5923
+ };
5924
+ } catch (error) {
5925
+ if (delivery != null) await delivery.failed(error);
5926
+ throw error;
5927
+ }
5928
+ if (observation != null && ++preparedCount === inboxes.length) await observation.prepared("delivery");
5929
+ try {
5930
+ await (observation == null ? outboxQueue.enqueue(message) : observation.enqueue([{
5931
+ inbox: inbox.href,
5932
+ queueMessageId: message.id
5933
+ }], "outbox", () => outboxQueue.enqueue(message)));
5934
+ } catch (error) {
5935
+ logger.error("Failed to enqueue activity {activityId} to forward to {inbox}; it will not be forwarded to the inbox again:\n{error}", {
5152
5936
  activityId: activityId.href,
5153
5937
  inbox: inbox.href,
5154
5938
  error
@@ -7863,6 +8647,786 @@ async function verifyPortableActivityId(activityId, jsonLd, options) {
7863
8647
  return null;
7864
8648
  }
7865
8649
  //#endregion
8650
+ //#region src/federation/outbound-observation.ts
8651
+ function snapshotDeliveryIdentity(identity) {
8652
+ if (identity == null || typeof identity.deliveryId !== "string" || identity.deliveryId.length < 1 || typeof identity.destinationInbox !== "string" || identity.destinationInbox.length < 1 || identity.broadcast == null || !(identity.broadcast.type === "unknown" || identity.broadcast.type === "known" && typeof identity.broadcast.id === "string" && identity.broadcast.id.length > 0) || !(identity.portableInbox == null || typeof identity.portableInbox === "string")) throw new TypeError("Invalid outbound delivery identity in queue metadata.");
8653
+ return Object.freeze({
8654
+ broadcast: Object.freeze(identity.broadcast.type === "known" ? {
8655
+ type: "known",
8656
+ id: identity.broadcast.id
8657
+ } : { type: "unknown" }),
8658
+ deliveryId: identity.deliveryId,
8659
+ destinationInbox: identity.destinationInbox,
8660
+ ...identity.portableInbox == null ? {} : { portableInbox: identity.portableInbox }
8661
+ });
8662
+ }
8663
+ async function legacyFanoutIdentity(taskId, inbox, portableInbox) {
8664
+ const bytes = new TextEncoder().encode(JSON.stringify([
8665
+ "fedify-legacy-fanout-delivery",
8666
+ taskId,
8667
+ inbox
8668
+ ]));
8669
+ const digest = await crypto.subtle.digest("SHA-256", bytes);
8670
+ return {
8671
+ broadcast: { type: "unknown" },
8672
+ deliveryId: `legacy-fanout:${encodeHex(new Uint8Array(digest))}`,
8673
+ destinationInbox: inbox,
8674
+ ...portableInbox == null ? {} : { portableInbox }
8675
+ };
8676
+ }
8677
+ async function notifyObserver(context, callback, event, unit) {
8678
+ if (callback == null) return;
8679
+ try {
8680
+ await withContext(sourceLogContext(event.provenance), () => callback(context, event));
8681
+ } catch (error) {
8682
+ logObserverFailure(unit, error);
8683
+ }
8684
+ }
8685
+ function logObserverFailure(unit, error) {
8686
+ try {
8687
+ getLogger([
8688
+ "fedify",
8689
+ "federation",
8690
+ "outbox"
8691
+ ]).error("An unexpected error occurred in outbound {unit} observer:\n{error}", {
8692
+ unit,
8693
+ error
8694
+ });
8695
+ } catch {}
8696
+ }
8697
+ function warnInvalidDeliveryIdentity(messageId, error) {
8698
+ try {
8699
+ getLogger([
8700
+ "fedify",
8701
+ "federation",
8702
+ "outbox"
8703
+ ]).warn("Ignoring invalid outbound delivery identity in queue message {messageId}: {error}", {
8704
+ messageId,
8705
+ error
8706
+ });
8707
+ } catch {}
8708
+ }
8709
+ function snapshotTrace(span, parentSpanContext) {
8710
+ const spanContext = span.spanContext();
8711
+ if (!isSpanContextValid(spanContext) || spanContext.isRemote || spanContext.traceId === parentSpanContext?.traceId && spanContext.spanId === parentSpanContext.spanId) return void 0;
8712
+ return Object.freeze({
8713
+ traceId: spanContext.traceId,
8714
+ spanId: spanContext.spanId
8715
+ });
8716
+ }
8717
+ /** State owned by one concrete send, never persisted in queue metadata. */
8718
+ var OutboundAttemptObservation = class {
8719
+ inboxUrl;
8720
+ report;
8721
+ reportExchange;
8722
+ headerPolicies;
8723
+ bodyCapture;
8724
+ senderPayload;
8725
+ #attemptId = crypto.randomUUID();
8726
+ #failures = [];
8727
+ #response = Object.freeze({ type: "not-observed" });
8728
+ #finalResponse;
8729
+ #responses = /* @__PURE__ */ new WeakMap();
8730
+ #trace;
8731
+ #finished = false;
8732
+ #exchangeCount = 0;
8733
+ #senderPayload;
8734
+ constructor(inboxUrl, report, reportExchange = void 0, headerPolicies = {}, bodyCapture = void 0, senderPayload = false) {
8735
+ this.inboxUrl = inboxUrl;
8736
+ this.report = report;
8737
+ this.reportExchange = reportExchange;
8738
+ this.headerPolicies = headerPolicies;
8739
+ this.bodyCapture = bodyCapture;
8740
+ this.senderPayload = senderPayload;
8741
+ this.#senderPayload = Object.freeze({
8742
+ stage: "sender-payload",
8743
+ type: "unavailable",
8744
+ reason: senderPayload ? "not-materialized" : "disabled"
8745
+ });
8746
+ }
8747
+ snapshotSenderPayload(input) {
8748
+ if (this.senderPayload) this.#senderPayload = projectSenderPayload(input);
8749
+ }
8750
+ setSpan(span, parentSpanContext) {
8751
+ this.#trace = snapshotTrace(span, parentSpanContext);
8752
+ }
8753
+ async fetch(request, submit) {
8754
+ const exchangeIndex = this.#exchangeCount++;
8755
+ const snapshot = Object.freeze({
8756
+ url: request.url,
8757
+ method: request.method,
8758
+ ...this.headerPolicies.requestHeaders == null ? {} : { headers: snapshotHeaders(request.headers, this.headerPolicies.requestHeaders) }
8759
+ });
8760
+ let response;
8761
+ try {
8762
+ response = await submit();
8763
+ } catch (error) {
8764
+ await this.#exchange(exchangeIndex, snapshot, {
8765
+ type: "failed",
8766
+ error
8767
+ });
8768
+ throw error;
8769
+ }
8770
+ const metadata = Object.freeze({
8771
+ status: response.status,
8772
+ statusText: response.statusText,
8773
+ ...response.url === "" ? {} : { url: response.url }
8774
+ });
8775
+ const observed = Object.freeze({
8776
+ exchangeIndex,
8777
+ metadata
8778
+ });
8779
+ this.#responses.set(response, observed);
8780
+ this.#response = Object.freeze({
8781
+ type: "undetermined",
8782
+ lastObserved: observed
8783
+ });
8784
+ const responseSnapshot = Object.freeze({
8785
+ ...metadata,
8786
+ ...this.headerPolicies.responseHeaders == null ? {} : { headers: snapshotHeaders(response.headers, this.headerPolicies.responseHeaders) }
8787
+ });
8788
+ const bodyCapture = this.bodyCapture == null ? void 0 : await captureResponseBody(response, this.bodyCapture, (body, signal) => this.bodyCapture.onBody({
8789
+ attemptId: this.#attemptId,
8790
+ inboxUrl: this.inboxUrl,
8791
+ exchangeIndex,
8792
+ observedAt: Temporal.Now.instant().epochMilliseconds,
8793
+ request: snapshot,
8794
+ response: responseSnapshot,
8795
+ body,
8796
+ signal
8797
+ }));
8798
+ if (bodyCapture?.type === "failed" && bodyCapture.phase === "observer") logObserverFailure("response-body", bodyCapture.error);
8799
+ await this.#exchange(exchangeIndex, snapshot, {
8800
+ type: "response",
8801
+ response: responseSnapshot
8802
+ }, bodyCapture);
8803
+ return response;
8804
+ }
8805
+ #exchange(exchangeIndex, request, outcome, bodyCapture) {
8806
+ return this.reportExchange?.({
8807
+ attemptId: this.#attemptId,
8808
+ inboxUrl: this.inboxUrl,
8809
+ exchangeIndex,
8810
+ observedAt: Temporal.Now.instant().epochMilliseconds,
8811
+ request,
8812
+ outcome: Object.freeze(outcome),
8813
+ ...bodyCapture == null ? {} : { bodyCapture }
8814
+ }) ?? Promise.resolve();
8815
+ }
8816
+ response(response, disposition) {
8817
+ const observed = this.#responses.get(response);
8818
+ if (observed == null) return;
8819
+ if (disposition === "final") {
8820
+ this.#finalResponse = Object.freeze({
8821
+ type: "final",
8822
+ ...observed
8823
+ });
8824
+ this.#response = this.#finalResponse;
8825
+ } else this.#response = Object.freeze({
8826
+ type: "intermediate-only",
8827
+ lastObserved: observed
8828
+ });
8829
+ }
8830
+ failure(failure) {
8831
+ this.#failures.push(Object.freeze(failure));
8832
+ }
8833
+ unknown(error) {
8834
+ if (!this.#failures.some((failure) => failure.error === error || failure.phase !== "cleanup")) this.failure({
8835
+ kind: "unknown",
8836
+ phase: "unknown",
8837
+ error
8838
+ });
8839
+ }
8840
+ succeeded() {
8841
+ if (this.#finished) return Promise.resolve();
8842
+ if (this.#finalResponse == null) throw new Error("A successful outbound attempt must have an observed final response.");
8843
+ return this.#finish({
8844
+ type: "succeeded",
8845
+ response: this.#finalResponse
8846
+ });
8847
+ }
8848
+ failed(error) {
8849
+ let failure;
8850
+ let diagnostics;
8851
+ if (this.#response.type === "final" && (this.#response.metadata.status < 200 || this.#response.metadata.status >= 300)) {
8852
+ failure = this.#failures.find((failure) => failure.kind === "http") ?? Object.freeze({
8853
+ kind: "http",
8854
+ phase: "response"
8855
+ });
8856
+ diagnostics = this.#failures.filter((failure) => failure.kind !== "http");
8857
+ } else {
8858
+ failure = this.#failures.find((failure) => failure.phase !== "cleanup") ?? this.#failures[0] ?? Object.freeze({
8859
+ kind: "unknown",
8860
+ phase: "unknown",
8861
+ error
8862
+ });
8863
+ diagnostics = this.#failures.filter((item) => item !== failure);
8864
+ }
8865
+ return this.#finish({
8866
+ type: "failed",
8867
+ failure,
8868
+ error,
8869
+ response: this.#response,
8870
+ ...diagnostics.length === 0 ? {} : { diagnostics: Object.freeze(diagnostics) }
8871
+ });
8872
+ }
8873
+ #finish(outcome) {
8874
+ if (this.#finished) return Promise.resolve();
8875
+ this.#finished = true;
8876
+ return this.report({
8877
+ attemptId: this.#attemptId,
8878
+ inboxUrl: this.inboxUrl,
8879
+ httpExchangeCount: this.#exchangeCount,
8880
+ senderPayload: this.#senderPayload,
8881
+ observedAt: Temporal.Now.instant().epochMilliseconds,
8882
+ ...this.#trace == null ? {} : { trace: this.#trace },
8883
+ outcome: Object.freeze(outcome)
8884
+ });
8885
+ }
8886
+ };
8887
+ function snapshotOutboxIdentity(message) {
8888
+ if (message.outbound != null) try {
8889
+ return snapshotDeliveryIdentity(message.outbound);
8890
+ } catch (error) {
8891
+ warnInvalidDeliveryIdentity(message.id, error);
8892
+ }
8893
+ return Object.freeze({
8894
+ broadcast: Object.freeze({ type: "unknown" }),
8895
+ deliveryId: message.id,
8896
+ ...message.portableInbox == null ? {} : { portableInbox: message.portableInbox }
8897
+ });
8898
+ }
8899
+ /** Per-execution state, independent of concrete attempt/exchange callbacks. */
8900
+ var OutboundWorkerObservation = class {
8901
+ context;
8902
+ callback;
8903
+ provenance;
8904
+ #attemptsStarted = 0;
8905
+ #actions;
8906
+ #trace;
8907
+ #preparationFailure;
8908
+ #preparationFailures = [];
8909
+ #queue;
8910
+ #delivery;
8911
+ #role;
8912
+ #currentContext;
8913
+ constructor(context, callback, message, provenance) {
8914
+ this.context = context;
8915
+ this.callback = callback;
8916
+ this.provenance = provenance;
8917
+ this.#role = message.type;
8918
+ this.#queue = Object.freeze({
8919
+ queueMessageId: message.id,
8920
+ ...message.type === "outbox" ? { queueAttempt: message.attempt } : {}
8921
+ });
8922
+ if (message.type === "outbox") this.#delivery = snapshotOutboxIdentity(message);
8923
+ }
8924
+ attemptStarted = () => {
8925
+ this.#attemptsStarted++;
8926
+ };
8927
+ enableActions() {
8928
+ this.#actions ??= [];
8929
+ }
8930
+ directAttemptStarted(inboxUrl, identity) {
8931
+ this.attemptStarted();
8932
+ this.#actions?.push(Object.freeze({
8933
+ type: "handoff",
8934
+ action: "direct",
8935
+ inboxUrls: Object.freeze([inboxUrl]),
8936
+ ...identity == null ? {} : { delivery: snapshotDeliveryIdentity(identity) }
8937
+ }));
8938
+ }
8939
+ async enqueueChildren(messages, enqueue) {
8940
+ if (this.#actions == null) return await enqueue();
8941
+ const facts = messages.map((message) => ({
8942
+ action: "enqueue",
8943
+ queue: Object.freeze({
8944
+ queueMessageId: message.id,
8945
+ queueAttempt: message.attempt
8946
+ }),
8947
+ inboxUrls: Object.freeze(message.gatewayInboxes != null && message.gatewayInboxes.length > 1 ? [...message.gatewayInboxes] : [message.inbox]),
8948
+ ...message.outbound == null ? {} : { delivery: snapshotDeliveryIdentity(message.outbound) }
8949
+ }));
8950
+ try {
8951
+ await enqueue();
8952
+ } catch (error) {
8953
+ for (const fact of facts) this.#actions.push(Object.freeze({
8954
+ type: "action-failed",
8955
+ ...fact,
8956
+ acceptance: "unknown",
8957
+ error
8958
+ }));
8959
+ throw error;
8960
+ }
8961
+ for (const fact of facts) this.#actions.push(Object.freeze({
8962
+ type: "scheduled",
8963
+ ...fact
8964
+ }));
8965
+ }
8966
+ async enqueue(action, message, delay, enqueue, preparationFailures) {
8967
+ if (this.#actions == null) return await enqueue();
8968
+ const facts = {
8969
+ action,
8970
+ queue: Object.freeze({
8971
+ queueMessageId: message.id,
8972
+ queueAttempt: message.attempt
8973
+ }),
8974
+ inboxUrls: Object.freeze(message.gatewayInboxes != null && message.gatewayInboxes.length > 1 ? [...message.gatewayInboxes] : [message.inbox]),
8975
+ delay: delay.toString(),
8976
+ ...preparationFailures == null || preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...preparationFailures]) },
8977
+ ...message.gatewayNotBefore == null ? {} : { notBefore: Object.freeze({ ...message.gatewayNotBefore }) }
8978
+ };
8979
+ try {
8980
+ await enqueue();
8981
+ } catch (error) {
8982
+ this.#actions.push(Object.freeze({
8983
+ type: "action-failed",
8984
+ ...facts,
8985
+ acceptance: "unknown",
8986
+ error
8987
+ }));
8988
+ throw error;
8989
+ }
8990
+ this.#actions.push(Object.freeze({
8991
+ type: "scheduled",
8992
+ ...facts
8993
+ }));
8994
+ }
8995
+ abandon(reason, inboxUrls, preparationFailures) {
8996
+ this.#actions?.push(Object.freeze({
8997
+ type: "abandoned",
8998
+ reason,
8999
+ inboxUrls: Object.freeze([...inboxUrls]),
9000
+ ...preparationFailures == null || preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...preparationFailures]) }
9001
+ }));
9002
+ }
9003
+ dropFailure(inboxUrls, error) {
9004
+ this.#actions?.push(Object.freeze({
9005
+ type: "action-failed",
9006
+ action: "drop",
9007
+ inboxUrls: Object.freeze([...inboxUrls]),
9008
+ error
9009
+ }));
9010
+ }
9011
+ async drop(inboxUrls, operation) {
9012
+ try {
9013
+ await operation();
9014
+ } catch (error) {
9015
+ this.dropFailure(inboxUrls, error);
9016
+ throw error;
9017
+ }
9018
+ }
9019
+ getContext() {
9020
+ return this.#currentContext ??= this.context();
9021
+ }
9022
+ useContext(context) {
9023
+ this.#currentContext = context;
9024
+ }
9025
+ setSpan(span, parentSpanContext) {
9026
+ this.#trace = snapshotTrace(span, parentSpanContext);
9027
+ }
9028
+ async prepare(phase, operation) {
9029
+ try {
9030
+ const result = await operation();
9031
+ this.#preparationFailure = void 0;
9032
+ return result;
9033
+ } catch (error) {
9034
+ this.#preparationFailure = Object.freeze({
9035
+ kind: "local",
9036
+ phase,
9037
+ error
9038
+ });
9039
+ throw error;
9040
+ }
9041
+ }
9042
+ recordPreparationFailure(error, inboxUrl) {
9043
+ const failure = this.#preparationFailure;
9044
+ if (failure == null || !Object.is(failure.error, error)) return [];
9045
+ const diagnostic = Object.freeze({
9046
+ inboxUrl,
9047
+ failure
9048
+ });
9049
+ this.#preparationFailures.push(diagnostic);
9050
+ return [diagnostic];
9051
+ }
9052
+ async run(operation) {
9053
+ try {
9054
+ await operation();
9055
+ } catch (error) {
9056
+ await this.#notify(this.#preparationFailure == null || !Object.is(this.#preparationFailure.error, error) ? {
9057
+ type: "failed",
9058
+ error
9059
+ } : {
9060
+ type: "preparation-failed",
9061
+ failure: this.#preparationFailure,
9062
+ error
9063
+ });
9064
+ throw error;
9065
+ }
9066
+ await this.#notify({ type: "completed" });
9067
+ }
9068
+ async #notify(result) {
9069
+ const observedAt = Temporal.Now.instant().epochMilliseconds;
9070
+ try {
9071
+ const context = this.getContext();
9072
+ await notifyObserver(context, this.callback, Object.freeze({
9073
+ role: this.#role,
9074
+ queue: this.#queue,
9075
+ ...this.#delivery == null ? {} : { delivery: this.#delivery },
9076
+ attemptsStarted: this.#attemptsStarted,
9077
+ actions: Object.freeze([...this.#actions ?? []]),
9078
+ ...this.#preparationFailures.length === 0 ? {} : { preparationFailures: Object.freeze([...this.#preparationFailures]) },
9079
+ observedAt,
9080
+ ...this.#trace == null ? {} : { trace: this.#trace },
9081
+ provenance: this.provenance,
9082
+ local: snapshotLocalContext(context),
9083
+ result: Object.freeze(result)
9084
+ }), "worker");
9085
+ } catch (error) {
9086
+ logObserverFailure("worker", error);
9087
+ }
9088
+ }
9089
+ };
9090
+ /** Selected callback and restored grouping, shared by concrete gateway attempts. */
9091
+ var OutboundAttemptObserver = class OutboundAttemptObserver {
9092
+ context;
9093
+ callback;
9094
+ identity;
9095
+ provenance;
9096
+ onHttpExchange;
9097
+ headerPolicies;
9098
+ bodyCapture;
9099
+ #queue;
9100
+ constructor(context, callback, identity, provenance, queue, onHttpExchange = void 0, headerPolicies = {}, bodyCapture = void 0) {
9101
+ this.context = context;
9102
+ this.callback = callback;
9103
+ this.identity = identity;
9104
+ this.provenance = provenance;
9105
+ this.onHttpExchange = onHttpExchange;
9106
+ this.headerPolicies = headerPolicies;
9107
+ this.bodyCapture = bodyCapture;
9108
+ this.#queue = queue == null ? void 0 : Object.freeze({ ...queue });
9109
+ }
9110
+ static resumeOutbox(context, callback, message, provenance, onHttpExchange, headerPolicies = {}, bodyCapture) {
9111
+ if (callback == null && onHttpExchange == null && bodyCapture == null) return;
9112
+ const identity = snapshotOutboxIdentity(message);
9113
+ return new OutboundAttemptObserver(context, callback, identity, provenance, {
9114
+ queueMessageId: message.id,
9115
+ queueAttempt: message.attempt
9116
+ }, onHttpExchange, headerPolicies, bodyCapture);
9117
+ }
9118
+ startAttempt(inboxUrl) {
9119
+ if (this.callback == null && this.onHttpExchange == null && this.bodyCapture == null) return void 0;
9120
+ const context = this.context();
9121
+ return new OutboundAttemptObservation(inboxUrl, (facts) => notifyObserver(context, this.callback, Object.freeze({
9122
+ ...this.identity,
9123
+ ...facts,
9124
+ ...this.#queue == null ? {} : { queue: this.#queue },
9125
+ provenance: this.provenance,
9126
+ local: snapshotLocalContext(context)
9127
+ }), "attempt"), (facts) => notifyObserver(context, this.onHttpExchange, Object.freeze({
9128
+ ...this.identity,
9129
+ ...facts,
9130
+ ...this.#queue == null ? {} : { queue: this.#queue },
9131
+ provenance: this.provenance,
9132
+ local: snapshotLocalContext(context)
9133
+ }), "http-exchange"), this.onHttpExchange == null && this.bodyCapture == null ? {} : this.headerPolicies, this.bodyCapture == null ? void 0 : {
9134
+ ...this.bodyCapture,
9135
+ onBody: (event) => withContext(sourceLogContext(this.provenance), () => this.bodyCapture.onBody(context, Object.freeze({
9136
+ ...this.identity,
9137
+ ...event,
9138
+ ...this.#queue == null ? {} : { queue: this.#queue },
9139
+ provenance: this.provenance,
9140
+ local: snapshotLocalContext(context)
9141
+ })))
9142
+ }, this.callback != null && this.headerPolicies.senderPayload === true);
9143
+ }
9144
+ };
9145
+ /** One producer's destination, without callbacks or live inputs in queues. */
9146
+ var OutboundDeliveryObservation = class {
9147
+ context;
9148
+ observer;
9149
+ provenance;
9150
+ trace;
9151
+ #finished = false;
9152
+ #reference;
9153
+ #plan;
9154
+ #attempts;
9155
+ constructor(context, observer, identity, destination, provenance, trace, onAttempt, onHttpExchange, queue, headerPolicies = {}, bodyCapture) {
9156
+ this.context = context;
9157
+ this.observer = observer;
9158
+ this.provenance = provenance;
9159
+ this.trace = trace;
9160
+ this.#reference = snapshotDeliveryIdentity(identity);
9161
+ this.#attempts = new OutboundAttemptObserver(() => this.context, onAttempt, this.#reference, this.provenance, queue, onHttpExchange, headerPolicies, bodyCapture);
9162
+ this.#plan = Object.freeze({
9163
+ actorIds: Object.freeze(Array.from(destination.actorIds)),
9164
+ sharedInbox: destination.sharedInbox,
9165
+ mode: destination.mode ?? (destination.portableInbox == null ? "ordinary" : "portable-fallback")
9166
+ });
9167
+ }
9168
+ startAttempt(inboxUrl) {
9169
+ return this.#attempts.startAttempt(inboxUrl);
9170
+ }
9171
+ get identity() {
9172
+ return this.#reference;
9173
+ }
9174
+ planned() {
9175
+ return notifyObserver(this.context, this.observer, Object.freeze({
9176
+ ...this.trace == null ? {} : { trace: this.trace },
9177
+ observedAt: Temporal.Now.instant().epochMilliseconds,
9178
+ provenance: this.provenance,
9179
+ local: snapshotLocalContext(this.context),
9180
+ ...this.#reference,
9181
+ ...this.#plan,
9182
+ stage: "planned"
9183
+ }), "delivery");
9184
+ }
9185
+ handoff(result) {
9186
+ if (this.#finished) return Promise.resolve();
9187
+ this.#finished = true;
9188
+ return notifyObserver(this.context, this.observer, Object.freeze({
9189
+ ...this.trace == null ? {} : { trace: this.trace },
9190
+ observedAt: Temporal.Now.instant().epochMilliseconds,
9191
+ provenance: this.provenance,
9192
+ local: snapshotLocalContext(this.context),
9193
+ ...this.#reference,
9194
+ stage: "handoff",
9195
+ result: Object.freeze(result)
9196
+ }), "delivery");
9197
+ }
9198
+ failed(error) {
9199
+ if (this.#finished) return Promise.resolve();
9200
+ this.#finished = true;
9201
+ return notifyObserver(this.context, this.observer, Object.freeze({
9202
+ ...this.trace == null ? {} : { trace: this.trace },
9203
+ observedAt: Temporal.Now.instant().epochMilliseconds,
9204
+ provenance: this.provenance,
9205
+ local: snapshotLocalContext(this.context),
9206
+ ...this.#reference,
9207
+ stage: "preparation-failed",
9208
+ failure: Object.freeze({
9209
+ kind: "local",
9210
+ phase: "request-preparation",
9211
+ error
9212
+ })
9213
+ }), "delivery");
9214
+ }
9215
+ };
9216
+ /** State owned by one outgoing invocation, never serialized into queues. */
9217
+ var OutboundBroadcastObservation = class OutboundBroadcastObservation {
9218
+ context;
9219
+ operation;
9220
+ provenance;
9221
+ queue;
9222
+ #broadcast;
9223
+ #finished = false;
9224
+ #phase = "unknown";
9225
+ #planning = "not-started";
9226
+ #destinationCount = 0;
9227
+ #trace;
9228
+ #activity = Object.freeze({});
9229
+ #onBroadcast;
9230
+ #onDelivery;
9231
+ #onAttempt;
9232
+ #onHttpExchange;
9233
+ #headerPolicies;
9234
+ #bodyCapture;
9235
+ #deliveries = /* @__PURE__ */ new Map();
9236
+ constructor(context, observers, operation, provenance = getOutboundProvenance(context), broadcast = {
9237
+ type: "known",
9238
+ id: crypto.randomUUID()
9239
+ }, queue = void 0) {
9240
+ this.context = context;
9241
+ this.operation = operation;
9242
+ this.provenance = provenance;
9243
+ this.queue = queue;
9244
+ this.#broadcast = broadcast;
9245
+ this.#onBroadcast = observers.onBroadcast;
9246
+ this.#onDelivery = observers.onDelivery;
9247
+ this.#onAttempt = observers.onAttempt;
9248
+ this.#onHttpExchange = observers.onHttpExchange;
9249
+ this.#bodyCapture = observers.responseBodyCapture;
9250
+ this.#headerPolicies = {
9251
+ requestHeaders: observers.requestHeaders,
9252
+ responseHeaders: observers.responseHeaders,
9253
+ senderPayload: observers.senderPayload
9254
+ };
9255
+ }
9256
+ /** Continue an existing fanout; do not emit a new broadcast invocation. */
9257
+ static async resumeFanout(context, observers, message, span, parentSpanContext) {
9258
+ if (observers.onDelivery == null && observers.onAttempt == null && observers.onHttpExchange == null && observers.responseBodyCapture == null && observers.onWorker == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null)) return;
9259
+ const observation = new OutboundBroadcastObservation(context, {
9260
+ onDelivery: observers.onDelivery,
9261
+ onAttempt: observers.onAttempt,
9262
+ onHttpExchange: observers.onHttpExchange,
9263
+ requestHeaders: observers.requestHeaders,
9264
+ responseHeaders: observers.responseHeaders,
9265
+ responseBodyCapture: observers.responseBodyCapture,
9266
+ senderPayload: observers.senderPayload
9267
+ }, "send", getOutboundProvenance(context), { type: "unknown" }, { queueMessageId: message.id });
9268
+ if (span != null) observation.setSpan(span, parentSpanContext);
9269
+ observation.#finished = true;
9270
+ observation.planned(Object.keys(message.inboxes).length);
9271
+ for (const [inbox, destination] of Object.entries(message.inboxes)) {
9272
+ let identity;
9273
+ if (destination.outbound != null) try {
9274
+ const snapshot = snapshotDeliveryIdentity(destination.outbound);
9275
+ if (snapshot.destinationInbox !== inbox) throw new TypeError("Outbound delivery identity does not match its fanout destination.");
9276
+ identity = snapshot;
9277
+ } catch (error) {
9278
+ warnInvalidDeliveryIdentity(message.id, error);
9279
+ }
9280
+ identity ??= await legacyFanoutIdentity(message.id, inbox, destination.portableInbox);
9281
+ observation.#deliveries.set(inbox, observation.#createDelivery(inbox, destination, identity));
9282
+ }
9283
+ return observation;
9284
+ }
9285
+ #createDelivery(inbox, destination, identity) {
9286
+ return new OutboundDeliveryObservation(this.context, this.#onDelivery, identity ?? {
9287
+ broadcast: this.#broadcast,
9288
+ deliveryId: crypto.randomUUID(),
9289
+ destinationInbox: inbox,
9290
+ ...destination.portableInbox == null ? {} : { portableInbox: destination.portableInbox }
9291
+ }, destination, this.provenance, this.#trace, this.#onAttempt, this.#onHttpExchange, this.queue, this.#headerPolicies, this.#bodyCapture);
9292
+ }
9293
+ continuation(inbox) {
9294
+ const delivery = this.delivery(inbox);
9295
+ return delivery == null ? void 0 : { outbound: delivery.identity };
9296
+ }
9297
+ async planDestinations(inboxes) {
9298
+ this.planned(Object.keys(inboxes).length);
9299
+ for (const [inbox, destination] of Object.entries(inboxes)) if (!this.#deliveries.has(inbox)) this.#deliveries.set(inbox, this.#createDelivery(inbox, destination));
9300
+ for (const delivery of this.#deliveries.values()) await delivery.planned();
9301
+ }
9302
+ delivery(inbox) {
9303
+ return this.#deliveries.get(inbox);
9304
+ }
9305
+ async enqueue(entries, queue, enqueue) {
9306
+ let result;
9307
+ try {
9308
+ result = await enqueue();
9309
+ } catch (error) {
9310
+ await Promise.all(entries.map(({ inbox }) => this.delivery(inbox)?.handoff({
9311
+ type: "enqueue-failed",
9312
+ acceptance: "unknown",
9313
+ error
9314
+ })));
9315
+ throw error;
9316
+ }
9317
+ await Promise.all(entries.map(({ inbox, queueMessageId }) => this.delivery(inbox)?.handoff({
9318
+ type: "enqueued",
9319
+ queue,
9320
+ queueMessageId
9321
+ })));
9322
+ return result;
9323
+ }
9324
+ /** Attach only the span supplied by this execution, never an ambient fallback. */
9325
+ setSpan(span, parentSpanContext) {
9326
+ this.#trace = snapshotTrace(span, parentSpanContext);
9327
+ }
9328
+ setPhase(phase) {
9329
+ this.#phase = phase;
9330
+ if (phase === "destination-planning") this.#planning = "incomplete";
9331
+ }
9332
+ /** Replace the Activity facts captured by the producer path. */
9333
+ setActivity(reference) {
9334
+ this.#activity = Object.freeze({ ...reference });
9335
+ }
9336
+ planned(destinationCount) {
9337
+ this.#planning = "complete";
9338
+ this.#destinationCount = destinationCount;
9339
+ }
9340
+ async planDestination(inbox, destination) {
9341
+ this.#destinationCount++;
9342
+ const delivery = this.#createDelivery(inbox, destination);
9343
+ this.#deliveries.set(inbox, delivery);
9344
+ await delivery.planned();
9345
+ }
9346
+ prepared(preparedFor) {
9347
+ return this.#finish(Object.freeze({
9348
+ type: "prepared",
9349
+ preparedFor,
9350
+ destinationCount: this.#destinationCount
9351
+ }));
9352
+ }
9353
+ skipped(reason = "no-destinations") {
9354
+ return this.#finish(Object.freeze({
9355
+ type: "skipped",
9356
+ reason
9357
+ }));
9358
+ }
9359
+ async run(send) {
9360
+ if (this.#broadcast.type === "known") await this.#notify(Object.freeze({
9361
+ broadcastId: this.#broadcast.id,
9362
+ operation: this.operation,
9363
+ stage: "started",
9364
+ activity: this.#activity
9365
+ }));
9366
+ try {
9367
+ return await send(this, this.provenance);
9368
+ } catch (error) {
9369
+ await this.failed(error);
9370
+ throw error;
9371
+ }
9372
+ }
9373
+ failed(error) {
9374
+ const failure = {
9375
+ kind: this.#phase === "unknown" ? "unknown" : "local",
9376
+ phase: this.#phase,
9377
+ error
9378
+ };
9379
+ return this.#finish(Object.freeze({
9380
+ type: "failed",
9381
+ failure: Object.freeze(failure),
9382
+ planning: this.#planning,
9383
+ plannedDestinationCount: this.#destinationCount
9384
+ }));
9385
+ }
9386
+ async #finish(result) {
9387
+ if (this.#finished) return;
9388
+ this.#finished = true;
9389
+ if (this.#broadcast.type === "known") await this.#notify(Object.freeze({
9390
+ broadcastId: this.#broadcast.id,
9391
+ operation: this.operation,
9392
+ stage: "preparation-finished",
9393
+ result,
9394
+ activity: this.#activity
9395
+ }));
9396
+ }
9397
+ #notify(event) {
9398
+ return notifyObserver(this.context, this.#onBroadcast, Object.freeze({
9399
+ ...event,
9400
+ ...this.#trace == null ? {} : { trace: this.#trace },
9401
+ observedAt: Temporal.Now.instant().epochMilliseconds,
9402
+ provenance: this.provenance,
9403
+ local: snapshotLocalContext(this.context)
9404
+ }), "broadcast");
9405
+ }
9406
+ };
9407
+ /** Observe preparation without changing the unobserved send's execution path. */
9408
+ function observeOutboundBroadcast(context, observers, send, operation = "send", dispatchId) {
9409
+ let provenance;
9410
+ let invalid;
9411
+ try {
9412
+ provenance = invocationProvenance(context, dispatchId);
9413
+ } catch (error) {
9414
+ provenance = invocationProvenance(context, void 0);
9415
+ invalid = { error };
9416
+ }
9417
+ const execute = async (observation) => {
9418
+ if (invalid != null) {
9419
+ observation?.setPhase("validation");
9420
+ throw invalid.error;
9421
+ }
9422
+ return await send(observation, provenance);
9423
+ };
9424
+ return withContext(sourceLogContext(provenance), () => {
9425
+ if (observers.onBroadcast == null && observers.onDelivery == null && observers.onAttempt == null && observers.onHttpExchange == null && observers.responseBodyCapture == null) return execute();
9426
+ return new OutboundBroadcastObservation(context, observers, operation, provenance).run(execute);
9427
+ });
9428
+ }
9429
+ //#endregion
7866
9430
  //#region src/nodeinfo/handler.ts
7867
9431
  /**
7868
9432
  * Handles a NodeInfo request. You would not typically call this function
@@ -9443,41 +11007,70 @@ var FederationImpl = class extends FederationBuilderImpl {
9443
11007
  }
9444
11008
  await Promise.all(promises);
9445
11009
  }
9446
- processQueuedTask(contextData, message) {
11010
+ async processQueuedTask(contextData, message) {
9447
11011
  const extractedContext = propagation.extract(context.active(), message.traceContext);
9448
- return withContext({ messageId: message.id }, async () => {
11012
+ let provenance = emptyProvenance;
11013
+ if (message.type !== "task") try {
11014
+ provenance = restoreProvenance(message.provenance);
11015
+ } catch (error) {
11016
+ try {
11017
+ getLogger([
11018
+ "fedify",
11019
+ "federation",
11020
+ "queue"
11021
+ ]).warn("Ignoring invalid provenance in queue message {messageId}: {error}", {
11022
+ messageId: message.id,
11023
+ error
11024
+ });
11025
+ } catch {}
11026
+ }
11027
+ if (message.type !== "task" && message.provenance !== void 0) message = {
11028
+ ...message,
11029
+ provenance
11030
+ };
11031
+ return await withContext({
11032
+ messageId: message.id,
11033
+ inboxRequestId: void 0,
11034
+ ...sourceLogContext(provenance)
11035
+ }, async () => {
9449
11036
  if (message.type === "fanout") {
11037
+ const observers = this.outboundObservers;
11038
+ const worker = this.#createOutboundWorkerObservation(contextData, message, provenance, observers.onWorker);
9450
11039
  const common = {
9451
11040
  role: "fanout",
9452
11041
  queue: this.fanoutQueue,
9453
11042
  activityType: message.activityType
9454
11043
  };
9455
- await this.#runWorkerSpan("activitypub.fanout", { "activitypub.activity.type": message.activityType }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
11044
+ const execute = () => this.#runWorkerSpan("activitypub.fanout", { "activitypub.activity.type": message.activityType }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
9456
11045
  if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
9457
- await this.#listenFanoutMessage(contextData, message);
11046
+ await this.#listenFanoutMessage(contextData, message, provenance, span, trace.getSpanContext(extractedContext), observers, worker);
9458
11047
  return { outcome: "completed" };
9459
- }, classifyAbortableError));
11048
+ }, classifyAbortableError), worker);
11049
+ await (worker == null ? execute() : worker.run(execute));
9460
11050
  } else if (message.type === "outbox") {
11051
+ const observers = this.outboundObservers;
11052
+ const worker = this.#createOutboundWorkerObservation(contextData, message, provenance, observers.onWorker);
9461
11053
  const common = {
9462
11054
  role: "outbox",
9463
11055
  queue: this.outboxQueue,
9464
11056
  activityType: message.activityType
9465
11057
  };
9466
- await this.#runWorkerSpan("activitypub.outbox", {
11058
+ const execute = () => this.#runWorkerSpan("activitypub.outbox", {
9467
11059
  "activitypub.activity.type": message.activityType,
9468
11060
  "activitypub.activity.retries": message.attempt
9469
11061
  }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
9470
11062
  if (message.activityId != null) span.setAttribute("activitypub.activity.id", message.activityId);
9471
- await this.#listenOutboxMessage(contextData, message, span);
11063
+ await this.#listenOutboxMessage(contextData, message, span, provenance, observers, worker);
9472
11064
  return { outcome: "completed" };
9473
- }, classifyAbortableError));
11065
+ }, classifyAbortableError), worker);
11066
+ await (worker == null ? execute() : worker.run(execute));
9474
11067
  } else if (message.type === "inbox") {
9475
11068
  const common = {
9476
11069
  role: "inbox",
9477
11070
  queue: this.inboxQueue
9478
11071
  };
9479
11072
  await this.#runWorkerSpan("activitypub.inbox", { "activitypub.shared_inbox": message.identifier == null }, extractedContext, this.#instrumentWorkerBody(common, async (span) => {
9480
- await this.#listenInboxMessage(contextData, message, span, (activityType) => {
11073
+ await this.#listenInboxMessage(contextData, message, span, provenance, (activityType) => {
9481
11074
  common.activityType = activityType;
9482
11075
  });
9483
11076
  return { outcome: "completed" };
@@ -9496,6 +11089,22 @@ var FederationImpl = class extends FederationBuilderImpl {
9496
11089
  }
9497
11090
  });
9498
11091
  }
11092
+ #createOutboundWorkerObservation(contextData, message, provenance, callback) {
11093
+ if (callback == null) return void 0;
11094
+ let documentLoader;
11095
+ let contextLoader;
11096
+ return new OutboundWorkerObservation(() => {
11097
+ const loaderOptions = this.#getLoaderOptions(message.baseUrl);
11098
+ return new ContextImpl({
11099
+ [outboundProvenance]: provenance,
11100
+ federation: this,
11101
+ url: new URL("/", message.baseUrl),
11102
+ data: contextData,
11103
+ documentLoader: (...args) => (documentLoader ??= this.documentLoaderFactory(loaderOptions))(...args),
11104
+ contextLoader: (...args) => (contextLoader ??= this.contextLoaderFactory(loaderOptions))(...args)
11105
+ });
11106
+ }, callback, message, provenance);
11107
+ }
9499
11108
  /**
9500
11109
  * Opens a CONSUMER span for a queue worker and scopes the extracted trace
9501
11110
  * context around its body. The body—typically built by
@@ -9507,11 +11116,12 @@ var FederationImpl = class extends FederationBuilderImpl {
9507
11116
  * @param instrumentedBody Builds the context-scoped worker callback for the
9508
11117
  * opened span.
9509
11118
  */
9510
- async #runWorkerSpan(spanName, spanAttributes, extractedContext, instrumentedBody) {
11119
+ async #runWorkerSpan(spanName, spanAttributes, extractedContext, instrumentedBody, worker) {
9511
11120
  await this._getTracer().startActiveSpan(spanName, {
9512
11121
  kind: SpanKind.CONSUMER,
9513
11122
  attributes: spanAttributes
9514
11123
  }, extractedContext, async (span) => {
11124
+ worker?.setSpan(span, trace.getSpanContext(extractedContext));
9515
11125
  const spanCtx = span.spanContext();
9516
11126
  return await withContext({
9517
11127
  traceId: spanCtx.traceId,
@@ -9554,7 +11164,8 @@ var FederationImpl = class extends FederationBuilderImpl {
9554
11164
  span.end();
9555
11165
  }
9556
11166
  };
9557
- async #listenFanoutMessage(data, message) {
11167
+ async #listenFanoutMessage(data, message, provenance, span, parentSpanContext, observers, worker) {
11168
+ worker?.enableActions();
9558
11169
  getLogger([
9559
11170
  "fedify",
9560
11171
  "federation",
@@ -9563,11 +11174,12 @@ var FederationImpl = class extends FederationBuilderImpl {
9563
11174
  activityId: message.activityId,
9564
11175
  inboxes: globalThis.Object.keys(message.inboxes).length
9565
11176
  });
9566
- const keys = await Promise.all(message.keys.map(async ({ keyId, privateKey }) => ({
11177
+ const importKeys = () => Promise.all(message.keys.map(async ({ keyId, privateKey }) => ({
9567
11178
  keyId: new URL(keyId),
9568
11179
  privateKey: await importJwk(privateKey, "private")
9569
11180
  })));
9570
- const activity = await Activity.fromJsonLd(message.activity, {
11181
+ const keys = worker == null ? await importKeys() : await worker.prepare("key-import", importKeys);
11182
+ const parseActivity = () => Activity.fromJsonLd(message.activity, {
9571
11183
  contextLoader: this.contextLoaderFactory({
9572
11184
  allowPrivateAddress: this.allowPrivateAddress,
9573
11185
  userAgent: this.userAgent
@@ -9578,19 +11190,30 @@ var FederationImpl = class extends FederationBuilderImpl {
9578
11190
  }),
9579
11191
  tracerProvider: this.tracerProvider
9580
11192
  });
9581
- const context = this.#createContext(new URL(message.baseUrl), data, { documentLoader: this.documentLoaderFactory({
9582
- allowPrivateAddress: this.allowPrivateAddress,
9583
- userAgent: this.userAgent
9584
- }) });
11193
+ const activity = worker == null ? await parseActivity() : await worker.prepare("activity-parsing", parseActivity);
11194
+ const createContext = () => this.#createContext(new URL(message.baseUrl), data, {
11195
+ [outboundProvenance]: provenance,
11196
+ documentLoader: this.documentLoaderFactory({
11197
+ allowPrivateAddress: this.allowPrivateAddress,
11198
+ userAgent: this.userAgent
11199
+ })
11200
+ });
11201
+ const context = worker == null ? createContext() : await worker.prepare("context-preparation", createContext);
11202
+ worker?.useContext(context);
11203
+ const observation = await OutboundBroadcastObservation.resumeFanout(context, observers, message, span, parentSpanContext);
9585
11204
  await this.sendActivity(keys, message.inboxes, activity, {
9586
11205
  collectionSync: message.collectionSync,
9587
11206
  orderingKey: message.orderingKey,
9588
11207
  normalizeExistingProofs: message.normalizeExistingProofs,
9589
11208
  activityJsonLd: message.activity,
11209
+ observation,
11210
+ worker,
9590
11211
  context
9591
11212
  });
9592
11213
  }
9593
- async #listenOutboxMessage(_, message, span) {
11214
+ async #listenOutboxMessage(contextData, message, span, provenance, observers, worker) {
11215
+ const { onAttempt, onHttpExchange } = observers;
11216
+ worker?.enableActions();
9594
11217
  const logger = getLogger([
9595
11218
  "fedify",
9596
11219
  "federation",
@@ -9607,14 +11230,26 @@ var FederationImpl = class extends FederationBuilderImpl {
9607
11230
  const keys = [];
9608
11231
  let rsaKeyPair = null;
9609
11232
  for (const { keyId, privateKey } of message.keys) {
9610
- const pair = {
11233
+ const importPair = async () => ({
9611
11234
  keyId: new URL(keyId),
9612
11235
  privateKey: await importJwk(privateKey, "private")
9613
- };
11236
+ });
11237
+ const pair = worker == null ? await importPair() : await worker.prepare("key-import", importPair);
9614
11238
  if (rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") rsaKeyPair = pair;
9615
11239
  keys.push(pair);
9616
11240
  }
9617
11241
  const loaderOptions = this.#getLoaderOptions(message.baseUrl);
11242
+ let observationContext;
11243
+ let observationDocumentLoader;
11244
+ let observationContextLoader;
11245
+ const observation = OutboundAttemptObserver.resumeOutbox(() => worker?.getContext() ?? (observationContext ??= new ContextImpl({
11246
+ [outboundProvenance]: provenance,
11247
+ federation: this,
11248
+ url: new URL("/", message.baseUrl),
11249
+ data: contextData,
11250
+ documentLoader: (...args) => (observationDocumentLoader ??= this.documentLoaderFactory(loaderOptions))(...args),
11251
+ contextLoader: (...args) => (observationContextLoader ??= this.contextLoaderFactory(loaderOptions))(...args)
11252
+ })), onAttempt, message, provenance, onHttpExchange, observers, observers.responseBodyCapture);
9618
11253
  let parsedActorIds;
9619
11254
  const getActorIds = () => {
9620
11255
  parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => {
@@ -9637,7 +11272,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9637
11272
  verifyPortableObject: createPortableObjectVerifier(documentLoader, contextLoader, this)
9638
11273
  });
9639
11274
  };
9640
- const enqueueHeldOutboxMessage = async (delay, heldSince) => {
11275
+ const enqueueHeldOutboxMessage = async (delay, heldSince, preparationFailures) => {
9641
11276
  const { outboxQueue } = this;
9642
11277
  if (outboxQueue == null) return;
9643
11278
  const heldMessage = {
@@ -9645,10 +11280,11 @@ var FederationImpl = class extends FederationBuilderImpl {
9645
11280
  circuitHeld: true,
9646
11281
  circuitHeldSince: heldSince.toString()
9647
11282
  };
9648
- await outboxQueue.enqueue(heldMessage, {
11283
+ const enqueueOptions = {
9649
11284
  delay: clampNegativeDelay(delay),
9650
11285
  orderingKey: message.orderingKey
9651
- });
11286
+ };
11287
+ await (worker == null ? outboxQueue.enqueue(heldMessage, enqueueOptions) : worker.enqueue("defer", heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), preparationFailures));
9652
11288
  this.metrics.recordQueueTaskEnqueued({
9653
11289
  role: "outbox",
9654
11290
  queue: outboxQueue,
@@ -9665,7 +11301,10 @@ var FederationImpl = class extends FederationBuilderImpl {
9665
11301
  heldSince
9666
11302
  });
9667
11303
  if (this.outboxPermanentFailureHandler != null) {
9668
- const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
11304
+ const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
11305
+ [outboundProvenance]: provenance,
11306
+ documentLoader: this.documentLoaderFactory(loaderOptions)
11307
+ });
9669
11308
  try {
9670
11309
  await this.outboxPermanentFailureHandler(ctx, {
9671
11310
  reason: "circuit-breaker-ttl",
@@ -9677,6 +11316,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9677
11316
  actorIds: getActorIds()
9678
11317
  });
9679
11318
  } catch (handlerError) {
11319
+ worker?.dropFailure([inbox.href], handlerError);
9680
11320
  logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
9681
11321
  ...logData,
9682
11322
  error: handlerError
@@ -9687,10 +11327,13 @@ var FederationImpl = class extends FederationBuilderImpl {
9687
11327
  };
9688
11328
  const { gatewayInboxes } = message;
9689
11329
  if (gatewayInboxes != null && gatewayInboxes.length > 1) {
9690
- await this.#listenGatewayOutboxMessage(_, {
11330
+ await this.#listenGatewayOutboxMessage(contextData, {
9691
11331
  ...message,
9692
11332
  gatewayInboxes
9693
11333
  }, span, {
11334
+ observation,
11335
+ worker,
11336
+ provenance,
9694
11337
  keys,
9695
11338
  logData,
9696
11339
  getActorIds,
@@ -9700,7 +11343,8 @@ var FederationImpl = class extends FederationBuilderImpl {
9700
11343
  return;
9701
11344
  }
9702
11345
  try {
9703
- const inbox = new URL(message.inbox);
11346
+ const parseInbox = () => new URL(message.inbox);
11347
+ const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
9704
11348
  const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
9705
11349
  const remoteHost = getRemoteHost(inbox);
9706
11350
  let decision;
@@ -9724,13 +11368,19 @@ var FederationImpl = class extends FederationBuilderImpl {
9724
11368
  return;
9725
11369
  }
9726
11370
  if (decision.type === "drop") {
9727
- const activity = await parseActivity();
9728
- await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
11371
+ worker?.abandon("held-ttl-expired", [inbox.href]);
11372
+ const drop = async () => {
11373
+ const activity = await parseActivity();
11374
+ await dropHeldOutboxMessage(circuit, remoteHost, inbox, decision.heldSince, activity);
11375
+ };
11376
+ await (worker == null ? drop() : worker.drop([inbox.href], drop));
9729
11377
  return;
9730
11378
  }
9731
11379
  if (decision.stateChange != null) recordCircuitBreakerSpanEvent(span, remoteHost, decision.stateChange);
9732
11380
  }
9733
11381
  await sendActivity({
11382
+ attemptStarted: worker?.attemptStarted,
11383
+ observation,
9734
11384
  allowPrivateAddress: this.allowPrivateAddress,
9735
11385
  keys,
9736
11386
  activity: message.activity,
@@ -9738,7 +11388,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9738
11388
  activityType: message.activityType,
9739
11389
  inbox,
9740
11390
  sharedInbox: message.sharedInbox,
9741
- headers: new Headers(message.headers),
11391
+ headers: worker == null ? new Headers(message.headers) : await worker.prepare("request-preparation", () => new Headers(message.headers)),
9742
11392
  specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
9743
11393
  meterProvider: this.meterProvider,
9744
11394
  tracerProvider: this.tracerProvider
@@ -9758,6 +11408,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9758
11408
  });
9759
11409
  }
9760
11410
  } catch (error) {
11411
+ const preparationFailures = worker?.recordPreparationFailure(error, message.inbox) ?? [];
9761
11412
  span.setStatus({
9762
11413
  code: SpanStatusCode.ERROR,
9763
11414
  message: String(error)
@@ -9766,7 +11417,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9766
11417
  if (error instanceof SendActivityError) return getRemoteHost(error.inbox);
9767
11418
  try {
9768
11419
  return getRemoteHost(new URL(message.inbox));
9769
- } catch (_) {
11420
+ } catch {
9770
11421
  logger.warn("Invalid inbox URL in queued outbox message: {inbox}", logData);
9771
11422
  return;
9772
11423
  }
@@ -9839,32 +11490,42 @@ var FederationImpl = class extends FederationBuilderImpl {
9839
11490
  });
9840
11491
  }
9841
11492
  if (circuitDrop != null) {
9842
- await dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
11493
+ worker?.abandon("held-ttl-expired", [circuitDrop.inbox.href], preparationFailures);
11494
+ const drop = () => dropHeldOutboxMessage(circuitDrop.circuit, circuitDrop.remoteHost, circuitDrop.inbox, circuitDrop.heldSince, activity);
11495
+ await (worker == null ? drop() : worker.drop([circuitDrop.inbox.href], drop));
9843
11496
  return;
9844
11497
  }
9845
11498
  if (isPermanentFailure) {
11499
+ worker?.abandon("permanent-failure", [message.inbox], preparationFailures);
9846
11500
  this.metrics.recordPermanentFailure(error.inbox, error.statusCode);
9847
11501
  logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not retrying.", {
9848
11502
  ...logData,
9849
11503
  status: error.statusCode
9850
11504
  });
9851
11505
  if (this.outboxPermanentFailureHandler != null) {
9852
- const ctx = this.#createContext(new URL(message.baseUrl), _, { documentLoader: this.documentLoaderFactory(loaderOptions) });
9853
- try {
9854
- await this.outboxPermanentFailureHandler(ctx, {
9855
- reason: "http",
9856
- inbox: new URL(message.inbox),
9857
- activity,
9858
- error,
9859
- statusCode: error.statusCode,
9860
- actorIds: getActorIds()
11506
+ const drop = async () => {
11507
+ const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
11508
+ [outboundProvenance]: provenance,
11509
+ documentLoader: this.documentLoaderFactory(loaderOptions)
9861
11510
  });
9862
- } catch (handlerError) {
9863
- logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
9864
- ...logData,
9865
- error: handlerError
9866
- });
9867
- }
11511
+ try {
11512
+ await this.outboxPermanentFailureHandler(ctx, {
11513
+ reason: "http",
11514
+ inbox: new URL(message.inbox),
11515
+ activity,
11516
+ error,
11517
+ statusCode: error.statusCode,
11518
+ actorIds: getActorIds()
11519
+ });
11520
+ } catch (handlerError) {
11521
+ worker?.dropFailure([message.inbox], handlerError);
11522
+ logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
11523
+ ...logData,
11524
+ error: handlerError
11525
+ });
11526
+ }
11527
+ };
11528
+ await (worker == null ? drop() : worker.drop([message.inbox], drop));
9868
11529
  }
9869
11530
  recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
9870
11531
  return;
@@ -9876,7 +11537,7 @@ var FederationImpl = class extends FederationBuilderImpl {
9876
11537
  });
9877
11538
  recordCircuitBreakerHeldSpanEvent(span, circuitHold.remoteHost, circuitHold.state);
9878
11539
  const circuit = this.circuitBreaker;
9879
- await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince);
11540
+ await enqueueHeldOutboxMessage(retryAfterDelay == null || circuit == null ? circuitHold.delay : circuit.capHeldDelay(circuitHold.heldSince, maxDelay(circuitHold.delay, retryAfterDelay)), circuitHold.heldSince, preparationFailures);
9880
11541
  return;
9881
11542
  }
9882
11543
  if (this.outboxQueue?.nativeRetrial && retryAfterDelay == null) {
@@ -9899,18 +11560,20 @@ var FederationImpl = class extends FederationBuilderImpl {
9899
11560
  };
9900
11561
  const { outboxQueue } = this;
9901
11562
  if (outboxQueue != null) {
9902
- await outboxQueue.enqueue(retryMessage, {
11563
+ const enqueueOptions = {
9903
11564
  delay: clampNegativeDelay(delay),
9904
11565
  orderingKey: message.orderingKey
9905
- });
11566
+ };
11567
+ await (worker == null ? outboxQueue.enqueue(retryMessage, enqueueOptions) : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => outboxQueue.enqueue(retryMessage, enqueueOptions), preparationFailures));
9906
11568
  this.metrics.recordQueueTaskEnqueued({
9907
11569
  role: "outbox",
9908
11570
  queue: outboxQueue,
9909
11571
  activityType: retryMessage.activityType
9910
11572
  }, retryMessage.attempt);
9911
11573
  recordOutboxActivity(this.meterProvider, "retried", retryMessage.activityType);
9912
- }
11574
+ } else worker?.abandon("queue-unavailable", [message.inbox], preparationFailures);
9913
11575
  } else {
11576
+ worker?.abandon("retry-policy-declined", [message.inbox], preparationFailures);
9914
11577
  logger.error("Failed to send activity {activityId} to {inbox} after {attempt} attempts; giving up:\n{error}", {
9915
11578
  ...logData,
9916
11579
  error
@@ -9945,25 +11608,32 @@ var FederationImpl = class extends FederationBuilderImpl {
9945
11608
  "federation",
9946
11609
  "circuit"
9947
11610
  ]);
9948
- const { keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = helpers;
11611
+ const { observation, worker, provenance, keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = helpers;
9949
11612
  const circuit = this.outboxQueue == null ? void 0 : this.circuitBreaker;
9950
11613
  const specDeterminer = new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl });
9951
11614
  const survivors = [];
11615
+ const preparationFailuresByRoute = /* @__PURE__ */ new Map();
11616
+ const permanentRoutes = /* @__PURE__ */ new Set();
9952
11617
  let lastError;
9953
11618
  let lastPermanent;
9954
11619
  let activity;
9955
11620
  const getActivity = async () => activity ??= await parseActivity();
9956
11621
  const drop = async (circuit, remoteHost, inbox, heldSince) => {
9957
- await dropHeldOutboxMessage(circuit, remoteHost, inbox, heldSince, await getActivity());
11622
+ const routes = message.gatewayInboxes.filter((route) => !permanentRoutes.has(route));
11623
+ worker?.abandon("held-ttl-expired", routes, routes.flatMap((route) => preparationFailuresByRoute.get(route) ?? []));
11624
+ const execute = async () => dropHeldOutboxMessage(circuit, remoteHost, inbox, heldSince, await getActivity());
11625
+ await (worker == null ? execute() : worker.drop(routes, execute));
9958
11626
  };
9959
11627
  const heldSince = parseInstant(message.circuitHeldSince);
9960
11628
  if (circuit != null && heldSince != null && Temporal.Instant.compare(heldSince.add(circuit.options.heldActivityTtl), Temporal.Now.instant()) <= 0) {
9961
- const inbox = new URL(message.gatewayInboxes[0]);
11629
+ const parseInbox = () => new URL(message.gatewayInboxes[0]);
11630
+ const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
9962
11631
  await drop(circuit, getRemoteHost(inbox), inbox, heldSince);
9963
11632
  return;
9964
11633
  }
9965
11634
  for (const inboxHref of message.gatewayInboxes) {
9966
- const inbox = new URL(inboxHref);
11635
+ const parseInbox = () => new URL(inboxHref);
11636
+ const inbox = worker == null ? parseInbox() : await worker.prepare("validation", parseInbox);
9967
11637
  const remoteHost = getRemoteHost(inbox);
9968
11638
  const now = Temporal.Now.instant();
9969
11639
  const notBefore = parseInstant(message.gatewayNotBefore?.[inboxHref]);
@@ -10001,6 +11671,8 @@ var FederationImpl = class extends FederationBuilderImpl {
10001
11671
  }
10002
11672
  try {
10003
11673
  await sendActivity({
11674
+ attemptStarted: worker?.attemptStarted,
11675
+ observation,
10004
11676
  allowPrivateAddress: this.allowPrivateAddress,
10005
11677
  keys,
10006
11678
  activity: message.activity,
@@ -10008,12 +11680,14 @@ var FederationImpl = class extends FederationBuilderImpl {
10008
11680
  activityType: message.activityType,
10009
11681
  inbox,
10010
11682
  sharedInbox: message.sharedInbox,
10011
- headers: new Headers(message.headers),
11683
+ headers: worker == null ? new Headers(message.headers) : await worker.prepare("request-preparation", () => new Headers(message.headers)),
10012
11684
  specDeterminer,
10013
11685
  meterProvider: this.meterProvider,
10014
11686
  tracerProvider: this.tracerProvider
10015
11687
  });
10016
11688
  } catch (error) {
11689
+ const preparationFailures = worker?.recordPreparationFailure(error, inboxHref) ?? [];
11690
+ if (preparationFailures.length > 0) preparationFailuresByRoute.set(inboxHref, preparationFailures);
10017
11691
  lastError = error;
10018
11692
  const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode);
10019
11693
  let retryAfter = !isPermanentFailure && error instanceof SendActivityError && (error.statusCode === 429 || error.statusCode === 503) ? parseRetryAfter(error.responseHeaders) : void 0;
@@ -10058,6 +11732,8 @@ var FederationImpl = class extends FederationBuilderImpl {
10058
11732
  });
10059
11733
  }
10060
11734
  if (isPermanentFailure) {
11735
+ permanentRoutes.add(inboxHref);
11736
+ worker?.abandon("permanent-failure", [inboxHref], preparationFailures);
10061
11737
  logger.warn("Permanent delivery failure for activity {activityId} to {inbox} ({status}); not trying the gateway again.", {
10062
11738
  ...logData,
10063
11739
  inbox: inboxHref,
@@ -10122,22 +11798,29 @@ var FederationImpl = class extends FederationBuilderImpl {
10122
11798
  portableInbox: message.portableInbox
10123
11799
  });
10124
11800
  if (this.outboxPermanentFailureHandler != null) {
10125
- const ctx = this.#createContext(new URL(message.baseUrl), contextData, { documentLoader: this.documentLoaderFactory(this.#getLoaderOptions(message.baseUrl)) });
10126
- try {
10127
- await this.outboxPermanentFailureHandler(ctx, {
10128
- reason: "http",
10129
- inbox,
10130
- activity: await getActivity(),
10131
- error,
10132
- statusCode: error.statusCode,
10133
- actorIds: getActorIds()
11801
+ const drop = async () => {
11802
+ const ctx = this.#createContext(new URL(message.baseUrl), contextData, {
11803
+ [outboundProvenance]: provenance,
11804
+ documentLoader: this.documentLoaderFactory(this.#getLoaderOptions(message.baseUrl))
10134
11805
  });
10135
- } catch (handlerError) {
10136
- logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
10137
- ...logData,
10138
- error: handlerError
10139
- });
10140
- }
11806
+ try {
11807
+ await this.outboxPermanentFailureHandler(ctx, {
11808
+ reason: "http",
11809
+ inbox,
11810
+ activity: await getActivity(),
11811
+ error,
11812
+ statusCode: error.statusCode,
11813
+ actorIds: getActorIds()
11814
+ });
11815
+ } catch (handlerError) {
11816
+ worker?.dropFailure(message.gatewayInboxes, handlerError);
11817
+ logger.error("An unexpected error occurred in outboxPermanentFailureHandler:\n{error}", {
11818
+ ...logData,
11819
+ error: handlerError
11820
+ });
11821
+ }
11822
+ };
11823
+ await (worker == null ? drop() : worker.drop(message.gatewayInboxes, drop));
10141
11824
  }
10142
11825
  recordOutboxActivity(this.meterProvider, "abandoned", message.activityType);
10143
11826
  return;
@@ -10173,10 +11856,11 @@ var FederationImpl = class extends FederationBuilderImpl {
10173
11856
  ...logData,
10174
11857
  portableInbox: message.portableInbox
10175
11858
  });
10176
- await outboxQueue.enqueue(heldMessage, {
11859
+ const enqueueOptions = {
10177
11860
  delay: clampNegativeDelay(delay),
10178
11861
  orderingKey: message.orderingKey
10179
- });
11862
+ };
11863
+ await (worker == null ? outboxQueue.enqueue(heldMessage, enqueueOptions) : worker.enqueue("defer", heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? [])));
10180
11864
  this.metrics.recordQueueTaskEnqueued({
10181
11865
  role: "outbox",
10182
11866
  queue: outboxQueue,
@@ -10194,6 +11878,7 @@ var FederationImpl = class extends FederationBuilderImpl {
10194
11878
  throw lastError;
10195
11879
  }
10196
11880
  if (policyDelay == null || outboxQueue == null) {
11881
+ worker?.abandon(outboxQueue == null ? "queue-unavailable" : "retry-policy-declined", survivors.map((route) => route.inbox), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? []));
10197
11882
  logger.error("Failed to send activity {activityId} to every gateway of {portableInbox} after {attempt} attempts; giving up:\n{error}", {
10198
11883
  ...logData,
10199
11884
  portableInbox: message.portableInbox,
@@ -10212,10 +11897,11 @@ var FederationImpl = class extends FederationBuilderImpl {
10212
11897
  portableInbox: message.portableInbox,
10213
11898
  error: lastError
10214
11899
  });
10215
- await outboxQueue.enqueue(retryMessage, {
11900
+ const enqueueOptions = {
10216
11901
  delay: clampNegativeDelay(delay),
10217
11902
  orderingKey: message.orderingKey
10218
- });
11903
+ };
11904
+ await (worker == null ? outboxQueue.enqueue(retryMessage, enqueueOptions) : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => outboxQueue.enqueue(retryMessage, enqueueOptions), survivors.flatMap((route) => preparationFailuresByRoute.get(route.inbox) ?? [])));
10219
11905
  this.metrics.recordQueueTaskEnqueued({
10220
11906
  role: "outbox",
10221
11907
  queue: outboxQueue,
@@ -10223,18 +11909,24 @@ var FederationImpl = class extends FederationBuilderImpl {
10223
11909
  }, retryMessage.attempt);
10224
11910
  recordOutboxActivity(this.meterProvider, "retried", retryMessage.activityType);
10225
11911
  }
10226
- async #listenInboxMessage(ctxData, message, span, onActivityType) {
11912
+ async #listenInboxMessage(ctxData, message, span, provenance, onActivityType) {
10227
11913
  const logger = getLogger([
10228
11914
  "fedify",
10229
11915
  "federation",
10230
11916
  "inbox"
10231
11917
  ]);
10232
11918
  const baseUrl = new URL(message.baseUrl);
10233
- let context = this.#createContext(baseUrl, ctxData);
10234
- if (message.identifier != null && this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(baseUrl, ctxData, { documentLoader: await context.getDocumentLoader({ identifier: message.identifier }) });
11919
+ let context = this.#createContext(baseUrl, ctxData, { [outboundProvenance]: provenance });
11920
+ if (message.identifier != null && this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(baseUrl, ctxData, {
11921
+ [outboundProvenance]: provenance,
11922
+ documentLoader: await context.getDocumentLoader({ identifier: message.identifier })
11923
+ });
10235
11924
  else if (message.identifier == null && this.sharedInboxKeyDispatcher != null) {
10236
11925
  const identity = await this.sharedInboxKeyDispatcher(context);
10237
- if (identity != null) context = this.#createContext(baseUrl, ctxData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
11926
+ if (identity != null) context = this.#createContext(baseUrl, ctxData, {
11927
+ [outboundProvenance]: provenance,
11928
+ documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
11929
+ });
10238
11930
  }
10239
11931
  await this._getTracer().startActiveSpan("activitypub.dispatch_inbox_listener", { kind: SpanKind.INTERNAL }, async (listenerSpan) => {
10240
11932
  let activity = null;
@@ -10657,6 +12349,7 @@ var FederationImpl = class extends FederationBuilderImpl {
10657
12349
  }
10658
12350
  const loaderOptions = this.#getLoaderOptions(url.origin);
10659
12351
  const ctxOptions = {
12352
+ [outboundProvenance]: opts[outboundProvenance],
10660
12353
  url,
10661
12354
  federation: this,
10662
12355
  data: contextData,
@@ -10688,22 +12381,37 @@ var FederationImpl = class extends FederationBuilderImpl {
10688
12381
  "federation",
10689
12382
  "outbox"
10690
12383
  ]);
10691
- const { immediate, collectionSync, orderingKey, context: ctx } = options;
10692
- if (activity.id == null) throw new TypeError("The activity to send must have an id.");
10693
- if (activity.actorId == null) throw new TypeError("The activity to send must have at least one actor property.");
10694
- else if (keys.length < 1) throw new TypeError("The keys must not be empty.");
10695
- const contextLoader = this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin));
10696
- const activityId = activity.id.href;
10697
- let rsaKey = null;
10698
- const ldSignable = !hasPortableActor(activity);
10699
- for (const { keyId, privateKey } of keys) {
10700
- validateCryptoKey(privateKey, "private");
10701
- if (ldSignable && rsaKey == null && privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && !isPortableKeyId(keyId)) rsaKey = {
10702
- keyId,
10703
- privateKey
12384
+ const { immediate, collectionSync, orderingKey, context: ctx, observation, worker, provenance = getOutboundProvenance(ctx) } = options;
12385
+ observation?.setPhase("validation");
12386
+ const validateActivity = () => {
12387
+ if (activity.id == null) throw new TypeError("The activity to send must have an id.");
12388
+ if (activity.actorId == null) throw new TypeError("The activity to send must have at least one actor property.");
12389
+ else if (keys.length < 1) throw new TypeError("The keys must not be empty.");
12390
+ return activity.id.href;
12391
+ };
12392
+ const activityId = worker == null ? validateActivity() : await worker.prepare("validation", validateActivity);
12393
+ observation?.setPhase("serialization");
12394
+ const createContextLoader = () => this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin));
12395
+ const contextLoader = worker == null ? createContextLoader() : await worker.prepare("serialization", createContextLoader);
12396
+ const resolveRsaKey = () => {
12397
+ const ldSignable = !hasPortableActor(activity);
12398
+ let rsaKey = null;
12399
+ for (const { keyId, privateKey } of keys) {
12400
+ validateCryptoKey(privateKey, "private");
12401
+ if (ldSignable && rsaKey == null && privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && !isPortableKeyId(keyId)) rsaKey = {
12402
+ keyId,
12403
+ privateKey
12404
+ };
12405
+ }
12406
+ return {
12407
+ rsaKey,
12408
+ ldSignable
10704
12409
  };
10705
- }
10706
- const signed = await signOutgoingActivity(activity, keys.map(({ keyId, privateKey }) => ({
12410
+ };
12411
+ observation?.setPhase("key-resolution");
12412
+ const { rsaKey, ldSignable } = worker == null ? resolveRsaKey() : await worker.prepare("key-resolution", resolveRsaKey);
12413
+ observation?.setPhase("signing");
12414
+ const signActivity = () => signOutgoingActivity(activity, keys.map(({ keyId, privateKey }) => ({
10707
12415
  verificationMethod: keyId,
10708
12416
  privateKey
10709
12417
  })), {
@@ -10711,14 +12419,23 @@ var FederationImpl = class extends FederationBuilderImpl {
10711
12419
  tracerProvider: this.tracerProvider,
10712
12420
  appendToExistingProofs: false
10713
12421
  });
12422
+ const signed = worker == null ? await signActivity() : await worker.prepare("signing", signActivity);
10714
12423
  activity = signed.activity;
10715
12424
  const { hasProof, proofCreated } = signed;
10716
- let jsonLd = !proofCreated && options.activityJsonLd != null ? options.activityJsonLd : await activity.toJsonLd({
10717
- format: "compact",
10718
- contextLoader
10719
- });
10720
- if (proofCreated || !hasProof || options.normalizeExistingProofs) jsonLd = await normalizeOutgoingActivityJsonLd(jsonLd, contextLoader, { preserveNestedSecuredDocuments: true });
10721
- assertSupportedCompoundProofShape(jsonLd, activityId);
12425
+ observation?.setPhase("serialization");
12426
+ const serializeActivity = async () => {
12427
+ let result = !proofCreated && options.activityJsonLd != null ? options.activityJsonLd : await activity.toJsonLd({
12428
+ format: "compact",
12429
+ contextLoader
12430
+ });
12431
+ if (proofCreated || !hasProof || options.normalizeExistingProofs) result = await normalizeOutgoingActivityJsonLd(result, contextLoader, { preserveNestedSecuredDocuments: true });
12432
+ return result;
12433
+ };
12434
+ let jsonLd = worker == null ? await serializeActivity() : await worker.prepare("serialization", serializeActivity);
12435
+ observation?.setPhase("validation");
12436
+ const validateProofShape = () => assertSupportedCompoundProofShape(jsonLd, activityId);
12437
+ if (worker == null) validateProofShape();
12438
+ else await worker.prepare("validation", validateProofShape);
10722
12439
  if (rsaKey == null && !ldSignable) logger.debug("The activity {activityId} is performed by a portable actor, so it is sent without a Linked Data signature.", { activityId });
10723
12440
  else if (rsaKey == null) logger.warn("No supported key found to create a Linked Data signature for the activity {activityId}. The activity will be sent without a Linked Data signature. In order to create a Linked Data signature, at least one RSASSA-PKCS1-v1_5 key must be provided.", {
10724
12441
  activityId,
@@ -10727,17 +12444,24 @@ var FederationImpl = class extends FederationBuilderImpl {
10727
12444
  privateKey: pair.privateKey
10728
12445
  }))
10729
12446
  });
10730
- else try {
10731
- jsonLd = await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, {
10732
- contextLoader,
10733
- tracerProvider: this.tracerProvider
10734
- });
10735
- } catch (error) {
10736
- if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error;
10737
- logger.warn("Failed to create a Linked Data signature for the activity {activityId}. The activity will be sent without a Linked Data signature.", {
10738
- activityId,
10739
- error
10740
- });
12447
+ else {
12448
+ const signLinkedData = async () => {
12449
+ try {
12450
+ observation?.setPhase("signing");
12451
+ return await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, {
12452
+ contextLoader,
12453
+ tracerProvider: this.tracerProvider
12454
+ });
12455
+ } catch (error) {
12456
+ if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error;
12457
+ logger.warn("Failed to create a Linked Data signature for the activity {activityId}. The activity will be sent without a Linked Data signature.", {
12458
+ activityId,
12459
+ error
12460
+ });
12461
+ return jsonLd;
12462
+ }
12463
+ };
12464
+ jsonLd = worker == null ? await signLinkedData() : await worker.prepare("signing", signLinkedData);
10741
12465
  }
10742
12466
  if (!hasProof) logger.warn("No supported key found to create a proof for the activity {activityId}. The activity will be sent without a proof. In order to create a proof, at least one Ed25519 key must be provided.", {
10743
12467
  activityId,
@@ -10746,6 +12470,7 @@ var FederationImpl = class extends FederationBuilderImpl {
10746
12470
  privateKey: pair.privateKey
10747
12471
  }))
10748
12472
  });
12473
+ observation?.setPhase("request-preparation");
10749
12474
  if (immediate || this.outboxQueue == null) {
10750
12475
  if (immediate) logger.debug("Sending activity immediately without queue since immediate option is set.", {
10751
12476
  activityId: activity.id.href,
@@ -10756,18 +12481,40 @@ var FederationImpl = class extends FederationBuilderImpl {
10756
12481
  activity: jsonLd
10757
12482
  });
10758
12483
  const promises = [];
10759
- for (const inbox in inboxes) promises.push(sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
10760
- allowPrivateAddress: this.allowPrivateAddress,
10761
- keys,
10762
- activity: jsonLd,
10763
- activityId: activity.id?.href,
10764
- activityType: getTypeId(activity).href,
10765
- sharedInbox: inboxes[inbox].sharedInbox,
10766
- headers: collectionSync == null ? void 0 : new Headers({ "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) }),
10767
- specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
10768
- meterProvider: this.meterProvider,
10769
- tracerProvider: this.tracerProvider
10770
- }));
12484
+ for (const inbox in inboxes) {
12485
+ const delivery = observation?.delivery(inbox);
12486
+ let sending;
12487
+ try {
12488
+ const createHeaders = collectionSync == null ? void 0 : async () => new Headers({ "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) });
12489
+ const sendActivityId = activity.id?.href;
12490
+ const activityType = getTypeId(activity).href;
12491
+ observation?.setActivity({
12492
+ ...sendActivityId == null ? {} : { id: sendActivityId },
12493
+ type: activityType
12494
+ });
12495
+ sending = sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
12496
+ attemptStarted: worker == null ? void 0 : (target) => worker.directAttemptStarted(target, observation?.continuation(inbox)?.outbound),
12497
+ observation: delivery,
12498
+ allowPrivateAddress: this.allowPrivateAddress,
12499
+ keys,
12500
+ activity: jsonLd,
12501
+ activityId: sendActivityId,
12502
+ activityType,
12503
+ sharedInbox: inboxes[inbox].sharedInbox,
12504
+ headers: createHeaders == null ? void 0 : await (worker == null ? createHeaders() : worker.prepare("request-preparation", createHeaders)),
12505
+ specDeterminer: new KvSpecDeterminer(this.kv, this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, { specTtl: this.httpMessageSignaturesSpecTtl }),
12506
+ meterProvider: this.meterProvider,
12507
+ tracerProvider: this.tracerProvider
12508
+ });
12509
+ } catch (error) {
12510
+ if (delivery != null) await delivery.failed(error);
12511
+ throw error;
12512
+ }
12513
+ if (observation != null) sending.catch(() => {});
12514
+ if (delivery != null) await delivery.handoff({ type: "direct" });
12515
+ promises.push(sending);
12516
+ }
12517
+ if (observation != null) await observation.prepared("delivery");
10771
12518
  await Promise.all(promises);
10772
12519
  return;
10773
12520
  }
@@ -10775,48 +12522,71 @@ var FederationImpl = class extends FederationBuilderImpl {
10775
12522
  activityId: activity.id.href,
10776
12523
  activity: jsonLd
10777
12524
  });
12525
+ observation?.setPhase("key-resolution");
10778
12526
  const keyJwkPairs = [];
10779
12527
  for (const { keyId, privateKey } of keys) {
10780
- const privateKeyJwk = await exportJwk(privateKey);
12528
+ const exportPrivateKey = () => exportJwk(privateKey);
12529
+ const privateKeyJwk = worker == null ? await exportPrivateKey() : await worker.prepare("key-export", exportPrivateKey);
10781
12530
  keyJwkPairs.push({
10782
12531
  keyId: keyId.href,
10783
12532
  privateKey: privateKeyJwk
10784
12533
  });
10785
12534
  }
10786
12535
  if (!this.manuallyStartQueue) this._startQueueInternal(ctx.data);
12536
+ observation?.setPhase("request-preparation");
10787
12537
  const carrier = {};
10788
- propagation.inject(context.active(), carrier);
12538
+ const injectTraceContext = () => propagation.inject(context.active(), carrier);
12539
+ if (worker == null) injectTraceContext();
12540
+ else await worker.prepare("request-preparation", injectTraceContext);
10789
12541
  const messages = [];
10790
- for (const inbox in inboxes) {
10791
- const { portableInbox, gatewayInboxes } = inboxes[inbox];
10792
- const messageOrderingKey = orderingKey == null ? void 0 : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`;
10793
- const message = {
10794
- type: "outbox",
10795
- id: crypto.randomUUID(),
10796
- baseUrl: ctx.origin,
10797
- keys: keyJwkPairs,
10798
- activity: jsonLd,
10799
- activityId: activity.id?.href,
10800
- activityType: getTypeId(activity).href,
10801
- inbox,
10802
- sharedInbox: inboxes[inbox].sharedInbox,
10803
- actorIds: [...inboxes[inbox].actorIds],
10804
- started: (/* @__PURE__ */ new Date()).toISOString(),
10805
- attempt: 0,
10806
- headers: collectionSync == null ? {} : { "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) },
10807
- orderingKey: messageOrderingKey,
10808
- ...getPortableMessageFields(portableInbox, gatewayInboxes),
10809
- traceContext: carrier
12542
+ for (const inbox in inboxes) try {
12543
+ const prepareMessage = async () => {
12544
+ const { portableInbox, gatewayInboxes } = inboxes[inbox];
12545
+ const messageOrderingKey = orderingKey == null ? void 0 : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`;
12546
+ const messageActivityId = activity.id?.href;
12547
+ const messageActivityType = getTypeId(activity).href;
12548
+ observation?.setActivity({
12549
+ ...messageActivityId == null ? {} : { id: messageActivityId },
12550
+ type: messageActivityType
12551
+ });
12552
+ return {
12553
+ message: {
12554
+ type: "outbox",
12555
+ ...serializeProvenance(provenance),
12556
+ id: crypto.randomUUID(),
12557
+ baseUrl: ctx.origin,
12558
+ keys: keyJwkPairs,
12559
+ activity: jsonLd,
12560
+ activityId: messageActivityId,
12561
+ activityType: messageActivityType,
12562
+ inbox,
12563
+ ...observation?.continuation(inbox),
12564
+ sharedInbox: inboxes[inbox].sharedInbox,
12565
+ actorIds: [...inboxes[inbox].actorIds],
12566
+ started: (/* @__PURE__ */ new Date()).toISOString(),
12567
+ attempt: 0,
12568
+ headers: collectionSync == null ? {} : { "Collection-Synchronization": await buildCollectionSynchronizationHeader(collectionSync, inboxes[inbox].actorIds) },
12569
+ orderingKey: messageOrderingKey,
12570
+ ...getPortableMessageFields(portableInbox, gatewayInboxes),
12571
+ traceContext: carrier
12572
+ },
12573
+ orderingKey: messageOrderingKey
12574
+ };
10810
12575
  };
10811
- messages.push({
10812
- message,
10813
- orderingKey: messageOrderingKey
10814
- });
12576
+ messages.push(await (worker == null ? prepareMessage() : worker.prepare("request-preparation", prepareMessage)));
12577
+ } catch (error) {
12578
+ if (observation != null) await observation.delivery(inbox)?.failed(error);
12579
+ throw error;
10815
12580
  }
12581
+ if (observation != null) await observation.prepared("delivery");
10816
12582
  const { outboxQueue } = this;
10817
12583
  if (outboxQueue.enqueueMany == null || orderingKey != null) {
10818
12584
  const errors = (await Promise.allSettled(messages.map(async (m) => {
10819
- await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
12585
+ const enqueue = () => worker == null ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) : worker.enqueueChildren([m.message], () => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }));
12586
+ await (observation == null ? enqueue() : observation.enqueue([{
12587
+ inbox: m.message.inbox,
12588
+ queueMessageId: m.message.id
12589
+ }], "outbox", enqueue));
10820
12590
  recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message);
10821
12591
  }))).filter((r) => r.status === "rejected").map((r) => r.reason);
10822
12592
  if (errors.length > 0) {
@@ -10829,7 +12599,11 @@ var FederationImpl = class extends FederationBuilderImpl {
10829
12599
  }
10830
12600
  } else {
10831
12601
  try {
10832
- await outboxQueue.enqueueMany(messages.map((m) => m.message));
12602
+ const enqueue = () => worker == null ? outboxQueue.enqueueMany(messages.map((m) => m.message)) : worker.enqueueChildren(messages.map((m) => m.message), () => outboxQueue.enqueueMany(messages.map((m) => m.message)));
12603
+ await (observation == null ? enqueue() : observation.enqueue(messages.map(({ message }) => ({
12604
+ inbox: message.inbox,
12605
+ queueMessageId: message.id
12606
+ })), "outbox", enqueue));
10833
12607
  } catch (error) {
10834
12608
  logger.error("Failed to enqueue activity {activityId} to send later: {error}", {
10835
12609
  activityId: activity.id.href,
@@ -10841,9 +12615,16 @@ var FederationImpl = class extends FederationBuilderImpl {
10841
12615
  }
10842
12616
  }
10843
12617
  fetch(request, options) {
10844
- return withContext({ requestId: getRequestId(request) }, async () => {
12618
+ const requestId = getRequestId(request);
12619
+ return withContext({
12620
+ requestId,
12621
+ inboxRequestId: void 0,
12622
+ dispatchId: void 0,
12623
+ sourceInboxRequestId: void 0,
12624
+ sourceRequestId: void 0
12625
+ }, async () => {
10845
12626
  const tracer = this._getTracer();
10846
- const metricState = {};
12627
+ const metricState = { requestId };
10847
12628
  const metricStart = performance.now();
10848
12629
  return await tracer.startActiveSpan(request.method, {
10849
12630
  kind: SpanKind.SERVER,
@@ -10888,7 +12669,8 @@ var FederationImpl = class extends FederationBuilderImpl {
10888
12669
  logger.error("An error occurred while serving request {method} {url}: {error}", {
10889
12670
  method: request.method,
10890
12671
  url: request.url,
10891
- error
12672
+ error,
12673
+ ...metricState.inboxRequest == null ? {} : { inboxRequestId: metricState.inboxRequest.inboxRequestId }
10892
12674
  });
10893
12675
  throw error;
10894
12676
  }
@@ -10908,6 +12690,7 @@ var FederationImpl = class extends FederationBuilderImpl {
10908
12690
  const url = new URL(request.url);
10909
12691
  const logTpl = "{method} {path}: {status}";
10910
12692
  const values = {
12693
+ ...metricState.inboxRequest == null ? {} : { inboxRequestId: metricState.inboxRequest.inboxRequestId },
10911
12694
  method: request.method,
10912
12695
  path: `${url.pathname}${url.search}`,
10913
12696
  url: request.url,
@@ -10960,9 +12743,11 @@ var FederationImpl = class extends FederationBuilderImpl {
10960
12743
  metricState.endpoint = getEndpointCategory(route.name);
10961
12744
  span.updateName(`${request.method} ${route.template}`);
10962
12745
  const routeName = route.name.replace(/:.*$/, "");
12746
+ const source = request.method === "POST" && (routeName === "inbox" || routeName === "sharedInbox") ? this.#createInboxRequestSource(request, metricState) : void 0;
12747
+ const provenance = globalThis.Object.freeze(source == null ? { originatingRequestId: metricState.requestId } : { inboxRequest: source });
10963
12748
  let context;
10964
12749
  try {
10965
- context = this.#createContext(request, contextData);
12750
+ context = withContext(source == null ? {} : { inboxRequestId: source.inboxRequestId }, () => this.#createContext(request, contextData, { [outboundProvenance]: provenance }));
10966
12751
  } catch (error) {
10967
12752
  if (request.method !== "POST" || routeName !== "inbox" && routeName !== "sharedInbox") throw error;
10968
12753
  return await this.#reportInboxPreparationError(request, contextData, {
@@ -11033,13 +12818,19 @@ var FederationImpl = class extends FederationBuilderImpl {
11033
12818
  const observation = new InboxObservation({
11034
12819
  kind: routeName === "inbox" ? "personal" : "shared",
11035
12820
  recipient
11036
- });
12821
+ }, source);
11037
12822
  this.#setInboxCompletion(metricState, observation, () => context);
11038
- return await observation.run(async () => {
11039
- if (routeName === "inbox") context = this.#createContext(request, contextData, { documentLoader: await context.getDocumentLoader({ identifier: recipient }) });
12823
+ return await withContext({ inboxRequestId: source.inboxRequestId }, () => observation.run(async () => {
12824
+ if (routeName === "inbox") context = this.#createContext(request, contextData, {
12825
+ [outboundProvenance]: getOutboundProvenance(context),
12826
+ documentLoader: await context.getDocumentLoader({ identifier: recipient })
12827
+ });
11040
12828
  else if (this.sharedInboxKeyDispatcher != null) {
11041
12829
  const identity = await this.sharedInboxKeyDispatcher(context);
11042
- if (identity != null) context = this.#createContext(request, contextData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
12830
+ if (identity != null) context = this.#createContext(request, contextData, {
12831
+ [outboundProvenance]: getOutboundProvenance(context),
12832
+ documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
12833
+ });
11043
12834
  }
11044
12835
  return await this.#handleInbox(request, {
11045
12836
  recipient,
@@ -11048,13 +12839,16 @@ var FederationImpl = class extends FederationBuilderImpl {
11048
12839
  onNotFound,
11049
12840
  observation
11050
12841
  });
11051
- }, span);
12842
+ }, span));
11052
12843
  }
11053
12844
  switch (routeName) {
11054
12845
  case "actor":
11055
12846
  case "actorAlias": {
11056
12847
  const identifier = route.name.startsWith("actorAlias:") ? route.name.substring(11) : route.values.identifier;
11057
- context = this.#createContext(request, contextData, { invokedFromActorDispatcher: { identifier } });
12848
+ context = this.#createContext(request, contextData, {
12849
+ [outboundProvenance]: getOutboundProvenance(context),
12850
+ invokedFromActorDispatcher: { identifier }
12851
+ });
11058
12852
  return await handleActor(request, {
11059
12853
  identifier,
11060
12854
  context,
@@ -11068,10 +12862,13 @@ var FederationImpl = class extends FederationBuilderImpl {
11068
12862
  const typeId = route.name.replace(/^object:/, "");
11069
12863
  const callbacks = this.objectCallbacks[typeId];
11070
12864
  const cls = this.objectTypeIds[typeId];
11071
- context = this.#createContext(request, contextData, { invokedFromObjectDispatcher: {
11072
- cls,
11073
- values: route.values
11074
- } });
12865
+ context = this.#createContext(request, contextData, {
12866
+ [outboundProvenance]: getOutboundProvenance(context),
12867
+ invokedFromObjectDispatcher: {
12868
+ cls,
12869
+ values: route.values
12870
+ }
12871
+ });
11075
12872
  return await handleObject(request, {
11076
12873
  values: route.values,
11077
12874
  context,
@@ -11129,7 +12926,10 @@ var FederationImpl = class extends FederationBuilderImpl {
11129
12926
  case "sharedInbox":
11130
12927
  if (this.sharedInboxKeyDispatcher != null) {
11131
12928
  const identity = await this.sharedInboxKeyDispatcher(context);
11132
- if (identity != null) context = this.#createContext(request, contextData, { documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity) });
12929
+ if (identity != null) context = this.#createContext(request, contextData, {
12930
+ [outboundProvenance]: getOutboundProvenance(context),
12931
+ documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity)
12932
+ });
11133
12933
  }
11134
12934
  return await this.#handleInbox(request, {
11135
12935
  recipient: route.values.identifier ?? null,
@@ -11273,14 +13073,17 @@ var FederationImpl = class extends FederationBuilderImpl {
11273
13073
  });
11274
13074
  return response;
11275
13075
  }
13076
+ const provenance = globalThis.Object.freeze({ originatingRequestId: metricState.requestId });
11276
13077
  if (collectionRoute != null) return await this.#fetchPortableCollection(request, portable, route, {
11277
13078
  onNotFound,
11278
13079
  onUnauthorized,
11279
- contextData
13080
+ contextData,
13081
+ provenance
11280
13082
  });
11281
13083
  if (isActor) {
11282
13084
  const identifier = route.name.startsWith("actorAlias:") ? route.name.substring(11) : route.values.identifier;
11283
13085
  const context = this.#createContext(request, contextData, {
13086
+ [outboundProvenance]: provenance,
11284
13087
  invokedFromActorDispatcher: { identifier },
11285
13088
  portableRequest: portable.portableRequest
11286
13089
  });
@@ -11301,6 +13104,7 @@ var FederationImpl = class extends FederationBuilderImpl {
11301
13104
  const callbacks = this.objectCallbacks[typeId];
11302
13105
  const cls = this.objectTypeIds[typeId];
11303
13106
  const context = this.#createContext(request, contextData, {
13107
+ [outboundProvenance]: provenance,
11304
13108
  invokedFromObjectDispatcher: {
11305
13109
  cls,
11306
13110
  values: route.values
@@ -11318,7 +13122,7 @@ var FederationImpl = class extends FederationBuilderImpl {
11318
13122
  onNotFound
11319
13123
  });
11320
13124
  }
11321
- async #fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData }) {
13125
+ async #fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData, provenance }) {
11322
13126
  const logger = getLogger([
11323
13127
  "fedify",
11324
13128
  "federation",
@@ -11326,8 +13130,11 @@ var FederationImpl = class extends FederationBuilderImpl {
11326
13130
  ]);
11327
13131
  const url = new URL(request.url);
11328
13132
  const { authority } = portable.portableRequest;
11329
- const ownerContext = this.#createContext(request, contextData);
11330
- const context = this.#createContext(request, contextData, { portableRequest: portable.portableRequest });
13133
+ const ownerContext = this.#createContext(request, contextData, { [outboundProvenance]: provenance });
13134
+ const context = this.#createContext(request, contextData, {
13135
+ [outboundProvenance]: provenance,
13136
+ portableRequest: portable.portableRequest
13137
+ });
11331
13138
  const reject = async (reason, identifier) => {
11332
13139
  logger.debug("Not serving the portable collection {collection} of the actor {identifier}: {reason}.", {
11333
13140
  collection: portable.canonicalId,
@@ -11420,6 +13227,11 @@ var FederationImpl = class extends FederationBuilderImpl {
11420
13227
  }
11421
13228
  return await onNotFound(request);
11422
13229
  }
13230
+ #createInboxRequestSource(request, metricState) {
13231
+ const source = createInboxRequestSource(request, metricState.requestId ?? getRequestId(request), this.origin?.webOrigin);
13232
+ metricState.inboxRequest = source;
13233
+ return source;
13234
+ }
11423
13235
  #setInboxCompletion(metricState, observation, getContext) {
11424
13236
  const handler = this.inboxRequestFinishedHandler;
11425
13237
  metricState.inboxCompletion = {
@@ -11451,6 +13263,7 @@ var FederationImpl = class extends FederationBuilderImpl {
11451
13263
  async #reportInboxPreparationError(request, contextData, inbox, error, span, metricState) {
11452
13264
  const unavailableLoader = () => Promise.reject(error);
11453
13265
  const context = new RequestContextImpl({
13266
+ [outboundProvenance]: globalThis.Object.freeze({ inboxRequest: metricState.inboxRequest }),
11454
13267
  url: new URL(request.url),
11455
13268
  request,
11456
13269
  federation: this,
@@ -11458,7 +13271,7 @@ var FederationImpl = class extends FederationBuilderImpl {
11458
13271
  documentLoader: unavailableLoader,
11459
13272
  contextLoader: unavailableLoader
11460
13273
  });
11461
- const observation = new InboxObservation(inbox);
13274
+ const observation = new InboxObservation(inbox, metricState.inboxRequest);
11462
13275
  this.#setInboxCompletion(metricState, observation, () => context);
11463
13276
  return await observation.run(() => Promise.reject(error), span);
11464
13277
  }
@@ -11516,9 +13329,10 @@ var FederationImpl = class extends FederationBuilderImpl {
11516
13329
  metricState.endpoint = "inbox";
11517
13330
  span.updateName(`${request.method} ${metricState.routeTemplate}`);
11518
13331
  const identifier = route.values.identifier;
13332
+ const source = this.#createInboxRequestSource(request, metricState);
11519
13333
  let context;
11520
13334
  try {
11521
- context = this.#createContext(request, contextData);
13335
+ context = withContext({ inboxRequestId: source.inboxRequestId }, () => this.#createContext(request, contextData, { [outboundProvenance]: globalThis.Object.freeze({ inboxRequest: source }) }));
11522
13336
  } catch (error) {
11523
13337
  return await this.#reportInboxPreparationError(request, contextData, {
11524
13338
  kind: "portable",
@@ -11528,9 +13342,9 @@ var FederationImpl = class extends FederationBuilderImpl {
11528
13342
  const observation = new InboxObservation({
11529
13343
  kind: "portable",
11530
13344
  recipient: identifier
11531
- });
13345
+ }, source);
11532
13346
  this.#setInboxCompletion(metricState, observation, () => context);
11533
- return await observation.run(async () => {
13347
+ return await withContext({ inboxRequestId: source.inboxRequestId }, () => observation.run(async () => {
11534
13348
  const actor = this.actorCallbacks?.dispatcher == null ? null : await this.actorCallbacks.dispatcher(context, identifier);
11535
13349
  const localOrigin = context.canonicalOrigin;
11536
13350
  const resolution = resolvePortableInboxRecipient(actor == null || actor instanceof Tombstone ? null : actor, {
@@ -11547,7 +13361,10 @@ var FederationImpl = class extends FederationBuilderImpl {
11547
13361
  return await onNotFound(request);
11548
13362
  }
11549
13363
  const { recipient } = resolution;
11550
- if (this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(request, contextData, { documentLoader: await context.getDocumentLoader({ identifier }) });
13364
+ if (this.actorCallbacks?.keyPairsDispatcher != null) context = this.#createContext(request, contextData, {
13365
+ [outboundProvenance]: getOutboundProvenance(context),
13366
+ documentLoader: await context.getDocumentLoader({ identifier })
13367
+ });
11551
13368
  const excludedOrigins = [localOrigin, new URL(request.url).origin];
11552
13369
  return await this.#handleInbox(request, {
11553
13370
  recipient: identifier,
@@ -11557,16 +13374,25 @@ var FederationImpl = class extends FederationBuilderImpl {
11557
13374
  onNotFound,
11558
13375
  portableInbox: {
11559
13376
  recipient,
11560
- forward: async (activity, activityId, activityType, signatures) => {
11561
- if (this.portableInboxForwarding.maxTargets < 1) return;
13377
+ forward: (activity, activityId, activityType, signatures) => observeOutboundBroadcast(context, this.outboundObservers, async (observation) => {
13378
+ observation?.setActivity({
13379
+ ...activityId == null ? {} : { id: activityId.href },
13380
+ type: activityType
13381
+ });
13382
+ if (this.portableInboxForwarding.maxTargets < 1) {
13383
+ if (observation != null) await observation.skipped("forwarding-disabled");
13384
+ return;
13385
+ }
11562
13386
  if (this.kv.cas == null) {
11563
13387
  if (!this.#portableInboxForwardingWarned) {
11564
13388
  this.#portableInboxForwardingWarned = true;
11565
13389
  logger.warn("Activities delivered to FEP-ef61 portable inboxes are not forwarded to the other gateways, as the key–value store does not support compare-and-swap (KvStore.cas()), which is needed to forward each activity at most once.");
11566
13390
  }
13391
+ if (observation != null) await observation.skipped("forwarding-unavailable");
11567
13392
  return;
11568
13393
  }
11569
13394
  await forwardPortableInboxActivity({
13395
+ provenance: getOutboundProvenance(context),
11570
13396
  recipient,
11571
13397
  activity,
11572
13398
  activityId,
@@ -11584,11 +13410,11 @@ var FederationImpl = class extends FederationBuilderImpl {
11584
13410
  options: this.portableInboxForwarding,
11585
13411
  meterProvider: this.meterProvider,
11586
13412
  tracerProvider: this.tracerProvider
11587
- });
11588
- }
13413
+ }, observation);
13414
+ }, "portable-replication")
11589
13415
  }
11590
13416
  });
11591
- }, span);
13417
+ }, span));
11592
13418
  }
11593
13419
  /**
11594
13420
  * Gets this server's gateway key pairs for the portable actor that owns
@@ -11658,7 +13484,7 @@ var FederationImpl = class extends FederationBuilderImpl {
11658
13484
  headers: { "Content-Type": "text/plain; charset=utf-8" }
11659
13485
  });
11660
13486
  }
11661
- const context = this.#createContext(request, contextData);
13487
+ const context = this.#createContext(request, contextData, { [outboundProvenance]: globalThis.Object.freeze({ originatingRequestId: metricState.requestId }) });
11662
13488
  const response = await this.hashlinkMediaDispatcher(context, media.media);
11663
13489
  if (response == null) return await onNotFound(request);
11664
13490
  return await discardBodyForHead(request, response, false);
@@ -11800,8 +13626,9 @@ var ContextImpl = class ContextImpl {
11800
13626
  verifyPortableObject;
11801
13627
  invokedFromActorKeyPairsDispatcher;
11802
13628
  #codec;
11803
- constructor({ url, federation, data, documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher }) {
13629
+ constructor({ url, federation, data, documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher, [outboundProvenance]: provenance = emptyProvenance }) {
11804
13630
  this.url = url;
13631
+ globalThis.Object.assign(this, { [outboundProvenance]: provenance });
11805
13632
  this.federation = federation;
11806
13633
  this.data = data;
11807
13634
  this.documentLoader = documentLoader;
@@ -11823,6 +13650,7 @@ var ContextImpl = class ContextImpl {
11823
13650
  }
11824
13651
  clone(data) {
11825
13652
  return new ContextImpl({
13653
+ [outboundProvenance]: getOutboundProvenance(this),
11826
13654
  url: this.url,
11827
13655
  federation: this.federation,
11828
13656
  data,
@@ -11833,6 +13661,7 @@ var ContextImpl = class ContextImpl {
11833
13661
  }
11834
13662
  toInboxContext(recipient, activity, activityId, activityType, metadata) {
11835
13663
  return new InboxContextImpl(recipient, activity, activityId, activityType, {
13664
+ [outboundProvenance]: getOutboundProvenance(this),
11836
13665
  url: this.url,
11837
13666
  federation: this.federation,
11838
13667
  data: this.data,
@@ -11843,6 +13672,7 @@ var ContextImpl = class ContextImpl {
11843
13672
  }
11844
13673
  toOutboxContext(identifier, activity, activityId, activityType) {
11845
13674
  return new OutboxContextImpl(identifier, activity, activityId, activityType, {
13675
+ [outboundProvenance]: getOutboundProvenance(this),
11846
13676
  url: this.url,
11847
13677
  federation: this.federation,
11848
13678
  data: this.data,
@@ -12232,36 +14062,49 @@ var ContextImpl = class ContextImpl {
12232
14062
  await this.#enqueueTasks(task, payloads, options);
12233
14063
  }
12234
14064
  sendActivity(sender, recipients, activity, options = {}) {
12235
- return this.tracerProvider.getTracer(name, version).startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
12236
- kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
12237
- attributes: {
12238
- "activitypub.activity.type": getTypeId(activity).href,
12239
- "activitypub.activity.to": activity.toIds.map((to) => to.href),
12240
- "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
12241
- "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
12242
- "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
12243
- }
12244
- }, async (span) => {
12245
- try {
12246
- if (activity.id != null) span.setAttribute("activitypub.activity.id", activity.id.href);
12247
- await this.sendActivityInternal(sender, recipients, activity, options, span);
12248
- } catch (e) {
12249
- span.setStatus({
12250
- code: SpanStatusCode.ERROR,
12251
- message: String(e)
12252
- });
12253
- throw e;
12254
- } finally {
12255
- span.end();
12256
- }
12257
- });
14065
+ return observeOutboundBroadcast(this, this.federation.outboundObservers, (observation, provenance) => {
14066
+ const tracer = this.tracerProvider.getTracer(name, version);
14067
+ const parentSpanContext = trace.getSpanContext(context.active());
14068
+ const activityType = getTypeId(activity).href;
14069
+ observation?.setActivity({ type: activityType });
14070
+ return tracer.startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
14071
+ kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
14072
+ attributes: {
14073
+ "activitypub.activity.type": activityType,
14074
+ "activitypub.activity.to": activity.toIds.map((to) => to.href),
14075
+ "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
14076
+ "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
14077
+ "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
14078
+ }
14079
+ }, async (span) => {
14080
+ observation?.setSpan(span, parentSpanContext);
14081
+ try {
14082
+ const activityId = activity.id?.href;
14083
+ if (activityId != null) span.setAttribute("activitypub.activity.id", activityId);
14084
+ observation?.setActivity({
14085
+ type: activityType,
14086
+ ...activityId == null ? {} : { id: activityId }
14087
+ });
14088
+ await this.sendActivityInternal(sender, recipients, activity, options, span, observation, provenance);
14089
+ } catch (e) {
14090
+ span.setStatus({
14091
+ code: SpanStatusCode.ERROR,
14092
+ message: String(e)
14093
+ });
14094
+ throw e;
14095
+ } finally {
14096
+ span.end();
14097
+ }
14098
+ });
14099
+ }, "send", options.dispatchId);
12258
14100
  }
12259
- async sendActivityInternal(sender, recipients, activity, options, span) {
14101
+ async sendActivityInternal(sender, recipients, activity, options, span, observation, provenance = getOutboundProvenance(this)) {
12260
14102
  const logger = getLogger([
12261
14103
  "fedify",
12262
14104
  "federation",
12263
14105
  "outbox"
12264
14106
  ]);
14107
+ observation?.setPhase("sender-resolution");
12265
14108
  let keys;
12266
14109
  let identifier = null;
12267
14110
  let actorKeyPairs = null;
@@ -12277,6 +14120,7 @@ var ContextImpl = class ContextImpl {
12277
14120
  }
12278
14121
  }
12279
14122
  span.setAttribute("fedify.actor.identifier", identifier);
14123
+ observation?.setPhase("key-resolution");
12280
14124
  if (this.federation.actorCallbacks?.keyPairsDispatcher == null) throw new Error("No actor key pairs dispatcher registered.");
12281
14125
  actorKeyPairs = await this.getActorKeyPairs(identifier);
12282
14126
  if (actorKeyPairs.length < 1) throw new Error(`No key pair found for actor ${JSON.stringify(identifier)}.`);
@@ -12285,11 +14129,14 @@ var ContextImpl = class ContextImpl {
12285
14129
  privateKey: kp.privateKey
12286
14130
  }));
12287
14131
  } else if (Array.isArray(sender)) {
14132
+ observation?.setPhase("key-resolution");
12288
14133
  if (sender.length < 1) throw new Error("The sender's key pairs are empty.");
12289
14134
  keys = sender;
12290
14135
  } else keys = [sender];
12291
14136
  if (keys.length < 1) throw new TypeError("The sender's keys must not be empty.");
14137
+ observation?.setPhase("key-resolution");
12292
14138
  for (const { privateKey } of keys) validateCryptoKey(privateKey, "private");
14139
+ observation?.setPhase("recipient-resolution");
12293
14140
  let expandedRecipients;
12294
14141
  let collectionSync;
12295
14142
  if (Array.isArray(recipients)) expandedRecipients = recipients;
@@ -12305,14 +14152,34 @@ var ContextImpl = class ContextImpl {
12305
14152
  } else expandedRecipients = [recipients];
12306
14153
  const opts = {
12307
14154
  context: this,
14155
+ observation,
14156
+ provenance,
12308
14157
  orderingKey: options.orderingKey,
12309
14158
  collectionSync,
12310
14159
  immediate: options.immediate,
12311
14160
  normalizeExistingProofs: options.normalizeExistingProofs
12312
14161
  };
12313
14162
  span.setAttribute("activitypub.inboxes", expandedRecipients.length);
14163
+ observation?.setPhase("transformation");
14164
+ if (this.federation.activityTransformers.length > 0) observation?.setActivity({});
12314
14165
  for (const activityTransformer of this.federation.activityTransformers) activity = activityTransformer(activity, this);
12315
- span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? "");
14166
+ let transformedActivityId;
14167
+ try {
14168
+ transformedActivityId = activity?.id?.href;
14169
+ } finally {
14170
+ if (observation != null) {
14171
+ let activityType;
14172
+ try {
14173
+ activityType = getTypeId(activity)?.href;
14174
+ } catch {}
14175
+ observation.setActivity({
14176
+ ...transformedActivityId == null ? {} : { id: transformedActivityId },
14177
+ ...activityType == null ? {} : { type: activityType }
14178
+ });
14179
+ }
14180
+ }
14181
+ span?.setAttribute("activitypub.activity.id", transformedActivityId ?? "");
14182
+ observation?.setPhase("validation");
12316
14183
  if (activity.actorId == null) {
12317
14184
  logger.error("Activity {activityId} to send does not have an actor.", {
12318
14185
  activity,
@@ -12320,6 +14187,7 @@ var ContextImpl = class ContextImpl {
12320
14187
  });
12321
14188
  throw new TypeError("The activity to send must have at least one actor property.");
12322
14189
  }
14190
+ observation?.setPhase("signing");
12323
14191
  const { activity: signedActivity, proofCreated } = await signOutgoingActivity(activity, actorKeyPairs == null ? keys.map(({ keyId, privateKey }) => ({
12324
14192
  verificationMethod: keyId,
12325
14193
  privateKey
@@ -12332,16 +14200,19 @@ var ContextImpl = class ContextImpl {
12332
14200
  appendToExistingProofs: actorKeyPairs != null
12333
14201
  });
12334
14202
  activity = signedActivity;
14203
+ observation?.setPhase("destination-planning");
12335
14204
  const inboxes = extractInboxes({
12336
14205
  recipients: expandedRecipients,
12337
14206
  preferSharedInbox: options.preferSharedInbox,
12338
14207
  excludeBaseUris: options.excludeBaseUris
12339
14208
  });
14209
+ if (observation != null) await observation.planDestinations(inboxes);
12340
14210
  if (globalThis.Object.keys(inboxes).length < 1) {
12341
14211
  logger.debug("No inboxes found for activity {activityId}.", {
12342
14212
  activityId: activity.id?.href,
12343
14213
  activity
12344
14214
  });
14215
+ if (observation != null) await observation.skipped();
12345
14216
  return false;
12346
14217
  }
12347
14218
  logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", {
@@ -12349,6 +14220,7 @@ var ContextImpl = class ContextImpl {
12349
14220
  activityId: activity.id?.href,
12350
14221
  activity
12351
14222
  });
14223
+ observation?.setPhase("serialization");
12352
14224
  const activityJsonLd = await activity.toJsonLd({
12353
14225
  format: "compact",
12354
14226
  contextLoader: this.contextLoader
@@ -12369,33 +14241,48 @@ var ContextImpl = class ContextImpl {
12369
14241
  });
12370
14242
  return true;
12371
14243
  }
14244
+ observation?.setPhase("validation");
12372
14245
  assertSupportedCompoundProofShape(activityJsonLd, activity.id?.href);
14246
+ observation?.setPhase("key-resolution");
12373
14247
  const keyJwkPairs = await Promise.all(keys.map(async ({ keyId, privateKey }) => ({
12374
14248
  keyId: keyId.href,
12375
14249
  privateKey: await exportJwk(privateKey)
12376
14250
  })));
14251
+ observation?.setPhase("request-preparation");
12377
14252
  const carrier = {};
12378
14253
  propagation.inject(context.active(), carrier);
14254
+ const activityId = activity.id?.href;
14255
+ const activityType = getTypeId(activity).href;
14256
+ observation?.setActivity({
14257
+ ...activityId == null ? {} : { id: activityId },
14258
+ type: activityType
14259
+ });
12379
14260
  const message = {
12380
14261
  type: "fanout",
14262
+ ...serializeProvenance(provenance),
12381
14263
  id: crypto.randomUUID(),
12382
14264
  baseUrl: this.origin,
12383
14265
  keys: keyJwkPairs,
12384
14266
  inboxes: globalThis.Object.fromEntries(globalThis.Object.entries(inboxes).map(([k, { actorIds, sharedInbox, portableInbox, gatewayInboxes }]) => [k, {
12385
14267
  actorIds: [...actorIds],
14268
+ ...observation?.continuation(k),
12386
14269
  sharedInbox,
12387
14270
  ...getPortableMessageFields(portableInbox, gatewayInboxes)
12388
14271
  }])),
12389
14272
  activity: activityJsonLd,
12390
- activityId: activity.id?.href,
12391
- activityType: getTypeId(activity).href,
14273
+ activityId,
14274
+ activityType,
12392
14275
  collectionSync: opts.collectionSync,
12393
14276
  orderingKey: options.orderingKey,
12394
14277
  normalizeExistingProofs: proofCreated || options.normalizeExistingProofs,
12395
14278
  traceContext: carrier
12396
14279
  };
12397
14280
  if (!this.federation.manuallyStartQueue) this.federation._startQueueInternal(this.data);
12398
- await this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey });
14281
+ if (observation != null) await observation.prepared("fanout");
14282
+ await (observation == null ? this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey }) : observation.enqueue(globalThis.Object.keys(inboxes).map((inbox) => ({
14283
+ inbox,
14284
+ queueMessageId: message.id
14285
+ })), "fanout", () => this.federation.fanoutQueue.enqueue(message, { orderingKey: options.orderingKey })));
12399
14286
  getFederationMetrics(this.federation.meterProvider).recordQueueTaskEnqueued({
12400
14287
  role: "fanout",
12401
14288
  queue: this.federation.fanoutQueue,
@@ -12656,6 +14543,7 @@ var RequestContextImpl = class RequestContextImpl extends ContextImpl {
12656
14543
  }
12657
14544
  clone(data) {
12658
14545
  return new RequestContextImpl({
14546
+ [outboundProvenance]: getOutboundProvenance(this),
12659
14547
  url: this.url,
12660
14548
  federation: this.federation,
12661
14549
  data,
@@ -12827,30 +14715,40 @@ var RequestContextImpl = class RequestContextImpl extends ContextImpl {
12827
14715
  }
12828
14716
  };
12829
14717
  function forwardActivity(ctx, loggerCategory, forwarder, recipients, options) {
12830
- return ctx.tracerProvider.getTracer(name, version).startActiveSpan(ctx.federation.outboxQueue == null || options?.immediate ? `activitypub.${loggerCategory}` : "activitypub.fanout", {
12831
- kind: ctx.federation.outboxQueue == null || options?.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
12832
- attributes: { "activitypub.activity.type": ctx.activityType }
12833
- }, async (span) => {
12834
- try {
12835
- if (ctx.activityId != null) span.setAttribute("activitypub.activity.id", ctx.activityId);
12836
- return await forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options);
12837
- } catch (e) {
12838
- span.setStatus({
12839
- code: SpanStatusCode.ERROR,
12840
- message: String(e)
12841
- });
12842
- throw e;
12843
- } finally {
12844
- span.end();
12845
- }
12846
- });
14718
+ return observeOutboundBroadcast(ctx, ctx.federation.outboundObservers, (observation, provenance) => {
14719
+ observation?.setActivity({
14720
+ ...ctx.activityId == null ? {} : { id: ctx.activityId },
14721
+ type: ctx.activityType
14722
+ });
14723
+ const tracer = ctx.tracerProvider.getTracer(name, version);
14724
+ const parentSpanContext = trace.getSpanContext(context.active());
14725
+ return tracer.startActiveSpan(ctx.federation.outboxQueue == null || options?.immediate ? `activitypub.${loggerCategory}` : "activitypub.fanout", {
14726
+ kind: ctx.federation.outboxQueue == null || options?.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
14727
+ attributes: { "activitypub.activity.type": ctx.activityType }
14728
+ }, async (span) => {
14729
+ observation?.setSpan(span, parentSpanContext);
14730
+ try {
14731
+ if (ctx.activityId != null) span.setAttribute("activitypub.activity.id", ctx.activityId);
14732
+ return await forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options, observation, provenance);
14733
+ } catch (e) {
14734
+ span.setStatus({
14735
+ code: SpanStatusCode.ERROR,
14736
+ message: String(e)
14737
+ });
14738
+ throw e;
14739
+ } finally {
14740
+ span.end();
14741
+ }
14742
+ });
14743
+ }, "forward", options?.dispatchId);
12847
14744
  }
12848
- async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options) {
14745
+ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipients, options, observation, provenance = getOutboundProvenance(ctx)) {
12849
14746
  const logger = getLogger([
12850
14747
  "fedify",
12851
14748
  "federation",
12852
14749
  loggerCategory
12853
14750
  ]);
14751
+ observation?.setPhase("sender-resolution");
12854
14752
  let keys;
12855
14753
  let identifier = null;
12856
14754
  if ("identifier" in forwarder || "username" in forwarder) {
@@ -12864,6 +14762,7 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12864
14762
  identifier = mapped;
12865
14763
  }
12866
14764
  }
14765
+ observation?.setPhase("key-resolution");
12867
14766
  const actorKeyPairs = await ctx.getActorKeyPairs(identifier);
12868
14767
  if (actorKeyPairs.length < 1) throw new Error(`No key pair found for actor ${JSON.stringify(identifier)}.`);
12869
14768
  keys = actorKeyPairs.map((kp) => ({
@@ -12871,12 +14770,17 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12871
14770
  privateKey: kp.privateKey
12872
14771
  }));
12873
14772
  } else if (Array.isArray(forwarder)) {
14773
+ observation?.setPhase("key-resolution");
12874
14774
  if (forwarder.length < 1) throw new Error("The forwarder's key pairs are empty.");
12875
14775
  keys = forwarder;
12876
14776
  } else keys = [forwarder];
14777
+ observation?.setPhase("validation");
12877
14778
  if (!hasSignatureLike(ctx.activity)) {
12878
14779
  if (!hasProofLike(ctx.activity)) {
12879
- if (options?.skipIfUnsigned) return false;
14780
+ if (options?.skipIfUnsigned) {
14781
+ if (observation != null) await observation.skipped("unsigned");
14782
+ return false;
14783
+ }
12880
14784
  logger.warn("The activity {activityId} is not signed; even if it is forwarded to other servers as is, it may not be accepted by them due to the lack of a signature/proof.", {
12881
14785
  activityId: ctx.activityId,
12882
14786
  activityType: ctx.activityType,
@@ -12884,23 +14788,27 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12884
14788
  });
12885
14789
  }
12886
14790
  }
14791
+ observation?.setPhase("recipient-resolution");
12887
14792
  if (recipients === "followers") {
12888
14793
  if (identifier == null) throw new Error("If recipients is \"followers\", forwarder must be an actor identifier or username.");
12889
14794
  const followers = [];
12890
14795
  for await (const recipient of ctx.getFollowers(identifier)) followers.push(recipient);
12891
14796
  recipients = followers;
12892
14797
  }
14798
+ observation?.setPhase("destination-planning");
12893
14799
  const inboxes = extractInboxes({
12894
14800
  recipients: Array.isArray(recipients) ? recipients : [recipients],
12895
14801
  preferSharedInbox: options?.preferSharedInbox,
12896
14802
  excludeBaseUris: options?.excludeBaseUris
12897
14803
  });
14804
+ if (observation != null) await observation.planDestinations(inboxes);
12898
14805
  if (globalThis.Object.keys(inboxes).length < 1) {
12899
14806
  logger.debug("No inboxes found for activity {activityId}.", {
12900
14807
  activityId: ctx.activityId,
12901
14808
  activityType: ctx.activityType,
12902
14809
  identifier: identifier ?? void 0
12903
14810
  });
14811
+ if (observation != null) await observation.skipped();
12904
14812
  return false;
12905
14813
  }
12906
14814
  logger.debug("Forwarding activity {activityId} to inboxes:\n{inboxes}", {
@@ -12911,18 +14819,33 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12911
14819
  if (options?.immediate || ctx.federation.outboxQueue == null) {
12912
14820
  if (options?.immediate) logger.debug("Forwarding activity immediately without queue since immediate option is set.");
12913
14821
  else logger.debug("Forwarding activity immediately without queue since queue is not set.");
14822
+ observation?.setPhase("request-preparation");
12914
14823
  const promises = [];
12915
- for (const inbox in inboxes) promises.push(sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
12916
- allowPrivateAddress: ctx.federation.allowPrivateAddress,
12917
- keys,
12918
- activity: ctx.activity,
12919
- activityId: ctx.activityId,
12920
- activityType: ctx.activityType,
12921
- sharedInbox: inboxes[inbox].sharedInbox,
12922
- meterProvider: ctx.meterProvider,
12923
- tracerProvider: ctx.tracerProvider,
12924
- specDeterminer: new KvSpecDeterminer(ctx.federation.kv, ctx.federation.kvPrefixes.httpMessageSignaturesSpec, ctx.federation.firstKnock, { specTtl: ctx.federation.httpMessageSignaturesSpecTtl })
12925
- }));
14824
+ for (const inbox in inboxes) {
14825
+ const delivery = observation?.delivery(inbox);
14826
+ let sending;
14827
+ try {
14828
+ sending = sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, {
14829
+ observation: delivery,
14830
+ allowPrivateAddress: ctx.federation.allowPrivateAddress,
14831
+ keys,
14832
+ activity: ctx.activity,
14833
+ activityId: ctx.activityId,
14834
+ activityType: ctx.activityType,
14835
+ sharedInbox: inboxes[inbox].sharedInbox,
14836
+ meterProvider: ctx.meterProvider,
14837
+ tracerProvider: ctx.tracerProvider,
14838
+ specDeterminer: new KvSpecDeterminer(ctx.federation.kv, ctx.federation.kvPrefixes.httpMessageSignaturesSpec, ctx.federation.firstKnock, { specTtl: ctx.federation.httpMessageSignaturesSpecTtl })
14839
+ });
14840
+ } catch (error) {
14841
+ if (delivery != null) await delivery.failed(error);
14842
+ throw error;
14843
+ }
14844
+ if (observation != null) sending.catch(() => {});
14845
+ if (delivery != null) await delivery.handoff({ type: "direct" });
14846
+ promises.push(sending);
14847
+ }
14848
+ if (observation != null) await observation.prepared("delivery");
12926
14849
  await Promise.all(promises);
12927
14850
  return true;
12928
14851
  }
@@ -12930,7 +14853,9 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12930
14853
  activityId: ctx.activityId,
12931
14854
  activity: ctx.activity
12932
14855
  });
14856
+ observation?.setPhase("request-preparation");
12933
14857
  if (!ctx.federation.manuallyStartQueue) ctx.federation._startQueueInternal(ctx.data);
14858
+ observation?.setPhase("key-resolution");
12934
14859
  const keyJwkPairs = [];
12935
14860
  for (const { keyId, privateKey } of keys) {
12936
14861
  const privateKeyJwk = await exportJwk(privateKey);
@@ -12939,15 +14864,17 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12939
14864
  privateKey: privateKeyJwk
12940
14865
  });
12941
14866
  }
14867
+ observation?.setPhase("request-preparation");
12942
14868
  const carrier = {};
12943
14869
  propagation.inject(context.active(), carrier);
12944
14870
  const orderingKey = options?.orderingKey;
12945
14871
  const started = (/* @__PURE__ */ new Date()).toISOString();
12946
14872
  const messages = [];
12947
- for (const inbox in inboxes) {
14873
+ for (const inbox in inboxes) try {
12948
14874
  const { portableInbox, gatewayInboxes } = inboxes[inbox];
12949
14875
  const message = {
12950
14876
  type: "outbox",
14877
+ ...serializeProvenance(provenance),
12951
14878
  id: crypto.randomUUID(),
12952
14879
  baseUrl: ctx.origin,
12953
14880
  keys: keyJwkPairs,
@@ -12955,6 +14882,7 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12955
14882
  activityId: ctx.activityId,
12956
14883
  activityType: ctx.activityType,
12957
14884
  inbox,
14885
+ ...observation?.continuation(inbox),
12958
14886
  sharedInbox: inboxes[inbox].sharedInbox,
12959
14887
  actorIds: [...inboxes[inbox].actorIds],
12960
14888
  started,
@@ -12968,11 +14896,18 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12968
14896
  message,
12969
14897
  orderingKey: message.orderingKey
12970
14898
  });
14899
+ } catch (error) {
14900
+ if (observation != null) await observation.delivery(inbox)?.failed(error);
14901
+ throw error;
12971
14902
  }
14903
+ if (observation != null) await observation.prepared("delivery");
12972
14904
  const { outboxQueue } = ctx.federation;
12973
14905
  if (outboxQueue.enqueueMany == null || orderingKey != null) {
12974
14906
  const errors = (await Promise.allSettled(messages.map(async (m) => {
12975
- await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey });
14907
+ await (observation == null ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) : observation.enqueue([{
14908
+ inbox: m.message.inbox,
14909
+ queueMessageId: m.message.id
14910
+ }], "outbox", () => outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey })));
12976
14911
  recordOutboxEnqueue(ctx.federation.meterProvider, outboxQueue, m.message);
12977
14912
  }))).filter((r) => r.status === "rejected").map((r) => r.reason);
12978
14913
  if (errors.length > 0) {
@@ -12985,7 +14920,10 @@ async function forwardActivityInternal(ctx, loggerCategory, forwarder, recipient
12985
14920
  }
12986
14921
  } else {
12987
14922
  try {
12988
- await outboxQueue.enqueueMany(messages.map((m) => m.message));
14923
+ await (observation == null ? outboxQueue.enqueueMany(messages.map((m) => m.message)) : observation.enqueue(messages.map(({ message }) => ({
14924
+ inbox: message.inbox,
14925
+ queueMessageId: message.id
14926
+ })), "outbox", () => outboxQueue.enqueueMany(messages.map((m) => m.message))));
12989
14927
  } catch (error) {
12990
14928
  logger.error("Failed to enqueue activity {activityId} to forward later:\n{error}", {
12991
14929
  activityId: ctx.activityId,
@@ -13021,6 +14959,7 @@ var InboxContextImpl = class InboxContextImpl extends ContextImpl {
13021
14959
  }
13022
14960
  clone(data) {
13023
14961
  return new InboxContextImpl(this.recipient, this.activity, this.activityId, this.activityType, {
14962
+ [outboundProvenance]: getOutboundProvenance(this),
13024
14963
  url: this.url,
13025
14964
  federation: this.federation,
13026
14965
  data,
@@ -13051,29 +14990,41 @@ var OutboxContextImpl = class OutboxContextImpl extends ContextImpl {
13051
14990
  return this.#deliveryState.delivered;
13052
14991
  }
13053
14992
  sendActivity(sender, recipients, activity, options = {}) {
13054
- return this.tracerProvider.getTracer(name, version).startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
13055
- kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
13056
- attributes: {
13057
- "activitypub.activity.type": getTypeId(activity).href,
13058
- "activitypub.activity.to": activity.toIds.map((to) => to.href),
13059
- "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
13060
- "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
13061
- "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
13062
- }
13063
- }, async (span) => {
13064
- try {
13065
- if (activity.id != null) span.setAttribute("activitypub.activity.id", activity.id.href);
13066
- if (await this.sendActivityInternal(sender, recipients, activity, options, span)) this.#deliveryState.delivered = true;
13067
- } catch (e) {
13068
- span.setStatus({
13069
- code: SpanStatusCode.ERROR,
13070
- message: String(e)
13071
- });
13072
- throw e;
13073
- } finally {
13074
- span.end();
13075
- }
13076
- });
14993
+ return observeOutboundBroadcast(this, this.federation.outboundObservers, (observation, provenance) => {
14994
+ const tracer = this.tracerProvider.getTracer(name, version);
14995
+ const parentSpanContext = trace.getSpanContext(context.active());
14996
+ const activityType = getTypeId(activity).href;
14997
+ observation?.setActivity({ type: activityType });
14998
+ return tracer.startActiveSpan(this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" : "activitypub.fanout", {
14999
+ kind: this.federation.outboxQueue == null || options.immediate ? SpanKind.CLIENT : SpanKind.PRODUCER,
15000
+ attributes: {
15001
+ "activitypub.activity.type": activityType,
15002
+ "activitypub.activity.to": activity.toIds.map((to) => to.href),
15003
+ "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href),
15004
+ "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href),
15005
+ "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href)
15006
+ }
15007
+ }, async (span) => {
15008
+ observation?.setSpan(span, parentSpanContext);
15009
+ try {
15010
+ const activityId = activity.id?.href;
15011
+ if (activityId != null) span.setAttribute("activitypub.activity.id", activityId);
15012
+ observation?.setActivity({
15013
+ type: activityType,
15014
+ ...activityId == null ? {} : { id: activityId }
15015
+ });
15016
+ if (await this.sendActivityInternal(sender, recipients, activity, options, span, observation, provenance)) this.#deliveryState.delivered = true;
15017
+ } catch (e) {
15018
+ span.setStatus({
15019
+ code: SpanStatusCode.ERROR,
15020
+ message: String(e)
15021
+ });
15022
+ throw e;
15023
+ } finally {
15024
+ span.end();
15025
+ }
15026
+ });
15027
+ }, "send", options.dispatchId);
13077
15028
  }
13078
15029
  forwardActivity(forwarder, recipients, options) {
13079
15030
  return forwardActivity(this, "outbox", forwarder, recipients, options).then((delivered) => {
@@ -13082,6 +15033,7 @@ var OutboxContextImpl = class OutboxContextImpl extends ContextImpl {
13082
15033
  }
13083
15034
  clone(data) {
13084
15035
  return new OutboxContextImpl(this.identifier, this.activity, this.activityId, this.activityType, {
15036
+ [outboundProvenance]: getOutboundProvenance(this),
13085
15037
  url: this.url,
13086
15038
  federation: this.federation,
13087
15039
  data,