@fedify/fedify 1.5.0-dev.729 → 1.5.0-dev.730

package/CHANGES.md

@@ -8,6 +8,18 @@ Version 1.5.0
 
 To be released.
 
+-  Improved activity delivery performance with large audiences through
+   a two-stage queuing system.  Sending activities to many recipients
+   (e.g., accounts with many followers) is now significantly faster and uses
+   less memory. [[#220]]
+
+     -  Added `FederationQueueOptions.fanout` option.
+     -  Changed the type of `FederationStartQueueOptions.queue` option to
+        `"inbox" | "outbox" | "fanout" | undefined` (was `"inbox" | "outbox" |
+        undefined`).
+     -  Added `SendActivityOptions.fanout` option.
+     -  Added OpenTelemetry instrumented span `activitypub.fanout`.
+
  -  A `Federation` object now can have a canonical origin for web URLs and
     a canonical host for fediverse handles.  This affects the URLs constructed
     by `Context` objects, and the WebFinger responses.
@@ -65,12 +77,14 @@ To be released.
  -  Added more log messages using the [LogTape] library.  Currently the below
     logger categories are used:
 
+     -  `["fedify", "federation", "fanout"]`
      -  `["fedify", "federation", "object"]`
 
 [#127]: https://github.com/fedify-dev/fedify/issues/127
 [#209]: https://github.com/fedify-dev/fedify/issues/209
 [#211]: https://github.com/fedify-dev/fedify/issues/211
 [#215]: https://github.com/fedify-dev/fedify/pull/215
+[#220]: https://github.com/fedify-dev/fedify/issues/220
 [multibase]: https://github.com/multiformats/js-multibase
 
 

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "@fedify/fedify",
-    "version": "1.5.0-dev.729+010d765a",
+    "version": "1.5.0-dev.730+390610bf",
     "license": "MIT",
     "exports": {
         ".": "./mod.ts",

package/esm/federation/middleware.js

@@ -37,8 +37,10 @@ export class FederationImpl {
     kvPrefixes;
     inboxQueue;
     outboxQueue;
+    fanoutQueue;
     inboxQueueStarted;
     outboxQueueStarted;
+    fanoutQueueStarted;
     manuallyStartQueue;
     origin;
     router;
@@ -83,17 +85,21 @@ export class FederationImpl {
         if (options.queue == null) {
             this.inboxQueue = undefined;
             this.outboxQueue = undefined;
+            this.fanoutQueue = undefined;
         }
         else if ("enqueue" in options.queue && "listen" in options.queue) {
             this.inboxQueue = options.queue;
             this.outboxQueue = options.queue;
+            this.fanoutQueue = options.queue;
         }
         else {
             this.inboxQueue = options.queue.inbox;
             this.outboxQueue = options.queue.outbox;
+            this.fanoutQueue = options.queue.fanout;
         }
         this.inboxQueueStarted = false;
         this.outboxQueueStarted = false;
+        this.fanoutQueueStarted = false;
         this.manuallyStartQueue = options.manuallyStartQueue ?? false;
         if (options.origin != null) {
             if (typeof options.origin === "string") {
@@ -206,7 +212,7 @@ export class FederationImpl {
     #getTracer() {
         return this.tracerProvider.getTracer(metadata.name, metadata.version);
     }
-    async #startQueue(ctxData, signal, queue) {
+    async _startQueueInternal(ctxData, signal, queue) {
         if (this.inboxQueue == null && this.outboxQueue == null)
             return;
         const logger = getLogger(["fedify", "federation", "queue"]);
@@ -225,13 +231,47 @@ export class FederationImpl {
             this.outboxQueueStarted = true;
             promises.push(this.outboxQueue.listen((msg) => this.#listenQueue(ctxData, msg), { signal }));
         }
+        if (this.fanoutQueue != null &&
+            this.fanoutQueue !== this.inboxQueue &&
+            this.fanoutQueue !== this.outboxQueue &&
+            (queue == null || queue === "fanout") &&
+            !this.fanoutQueueStarted) {
+            logger.debug("Starting a fanout task worker.");
+            this.fanoutQueueStarted = true;
+            promises.push(this.fanoutQueue.listen((msg) => this.#listenQueue(ctxData, msg), { signal }));
+        }
         await Promise.all(promises);
     }
     #listenQueue(ctxData, message) {
         const tracer = this.#getTracer();
         const extractedContext = propagation.extract(context.active(), message.traceContext);
         return withContext({ messageId: message.id }, async () => {
-            if (message.type === "outbox") {
+            if (message.type === "fanout") {
+                await tracer.startActiveSpan("activitypub.fanout", {
+                    kind: SpanKind.CONSUMER,
+                    attributes: {
+                        "activitypub.activity.type": message.activityType,
+                    },
+                }, extractedContext, async (span) => {
+                    if (message.activityId != null) {
+                        span.setAttribute("activitypub.activity.id", message.activityId);
+                    }
+                    try {
+                        await this.#listenFanoutMessage(ctxData, message);
+                    }
+                    catch (e) {
+                        span.setStatus({
+                            code: SpanStatusCode.ERROR,
+                            message: String(e),
+                        });
+                        throw e;
+                    }
+                    finally {
+                        span.end();
+                    }
+                });
+            }
+            else if (message.type === "outbox") {
                 await tracer.startActiveSpan("activitypub.outbox", {
                     kind: SpanKind.CONSUMER,
                     attributes: {
@@ -281,6 +321,33 @@ export class FederationImpl {
             }
         });
     }
+    async #listenFanoutMessage(data, message) {
+        const keys = await Promise.all(message.keys.map(async ({ keyId, privateKey }) => ({
+            keyId: new URL(keyId),
+            privateKey: await importJwk(privateKey, "private"),
+        })));
+        const activity = await Activity.fromJsonLd(message.activity, {
+            contextLoader: this.contextLoaderFactory({
+                allowPrivateAddress: this.allowPrivateAddress,
+                userAgent: this.userAgent,
+            }),
+            documentLoader: this.documentLoaderFactory({
+                allowPrivateAddress: this.allowPrivateAddress,
+                userAgent: this.userAgent,
+            }),
+            tracerProvider: this.tracerProvider,
+        });
+        const context = this.#createContext(new URL(message.baseUrl), data, {
+            documentLoader: this.documentLoaderFactory({
+                allowPrivateAddress: this.allowPrivateAddress,
+                userAgent: this.userAgent,
+            }),
+        });
+        await this.sendActivity(keys, message.inboxes, activity, {
+            collectionSync: message.collectionSync,
+            context,
+        });
+    }
     async #listenOutboxMessage(_, message, span) {
         const logger = getLogger(["fedify", "federation", "outbox"]);
         const logData = {
@@ -485,7 +552,7 @@ export class FederationImpl {
         });
     }
     startQueue(contextData, options = {}) {
-        return this.#startQueue(contextData, options.signal, options.queue);
+        return this._startQueueInternal(contextData, options.signal, options.queue);
     }
     createContext(urlOrRequest, contextData) {
         return urlOrRequest instanceof Request
@@ -1118,33 +1185,9 @@ export class FederationImpl {
         };
         return setters;
     }
-    async sendActivity(keys, recipients, activity, options, span) {
+    async sendActivity(keys, inboxes, activity, options) {
         const logger = getLogger(["fedify", "federation", "outbox"]);
-        const { preferSharedInbox, immediate, excludeBaseUris, collectionSync, context: ctx, } = options;
-        if (keys.length < 1) {
-            throw new TypeError("The sender's keys must not be empty.");
-        }
-        for (const { privateKey } of keys) {
-            validateCryptoKey(privateKey, "private");
-        }
-        for (const activityTransformer of this.activityTransformers) {
-            activity = activityTransformer(activity, ctx);
-        }
-        span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? "");
-        if (activity.actorId == null) {
-            logger.error("Activity {activityId} to send does not have an actor.", { activity, activityId: activity?.id?.href });
-            throw new TypeError("The activity to send must have at least one actor property.");
-        }
-        const inboxes = extractInboxes({
-            recipients: Array.isArray(recipients) ? recipients : [recipients],
-            preferSharedInbox,
-            excludeBaseUris,
-        });
-        logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", {
-            inboxes: globalThis.Object.keys(inboxes),
-            activityId: activity.id?.href,
-            activity,
-        });
+        const { immediate, collectionSync, context: ctx } = options;
         if (activity.id == null) {
             throw new TypeError("The activity to send must have an id.");
         }
@@ -1238,7 +1281,7 @@ export class FederationImpl {
             keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk });
         }
         if (!this.manuallyStartQueue)
-            this.#startQueue(ctx.data);
+            this._startQueueInternal(ctx.data);
         const carrier = {};
         propagation.inject(context.active(), carrier);
         const promises = [];
@@ -1441,7 +1484,7 @@ export class FederationImpl {
                     }
                 }
                 if (!this.manuallyStartQueue)
-                    this.#startQueue(contextData);
+                    this._startQueueInternal(contextData);
                 return await handleInbox(request, {
                     recipient: route.values.identifier ?? route.values.handle ?? null,
                     context,
@@ -1539,6 +1582,7 @@ export class FederationImpl {
         }
     }
 }
+const FANOUT_THRESHOLD = 5;
 export class ContextImpl {
     url;
     federation;
@@ -1946,7 +1990,9 @@ export class ContextImpl {
     }
     sendActivity(sender, recipients, activity, options = {}) {
         const tracer = this.tracerProvider.getTracer(metadata.name, metadata.version);
-        return tracer.startActiveSpan("activitypub.outbox", {
+        return tracer.startActiveSpan(this.federation.outboxQueue == null || options.immediate
+            ? "activitypub.outbox"
+            : "activitypub.fanout", {
             kind: this.federation.outboxQueue == null || options.immediate
                 ? SpanKind.CLIENT
                 : SpanKind.PRODUCER,
@@ -1974,6 +2020,7 @@ export class ContextImpl {
         });
     }
     async sendActivityInternal(sender, recipients, activity, options = {}, span) {
+        const logger = getLogger(["fedify", "federation", "outbox"]);
         let keys;
         let identifier = null;
         if ("identifier" in sender || "username" in sender || "handle" in sender) {
@@ -1987,7 +2034,7 @@ export class ContextImpl {
                 }
                 else {
                     username = sender.handle;
-                    getLogger(["fedify", "federation", "outbox"]).warn('The "handle" property for the sender parameter is deprecated; ' +
+                    logger.warn('The "handle" property for the sender parameter is deprecated; ' +
                         'use "identifier" or "username" instead.', { sender });
                 }
                 if (this.federation.actorCallbacks?.handleMapper == null) {
@@ -2016,10 +2063,13 @@ export class ContextImpl {
         else {
             keys = [sender];
         }
-        const opts = {
-            context: this,
-            ...options,
-        };
+        if (keys.length < 1) {
+            throw new TypeError("The sender's keys must not be empty.");
+        }
+        for (const { privateKey } of keys) {
+            validateCryptoKey(privateKey, "private");
+        }
+        const opts = { context: this };
         let expandedRecipients;
         if (Array.isArray(recipients)) {
             expandedRecipients = recipients;
@@ -2042,7 +2092,55 @@ export class ContextImpl {
             expandedRecipients = [recipients];
         }
         span.setAttribute("activitypub.inboxes", expandedRecipients.length);
-        return await this.federation.sendActivity(keys, expandedRecipients, activity, opts, span);
+        for (const activityTransformer of this.federation.activityTransformers) {
+            activity = activityTransformer(activity, this);
+        }
+        span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? "");
+        if (activity.actorId == null) {
+            logger.error("Activity {activityId} to send does not have an actor.", { activity, activityId: activity?.id?.href });
+            throw new TypeError("The activity to send must have at least one actor property.");
+        }
+        const inboxes = extractInboxes({
+            recipients: expandedRecipients,
+            preferSharedInbox: options.preferSharedInbox,
+            excludeBaseUris: options.excludeBaseUris,
+        });
+        logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", {
+            inboxes: globalThis.Object.keys(inboxes),
+            activityId: activity.id?.href,
+            activity,
+        });
+        if (this.federation.fanoutQueue == null || options.immediate ||
+            options.fanout === "skip" || (options.fanout ?? "auto") === "auto" &&
+            globalThis.Object.keys(inboxes).length < FANOUT_THRESHOLD) {
+            await this.federation.sendActivity(keys, inboxes, activity, opts);
+            return;
+        }
+        const keyJwkPairs = await Promise.all(keys.map(async ({ keyId, privateKey }) => ({
+            keyId: keyId.href,
+            privateKey: await exportJwk(privateKey),
+        })));
+        const carrier = {};
+        propagation.inject(context.active(), carrier);
+        const message = {
+            type: "fanout",
+            id: dntShim.crypto.randomUUID(),
+            baseUrl: this.origin,
+            keys: keyJwkPairs,
+            inboxes: globalThis.Object.fromEntries(globalThis.Object.entries(inboxes).map(([k, { actorIds, sharedInbox }]) => [k, { actorIds: [...actorIds], sharedInbox }])),
+            activity: await activity.toJsonLd({
+                format: "compact",
+                contextLoader: this.contextLoader,
+            }),
+            activityId: activity.id?.href,
+            activityType: getTypeId(activity).href,
+            collectionSync: opts.collectionSync,
+            traceContext: carrier,
+        };
+        if (!this.federation.manuallyStartQueue) {
+            this.federation._startQueueInternal(this.data);
+        }
+        this.federation.fanoutQueue.enqueue(message);
     }
     async *getFollowers(identifier) {
         if (this.federation.followersCallbacks == null) {