@evanp/activitypub-bot 0.34.1 → 0.35.0

package/README.md

@@ -58,6 +58,12 @@ Options:
   --port <number>            Port to listen on
   --bots-config-file <path>  Path to bots config module
   --log-level <level>        Log level (e.g., info, debug)
+  --delivery <number>        Number of background delivery workers
+  --distribution <number>    Number of background distribution workers
+  --fanout <number>          Number of background fanout workers
+  --intake <number>          Number of background intake workers
+  --index-file <path>        HTML page to show at root path
+  --profile-file <path>      HTML page to show for bot profiles
   -h, --help                 Show this help
 ```
 
@@ -109,11 +115,29 @@ The number of background distribution workers to run for this server.  These wor
 
 Falls back to the `DISTRIBUTION` environment variable. Default is 8.
 
+#### --fanout
+
+The number of background fanout workers to run for this server. These workers resolve the recipients for bot activities — expanding follower collections and other addressing — and enqueue individual delivery jobs. If activities are slow to reach recipients, increase this number.
+
+Falls back to the `FANOUT` environment variable. Default is 4.
+
+#### --intake
+
+The number of background intake workers to run for this server. These workers accept activities arriving at the shared inbox and route them to the appropriate local bots. If shared-inbox processing is a bottleneck, increase this number.
+
+Falls back to the `INTAKE` environment variable. Default is 2.
+
 #### --index-file
 
 Path to the HTML file to show for the home page of your server. The activitypub.bot server doesn't support any other files, so any images or CSS stylesheets or JavaScript in this page have to be hosted elsewhere. Or just skip them!
 
-Falls back to the 'INDEX_FILE' environment variable. The default index file is in `web/index.html` and just says that this is an activitypub.bot server with a link to the GitHub repo.
+Falls back to the `INDEX_FILE` environment variable. The default index file is in `web/index.html` and just says that this is an activitypub.bot server with a link to the GitHub repo.
+
+#### --profile-file
+
+Path to the HTML file to show for bot profile pages. Like `--index-file`, any external assets must be hosted elsewhere.
+
+Falls back to the `PROFILE_FILE` environment variable. The default profile file is in `web/profile.html`.
 
 ### Config file
 
@@ -184,9 +208,15 @@ Bots will receive a [BotContext](#botcontext) object at initialization. The BotC
 
 The Bot interface has the following methods.
 
-#### constructor (username)
+#### constructor (username, options)
+
+The constructor; receives the `username` and an optional `options` object. Initialization should probably be deferred to the `initialize()` method. The default implementation stores the username and the following options:
 
-The constructor; receives the `username` by default. Initialization should probably be deferred to the initialize() method. The default implementation stores the username.
+- `fullname` — display name for the bot. Default: `'Bot'`.
+- `description` — bio text for the bot. Default: `'Default bot'`.
+- `icon` — URL string for the bot's avatar image. Default: `null`.
+- `image` — URL string for the bot's header/banner image. Default: `null`.
+- `checkSignature` — whether to require valid HTTP signatures on incoming activities. Default: `true`.
 
 #### async initialize (context)
 
@@ -200,6 +230,18 @@ A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Fun
 
 A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the bio of the bot.
 
+#### get icon ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the avatar image URL of the bot. Returns `null` if not set.
+
+#### get image ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the header/banner image URL of the bot. Returns `null` if not set.
+
+#### get checkSignature ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for whether the bot requires valid HTTP signatures on incoming activities.
+
 #### get username ()
 
 A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the username of the bot. Should match the constructor argument.

package/bin/activitypub-bot.js

@@ -14,6 +14,8 @@ const { values } = parseArgs({
     'log-level': { type: 'string' },
     delivery: { type: 'string' },
     distribution: { type: 'string' },
+    fanout: { type: 'string' },
+    intake: { type: 'string' },
     'index-file': { type: 'string' },
     help: { type: 'boolean', short: 'h' }
   },
@@ -31,6 +33,8 @@ Options:
   --log-level <level>        Log level (e.g., info, debug)
   --delivery <number>        Number of background delivery workers
   --distribution <number>    Number of background distribution workers
+  --fanout <number>          Number of background fanout workers
+  --intake <number>          Number of background intake workers
   --index-file <path>        HTML page to show at root path
   --profile-file <path>      HTML page to show for bot profiles
   -h, --help                 Show this help
@@ -61,6 +65,8 @@ const LOG_LEVEL =
   (process.env.NODE_ENV === 'test' ? 'silent' : 'info')
 const DELIVERY = parseNumber(values.delivery) || parseNumber(process.env.DELIVERY) || 2
 const DISTRIBUTION = parseNumber(values.distribution) || parseNumber(process.env.DISTRIBUTION) || 8
+const FANOUT = parseNumber(values.fanout) || parseNumber(process.env.FANOUT) || 4
+const INTAKE = parseNumber(values.intake) || parseNumber(process.env.INTAKE) || 2
 const INDEX_FILE = values['index-file'] || process.env.INDEX_FILE || DEFAULT_INDEX_FILE
 const PROFILE_FILE = values['profile-file'] || process.env.PROFILE_FILE || DEFAULT_PROFILE_FILE
 
@@ -73,6 +79,8 @@ const app = await makeApp({
   logLevel: LOG_LEVEL,
   deliveryWorkerCount: DELIVERY,
   distributionWorkerCount: DISTRIBUTION,
+  fanoutWorkerCount: FANOUT,
+  intakeWorkerCount: INTAKE,
   indexFileName: INDEX_FILE,
   profileFileName: PROFILE_FILE
 })

package/lib/activitydeliverer.js

@@ -23,6 +23,7 @@ const COLLECTION_TYPES = [
 
 export class ActivityDeliverer {
   static #QUEUE_ID = 'delivery'
+  static #INTAKE_QUEUE_ID = 'intake'
   static #TTL_SEEN_PUBLIC = 3 * 24 * 60 * 60 * 1000 // 3 days
   static #MAX_SEEN_PUBLIC = 1000000
   #actorStorage
@@ -106,10 +107,19 @@ export class ActivityDeliverer {
 
   async onIdle () {
     this.#logger.debug('Awaiting delivery queues')
+    await this.#jobQueue.onIdle(ActivityDeliverer.#INTAKE_QUEUE_ID)
     await this.#jobQueue.onIdle(ActivityDeliverer.#QUEUE_ID)
     this.#logger.debug('Done awaiting delivery queues')
   }
 
+  async intake (activity, subject) {
+    const raw = await activity.export({ useOriginalContext: true })
+    await this.#jobQueue.enqueue(
+      ActivityDeliverer.#INTAKE_QUEUE_ID,
+      { activity: raw, subject }
+    )
+  }
+
   async deliverToAll (activity, bots) {
     await this.#deliverToAll(activity, bots)
   }

package/lib/activitydistributor.js

@@ -14,6 +14,7 @@ const COLLECTION_TYPES = [
 export class ActivityDistributor {
   static #DISTRIBUTION_QUEUE_ID = 'distribution'
   static #DELIVERY_QUEUE_ID = 'delivery'
+  static #FANOUT_QUEUE_ID = 'fanout'
   static #MAX_CACHE_SIZE = 1000000
   static #PUBLIC = [
     'https://www.w3.org/ns/activitystreams#Public',
@@ -45,6 +46,14 @@ export class ActivityDistributor {
   }
 
   async distribute (activity, username) {
+    const raw = await activity.export({ useOriginalContext: true })
+    await this.#jobQueue.enqueue(
+      ActivityDistributor.#FANOUT_QUEUE_ID,
+      { activity: raw, username }
+    )
+  }
+
+  async fanout (activity, username) {
     const stripped = await this.#strip(activity)
     const raw = await stripped.export()
     const actorId = this.#formatter.format({ username })
@@ -102,8 +111,9 @@ export class ActivityDistributor {
   }
 
   async onIdle () {
-    await this.#jobQueue.onIdle(ActivityDistributor.#DELIVERY_QUEUE_ID)
+    await this.#jobQueue.onIdle(ActivityDistributor.#FANOUT_QUEUE_ID)
     await this.#jobQueue.onIdle(ActivityDistributor.#DISTRIBUTION_QUEUE_ID)
+    await this.#jobQueue.onIdle(ActivityDistributor.#DELIVERY_QUEUE_ID)
   }
 
   async * #public (activity, username) {

package/lib/activityhandler.js

@@ -311,14 +311,14 @@ export class ActivityHandler {
       actor: actor.id
     })
     await this.#actorStorage.addToCollection(bot.username, 'followers', actor)
-    this.#logger.info(
-      'Sending accept',
-      { actor: actor.id }
-    )
     const followersId = this.#formatter.format({
       username: bot.username,
       collection: 'followers'
     })
+    this.#logger.debug(
+      'Sending add activity for adding actor to followers',
+      { actor: actor.id, followers: followersId, follow: activity.id }
+    )
     const add = await this.#doActivity(bot, {
       type: 'Add',
       object: actor.id,
@@ -329,6 +329,10 @@ export class ActivityHandler {
     const recipients = await this.#getRecipients(activity)
     this.#addRecipient(recipients, actor, 'to')
     this.#removeRecipient(recipients, await this.#botActor(bot))
+    this.#logger.debug(
+      'Sending accept activity for follow',
+      { bot: this.#botId(bot), actor: actor.id, follow: activity.id }
+    )
     const accept = await this.#doActivity(bot, {
       type: 'Accept',
       object: activity.id,
@@ -337,7 +341,9 @@ export class ActivityHandler {
     this.#logger.debug({ accept: await accept.export() }, 'accepted follow')
     this.#logger.debug({
       msg: 'Notifying bot of new follow',
-      actor: actor.id
+      actor: actor.id,
+      bot: this.#botId(bot),
+      follow: activity.id
     })
     await bot.onFollow(actor, activity)
   }

package/lib/actorstorage.js

@@ -6,7 +6,7 @@ const AS2_NS = 'https://www.w3.org/ns/activitystreams#'
 export class ActorStorage {
   #connection = null
   #formatter = null
-  static #MAX_ITEMS_PER_PAGE = 20
+  static #MAX_ITEMS_PER_PAGE = 256
   constructor (connection, formatter) {
     this.#connection = connection
     this.#formatter = formatter

package/lib/app.js

@@ -41,12 +41,20 @@ import { DeliveryWorker } from './deliveryworker.js'
 import { DistributionWorker } from './distributionworker.js'
 import { RateLimiter } from '../lib/ratelimiter.js'
 import DoNothingBot from './bots/donothing.js'
+import { FanoutWorker } from './fanoutworker.js'
+import { IntakeWorker } from './intakeworker.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
 const DEFAULT_PROFILE_FILENAME = resolve(currentDir, '..', 'web', 'profile.html')
 
-export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME }) {
+function createWorkers (logger, count, WorkerClass, ...args) {
+  const workers = Array.from({ length: count }, () => new WorkerClass(...args))
+  const runs = workers.map(w => w.run().catch(err => logger.error({ err }, `unexpected error in ${WorkerClass.name}`)))
+  return { workers, runs }
+}
+
+export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, fanoutWorkerCount = 4, intakeWorkerCount = 2, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME }) {
   const logger = Logger({
     level: logLevel
   })
@@ -136,35 +144,10 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
     logger
   ))
 
-  const deliveryWorkers = new Array(deliveryWorkerCount)
-  const deliveryWorkerRuns = new Array(deliveryWorkerCount)
-
-  for (let i = 0; i < deliveryWorkers.length; i++) {
-    deliveryWorkers[i] = new DeliveryWorker(
-      jobQueue,
-      actorStorage,
-      activityHandler,
-      logger,
-      bots
-    )
-    deliveryWorkerRuns[i] = deliveryWorkers[i].run().catch((err) => {
-      logger.error({ err, workerIndex: i }, 'unexpected error in delivery worker')
-    })
-  }
-
-  const distributionWorkers = new Array(distributionWorkerCount)
-  const distributionWorkerRuns = new Array(distributionWorkerCount)
-
-  for (let i = 0; i < distributionWorkers.length; i++) {
-    distributionWorkers[i] = new DistributionWorker(
-      jobQueue,
-      client,
-      logger
-    )
-    distributionWorkerRuns[i] = distributionWorkers[i].run().catch((err) => {
-      logger.error({ err, workerIndex: i }, 'unexpected error in distribution worker')
-    })
-  }
+  const { workers: deliveryWorkers, runs: deliveryWorkerRuns } = createWorkers(logger, deliveryWorkerCount, DeliveryWorker, jobQueue, logger, { actorStorage, activityHandler, bots })
+  const { workers: distributionWorkers, runs: distributionWorkerRuns } = createWorkers(logger, distributionWorkerCount, DistributionWorker, jobQueue, logger, { client })
+  const { workers: fanoutWorkers, runs: fanoutWorkerRuns } = createWorkers(logger, fanoutWorkerCount, FanoutWorker, jobQueue, logger, { distributor })
+  const { workers: intakeWorkers, runs: intakeWorkerRuns } = createWorkers(logger, intakeWorkerCount, IntakeWorker, jobQueue, logger, { deliverer, bots })
 
   const jobReaper = new JobReaper(jobQueue, logger)
   const jobReaperRun = jobReaper.run().catch((err) => {
@@ -273,23 +256,34 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
 
   app.onIdle = async () => {
     logger.debug('Awaiting components')
-    await distributor.onIdle()
-    await deliverer.onIdle()
+    await jobQueue.onIdle('intake')
+    await jobQueue.onIdle('delivery')
+    await jobQueue.onIdle('fanout')
+    await jobQueue.onIdle('delivery')
+    await jobQueue.onIdle('distribution')
     logger.debug('Done awaiting components')
   }
 
   app.cleanup = async () => {
     logger.info('Closing app')
-    for (const worker of deliveryWorkers) {
+    for (const worker of intakeWorkers) {
+      worker.stop()
+    }
+    for (const worker of fanoutWorkers) {
       worker.stop()
     }
     for (const worker of distributionWorkers) {
       worker.stop()
     }
+    for (const worker of deliveryWorkers) {
+      worker.stop()
+    }
     jobReaper.stop()
     jobQueue.abort()
-    await Promise.allSettled(deliveryWorkerRuns)
+    await Promise.allSettled(intakeWorkerRuns)
+    await Promise.allSettled(fanoutWorkerRuns)
     await Promise.allSettled(distributionWorkerRuns)
+    await Promise.allSettled(deliveryWorkerRuns)
     await jobReaperRun
     logger.info('Closing database connection')
     await connection.close()

package/lib/deliveryworker.js

@@ -1,71 +1,36 @@
 import assert from 'node:assert'
 
-import { nanoid } from 'nanoid'
-
 import as2 from './activitystreams.js'
 import BotMaker from './botmaker.js'
+import { Worker } from './worker.js'
 
-export class DeliveryWorker {
-  static #QUEUE_ID = 'delivery'
-  #queue
+export class DeliveryWorker extends Worker {
   #actorStorage
   #activityHandler
-  #logger
-  #running
-  #workerId
   #bots
 
-  constructor (queue, actorStorage, activityHandler, logger, bots) {
-    this.#queue = queue
-    this.#actorStorage = actorStorage
-    this.#activityHandler = activityHandler
-    this.#workerId = nanoid()
-    this.#logger = logger.child({
-      class: this.constructor.name,
-      worker: this.#workerId
-    })
-    this.#bots = bots
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.actorStorage)
+    assert.ok(options.activityHandler)
+    assert.ok(options.bots)
+    this.#actorStorage = options.actorStorage
+    this.#activityHandler = options.activityHandler
+    this.#bots = options.bots
   }
 
-  async run () {
-    this.#running = true
-    while (this.#running) {
-      let jobId
-      let payload
-      try {
-        this.#logger.debug('dequeueing');
-        ({ jobId, payload } = await this.#queue.dequeue(
-          DeliveryWorker.#QUEUE_ID,
-          this.#workerId
-        ))
-        this.#logger.debug({ jobId, payload }, 'got a job')
-        const activity = await as2.import(payload.activity)
-        assert.ok(payload.botUsername)
-        const bot = await BotMaker.makeBot(this.#bots, payload.botUsername)
-        assert.ok(bot)
-        assert.ok(bot.username)
-        this.#logger.debug(
-          { bot: bot.username, activity: activity.id, jobId },
-          'delivering to bot'
-        )
-        await this.#deliverTo(activity, bot)
-        this.#logger.debug({ bot: bot.username, activity: activity.id, jobId }, 'done')
-        await this.#queue.complete(jobId, this.#workerId)
-      } catch (err) {
-        if (err?.name === 'AbortError') {
-          this.#logger.info('Worker received abort signal')
-          break
-        }
-        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
-        if (jobId) {
-          await this.#queue.release(jobId, this.#workerId)
-        }
-      }
-    }
-  }
-
-  stop () {
-    this.#running = false
+  async doJob (payload, attempts) {
+    const activity = await as2.import(payload.activity)
+    assert.ok(payload.botUsername)
+    const bot = await BotMaker.makeBot(this.#bots, payload.botUsername)
+    assert.ok(bot)
+    assert.ok(bot.username)
+    this._logger.debug(
+      { bot: bot.username, activity: activity.id },
+      'delivering to bot'
+    )
+    await this.#deliverTo(activity, bot)
+    this._logger.debug({ bot: bot.username, activity: activity.id }, 'done')
   }
 
   async #deliverTo (activity, bot) {
@@ -75,7 +40,7 @@ export class DeliveryWorker {
     assert.strictEqual(typeof bot, 'object')
     assert.strictEqual(typeof bot.username, 'string')
     if (await this.#actorStorage.isInCollection(bot.username, 'inbox', activity)) {
-      this.#logger.info(
+      this._logger.info(
         { activity: activity.id, username: bot.username },
         'skipping redelivery for activity already in the inbox'
       )
@@ -84,13 +49,13 @@ export class DeliveryWorker {
     try {
       await this.#activityHandler.handleActivity(bot, activity)
     } catch (err) {
-      this.#logger.warn(
+      this._logger.warn(
         { err, activity: activity.id, bot: bot.username },
         'handler failed for activity'
       )
     }
-    this.#logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
+    this._logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
     await this.#actorStorage.addToCollection(bot.username, 'inbox', activity)
-    this.#logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
+    this._logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
   }
 }

package/lib/distributionworker.js

@@ -1,123 +1,87 @@
-import { nanoid } from 'nanoid'
+import assert from 'node:assert'
 
 import as2 from './activitystreams.js'
+import { Worker, RecoverableError } from './worker.js'
 
-export class DistributionWorker {
-  static #QUEUE_ID = 'distribution'
+export class DistributionWorker extends Worker {
   static #MAX_ATTEMPTS = 21 // ~24 days
-  #jobQueue
   #client
-  #logger
-  #running
-  #workerId
 
-  constructor (jobQueue, client, logger) {
-    this.#jobQueue = jobQueue
-    this.#client = client
-    this.#workerId = nanoid()
-    this.#logger = logger.child({
-      class: this.constructor.name,
-      workerId: this.#workerId
-    })
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.client)
+    this.#client = options.client
   }
 
-  async run () {
-    this.#running = true
-    while (this.#running) {
-      let jobId
-      let payload
-      let attempts
-      try {
-        this.#logger.debug('dequeueing');
-        ({ jobId, payload, attempts } = await this.#jobQueue.dequeue(
-          DistributionWorker.#QUEUE_ID,
-          this.#workerId
-        ))
-        this.#logger.debug({ jobId, payload, attempts }, 'got a job')
-        const { inbox, activity, username } = payload
-        const activityObj = await as2.import(activity)
-        try {
-          await this.#client.post(inbox, activityObj, username)
-          this.#logger.info(`Delivered ${activity.id} to ${inbox}`)
-          await this.#jobQueue.complete(jobId, this.#workerId)
-          this.#logger.info({ jobId }, 'completed job')
-        } catch (error) {
-          if (!error.status) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity and no HTTP status available')
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          } else if (error.status >= 300 && error.status < 400) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity and unexpected redirect code'
-            )
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          } else if (error.status >= 400 && error.status < 500) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity due to client error'
-            )
-            if ([408, 425, 429].includes(error.status)) {
-              this.#logger.debug(
-                { error, activity: activity.id, inbox },
-                'Retrying on recoverable status'
-              )
-              const delay = this.#retryDelay(error.headers, attempts)
-              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
-            } else {
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            }
-          } else if (error.status >= 500 && error.status < 600) {
-            if ([501, 505, 508, 510].includes(error.status)) {
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts },
-                'Could not deliver activity due to unrecoverable server error'
-              )
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            } else if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts },
-                'Could not deliver activity due to server error; no more attempts'
-              )
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            } else {
-              const delay = this.#retryDelay(error.headers, attempts)
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts, delay },
-                'Could not deliver activity due to server error; will retry'
-              )
-              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
-            }
-          } else {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity due to unexpected status range'
-            )
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          }
-        }
-      } catch (err) {
-        if (err?.name === 'AbortError') {
-          this.#logger.info('Worker received abort signal')
-          break
+  async doJob (payload, attempts) {
+    const { inbox, activity, username } = payload
+    const activityObj = await as2.import(activity)
+    try {
+      await this.#client.post(inbox, activityObj, username)
+      this._logger.info(`Delivered ${activity.id} to ${inbox}`)
+    } catch (error) {
+      if (!error.status) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity and no HTTP status available')
+        throw error
+      } else if (error.status >= 300 && error.status < 400) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity and unexpected redirect code'
+        )
+        throw error
+      } else if (error.status >= 400 && error.status < 500) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity due to client error'
+        )
+        if ([408, 425, 429].includes(error.status)) {
+          this._logger.debug(
+            { error, activity: activity.id, inbox },
+            'Retrying on recoverable status'
+          )
+          const recoverable = new RecoverableError(error.message)
+          recoverable.delay = this.#retryDelay(error.headers, attempts)
+          throw recoverable
+        } else {
+          throw error
         }
-        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
-        if (jobId) {
-          const delay = this.#retryDelay(null, attempts ?? 1)
-          this.#logger.warn(
-            { err, jobId, attempts, delay },
-            'Retrying job after a delay'
+      } else if (error.status >= 500 && error.status < 600) {
+        if ([501, 505, 508, 510].includes(error.status)) {
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts },
+            'Could not deliver activity due to unrecoverable server error'
+          )
+          throw error
+        } else if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts },
+            'Could not deliver activity due to server error; no more attempts'
+          )
+          throw error
+        } else {
+          const recoverable = new RecoverableError(error.message)
+          recoverable.delay = this.#retryDelay(error.headers, attempts)
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts, delay: recoverable.delay },
+            'Could not deliver activity due to server error; will retry'
           )
-          await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+          throw recoverable
         }
+      } else {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity due to unexpected status range'
+        )
+        throw error
       }
     }
   }
 
   #retryDelay (headers, attempts) {
     if (headers?.['retry-after']) {
-      this.#logger.debug('using retry-after header')
+      this._logger.debug('using retry-after header')
       const retryAfter = headers['retry-after']
       if (/^\d+$/.test(retryAfter)) {
         return parseInt(retryAfter, 10) * 1000
@@ -125,11 +89,7 @@ export class DistributionWorker {
         return new Date(retryAfter) - Date.now()
       }
     }
-    this.#logger.debug('exponential backoff')
+    this._logger.debug('exponential backoff')
     return Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
   }
-
-  stop () {
-    this.#running = false
-  }
 }

package/lib/fanoutworker.js

@@ -0,0 +1,20 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+import { Worker } from './worker.js'
+
+export class FanoutWorker extends Worker {
+  #distributor
+
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.distributor)
+    this.#distributor = options.distributor
+  }
+
+  async doJob (payload, attempts) {
+    const activity = await as2.import(payload.activity)
+    const { username } = payload
+    await this.#distributor.fanout(activity, username)
+  }
+}

package/lib/intakeworker.js

@@ -0,0 +1,22 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+import { Worker } from './worker.js'
+
+export class IntakeWorker extends Worker {
+  #deliverer
+  #bots
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.deliverer)
+    assert.ok(options.bots)
+    this.#deliverer = options.deliverer
+    this.#bots = options.bots
+  }
+
+  async doJob (payload, attempts) {
+    const raw = payload.activity
+    const activity = await as2.import(raw)
+    await this.#deliverer.deliverToAll(activity, this.#bots)
+  }
+}

package/lib/objectstorage.js

@@ -11,7 +11,7 @@ export class NoSuchObjectError extends Error {
 
 export class ObjectStorage {
   #connection = null
-  static #MAX_ITEMS_PER_PAGE = 20
+  static #MAX_ITEMS_PER_PAGE = 256
 
   constructor (connection) {
     this.#connection = connection

package/lib/routes/sharedinbox.js

@@ -69,7 +69,7 @@ router.post('/shared/inbox', async (req, res, next) => {
 
   logger.info(`Activity ${activity.id} received at shared inbox`)
 
-  await deliverer.deliverToAll(activity, bots)
+  await deliverer.intake(activity, subject)
 
   res.status(202)
   res.type('text/plain')

package/lib/worker.js

@@ -0,0 +1,86 @@
+import assert from 'node:assert'
+import { nanoid } from 'nanoid'
+
+export class RecoverableError extends Error {
+  delay = 1000
+}
+
+export class Worker {
+  #jobQueue
+  #logger
+  #running
+  #workerId
+
+  constructor (jobQueue, logger, options = {}) {
+    this.#jobQueue = jobQueue
+    this.#workerId = nanoid()
+    this.#logger = logger.child({
+      class: this.constructor.name,
+      worker: this.#workerId
+    })
+  }
+
+  get _logger () {
+    return this.#logger
+  }
+
+  get queueId () {
+    const name = this.constructor.name
+    return name.slice(0, name.length - 6).toLowerCase()
+  }
+
+  async run () {
+    this.#running = true
+    while (this.#running) {
+      let jobId
+      let payload
+      let attempts
+      try {
+        this.#logger.debug('dequeueing');
+        ({ jobId, payload, attempts } = await this.#jobQueue.dequeue(
+          this.queueId,
+          this.#workerId
+        ))
+        this.#logger.debug({ jobId, payload, attempts }, 'got a job')
+      } catch (err) {
+        if (err?.name === 'AbortError') {
+          this.#logger.info('Worker received abort signal')
+          break
+        } else {
+          this.#logger.warn({ err, jobId }, 'error before job, retrying')
+          if (jobId) {
+            const delay = this.#retryDelay(attempts ?? 1)
+            await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+          }
+          continue
+        }
+      }
+
+      assert.ok(jobId)
+      assert.ok(payload)
+
+      try {
+        await this.doJob(payload, attempts)
+        await this.#jobQueue.complete(jobId, this.#workerId)
+      } catch (err) {
+        if (err instanceof RecoverableError) {
+          await this.#jobQueue.retryAfter(jobId, this.#workerId, err.delay)
+        } else {
+          await this.#jobQueue.fail(jobId, this.#workerId)
+        }
+      }
+    }
+  }
+
+  async doJob (payload, attempts) {
+    throw new Error('Must implement doJob')
+  }
+
+  stop () {
+    this.#running = false
+  }
+
+  #retryDelay (attempts) {
+    return Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.34.1",
+  "version": "0.35.0",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",