@evanp/activitypub-bot 0.21.0 → 0.21.2

package/lib/activitydeliverer.js

@@ -1,7 +1,6 @@
 import BotMaker from './botmaker.js'
 import assert from 'node:assert'
 import as2 from './activitystreams.js'
-import PQueue from 'p-queue'
 import * as ttlcachePkg from '@isaacs/ttlcache'
 
 const TTLCache =
@@ -21,27 +20,28 @@ const COLLECTION_TYPES = [
 ]
 
 export class ActivityDeliverer {
-  static #MAX_ALL_QUEUE = 16
-  static #MAX_BOT_QUEUE = 32
+  static #QUEUE_ID = 'delivery'
   static #TTL_SEEN_PUBLIC = 3 * 24 * 60 * 60 * 1000 // 3 days
   static #MAX_SEEN_PUBLIC = 1000000
   #actorStorage
-  #activityHandler
   #formatter
   #logger
   #client
-  #deliverAllQueue
-  #deliverBotQueue
+  #jobQueue
   #seenPublic
 
-  constructor (actorStorage, activityHandler, formatter, logger, client) {
+  constructor (actorStorage, formatter, logger, client, jobQueue) {
+    assert.strictEqual(typeof actorStorage, 'object')
+    assert.strictEqual(typeof formatter, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    assert.strictEqual(typeof client, 'object')
+    assert.strictEqual(typeof jobQueue, 'object')
+
     this.#actorStorage = actorStorage
-    this.#activityHandler = activityHandler
     this.#formatter = formatter
     this.#client = client
     this.#logger = logger.child({ class: this.constructor.name })
-    this.#deliverAllQueue = new PQueue({ max: ActivityDeliverer.#MAX_ALL_QUEUE })
-    this.#deliverBotQueue = new PQueue({ max: ActivityDeliverer.#MAX_BOT_QUEUE })
+    this.#jobQueue = jobQueue
     this.#seenPublic = new TTLCache({
       ttl: ActivityDeliverer.#TTL_SEEN_PUBLIC,
       max: ActivityDeliverer.#MAX_SEEN_PUBLIC
@@ -96,40 +96,20 @@ export class ActivityDeliverer {
   }
 
   async deliverTo (activity, bot) {
-    this.#deliverBotQueue.add(() =>
-      this.#deliverTo(activity, bot)
+    await this.#jobQueue.enqueue(
+      ActivityDeliverer.#QUEUE_ID,
+      { botUsername: bot.username, activity: await activity.export() }
     )
   }
 
   async onIdle () {
     this.#logger.debug('Awaiting delivery queues')
-    await this.#deliverAllQueue.onIdle()
-    await this.#deliverBotQueue.onIdle()
+    await this.#jobQueue.onIdle(ActivityDeliverer.#QUEUE_ID)
     this.#logger.debug('Done awaiting delivery queues')
   }
 
-  async #deliverTo (activity, bot) {
-    if (await this.#actorStorage.isInCollection(bot.username, 'inbox', activity)) {
-      this.#logger.info(
-        { activity: activity.id, username: bot.username },
-        'skipping redelivery for activity already in the inbox'
-      )
-      return
-    }
-    try {
-      await this.#activityHandler.handleActivity(bot, activity)
-    } catch (err) {
-      this.#logger.warn(err)
-    }
-    this.#logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
-    await this.#actorStorage.addToCollection(bot.username, 'inbox', activity)
-    this.#logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
-  }
-
   async deliverToAll (activity, bots) {
-    this.#deliverAllQueue.add(() =>
-      this.#deliverToAll(activity, bots)
-    )
+    await this.#deliverToAll(activity, bots)
   }
 
   async #deliverToAll (activity, bots) {

package/lib/activitydistributor.js

@@ -1,8 +1,6 @@
 import assert from 'node:assert'
 import as2 from './activitystreams.js'
 import { LRUCache } from 'lru-cache'
-import PQueue from 'p-queue'
-import { setTimeout } from 'node:timers/promises'
 
 const NS = 'https://www.w3.org/ns/activitystreams#'
 
@@ -12,9 +10,9 @@ const COLLECTION_TYPES = [
 ]
 
 export class ActivityDistributor {
+  static #DISTRIBUTION_QUEUE_ID = 'distribution'
+  static #DELIVERY_QUEUE_ID = 'delivery'
   static #MAX_CACHE_SIZE = 1000000
-  static #CONCURRENCY = 32
-  static #MAX_ATTEMPTS = 16
   static #PUBLIC = [
     'https://www.w3.org/ns/activitystreams#Public',
     'as:Public',
@@ -26,23 +24,27 @@ export class ActivityDistributor {
   #actorStorage = null
   #directInboxCache = null
   #sharedInboxCache = null
-  #queue = null
-  #retryQueue = null
   #logger = null
-
-  constructor (client, formatter, actorStorage, logger = null) {
+  #jobQueue = null
+
+  constructor (client, formatter, actorStorage, logger, jobQueue) {
+    assert.ok(client)
+    assert.ok(formatter)
+    assert.ok(actorStorage)
+    assert.ok(logger)
+    assert.ok(jobQueue)
     this.#client = client
     this.#formatter = formatter
     this.#actorStorage = actorStorage
     this.#logger = logger.child({ class: this.constructor.name })
+    this.#jobQueue = jobQueue
     this.#directInboxCache = new LRUCache({ max: ActivityDistributor.#MAX_CACHE_SIZE })
     this.#sharedInboxCache = new LRUCache({ max: ActivityDistributor.#MAX_CACHE_SIZE })
-    this.#queue = new PQueue({ concurrency: ActivityDistributor.#CONCURRENCY })
-    this.#retryQueue = new PQueue()
   }
 
   async distribute (activity, username) {
     const stripped = await this.#strip(activity)
+    const raw = await stripped.export()
     const actorId = this.#formatter.format({ username })
 
     const delivered = new Set()
@@ -52,8 +54,11 @@ export class ActivityDistributor {
       if (await this.#isLocal(recipient)) {
         if (recipient !== actorId && !localDelivered.has(recipient)) {
           localDelivered.add(recipient)
-          this.#queue.add(() =>
-            this.#deliverLocal(recipient, stripped, username))
+          const parts = this.#formatter.unformat(recipient)
+          await this.#jobQueue.enqueue(
+            ActivityDistributor.#DELIVERY_QUEUE_ID,
+            { botUsername: parts.username, activity: raw }
+          )
         }
       } else {
         const inbox = await this.#getInbox(recipient, username)
@@ -61,8 +66,9 @@ export class ActivityDistributor {
           this.#logger.warn({ id: recipient.id }, 'No inbox')
         } else if (!delivered.has(inbox)) {
           delivered.add(inbox)
-          this.#queue.add(() =>
-            this.#deliver(inbox, stripped, username)
+          await this.#jobQueue.enqueue(
+            ActivityDistributor.#DISTRIBUTION_QUEUE_ID,
+            { inbox, activity: raw, username }
           )
         }
       }
@@ -72,8 +78,11 @@ export class ActivityDistributor {
       if (await this.#isLocal(recipient)) {
         if (recipient !== actorId && !localDelivered.has(recipient)) {
           localDelivered.add(recipient)
-          this.#queue.add(() =>
-            this.#deliverLocal(recipient, stripped, username))
+          const parts = this.#formatter.unformat(recipient)
+          await this.#jobQueue.enqueue(
+            ActivityDistributor.#DELIVERY_QUEUE_ID,
+            { botUsername: parts.username, activity: raw }
+          )
         }
       } else {
         const inbox = await this.#getDirectInbox(recipient, username)
@@ -81,8 +90,9 @@ export class ActivityDistributor {
           this.#logger.warn({ id: recipient.id }, 'No direct inbox')
         } else if (!delivered.has(inbox)) {
           delivered.add(inbox)
-          this.#queue.add(() =>
-            this.#deliver(inbox, stripped, username)
+          await this.#jobQueue.enqueue(
+            ActivityDistributor.#DISTRIBUTION_QUEUE_ID,
+            { inbox, activity: raw, username }
           )
         }
       }
@@ -90,8 +100,8 @@ export class ActivityDistributor {
   }
 
   async onIdle () {
-    await this.#retryQueue.onIdle()
-    await this.#queue.onIdle()
+    await this.#jobQueue.onIdle(ActivityDistributor.#DELIVERY_QUEUE_ID)
+    await this.#jobQueue.onIdle(ActivityDistributor.#DISTRIBUTION_QUEUE_ID)
   }
 
   async * #public (activity, username) {
@@ -237,58 +247,10 @@ export class ActivityDistributor {
     return await as2.import(exported)
   }
 
-  async #deliver (inbox, activity, username, attempt = 1) {
-    try {
-      await this.#client.post(inbox, activity, username)
-      this.#logInfo(`Delivered ${activity.id} to ${inbox}`)
-    } catch (error) {
-      if (!error.status) {
-        this.#logError(`Could not deliver ${activity.id} to ${inbox}: ${error.message}`)
-        this.#logError(error.stack)
-      } else if (error.status >= 300 && error.status < 400) {
-        this.#logError(`Unexpected redirect code delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
-      } else if (error.status >= 400 && error.status < 500) {
-        this.#logError(`Bad request delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
-      } else if (error.status >= 500 && error.status < 600) {
-        if (attempt >= ActivityDistributor.#MAX_ATTEMPTS) {
-          this.#logError(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; giving up after ${attempt} attempts`)
-        }
-        const delay = Math.round((2 ** (attempt - 1) * 1000) * (0.5 + Math.random()))
-        this.#logWarning(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; will retry in ${delay} ms (${attempt} of ${ActivityDistributor.#MAX_ATTEMPTS})`)
-        this.#retryQueue.add(() => setTimeout(delay).then(() => this.#deliver(inbox, activity, username, attempt + 1)))
-      }
-    }
-  }
-
-  #logError (message) {
-    if (this.#logger) {
-      this.#logger.error(message)
-    }
-  }
-
-  #logWarning (message) {
-    if (this.#logger) {
-      this.#logger.warn(message)
-    }
-  }
-
-  #logInfo (message) {
-    if (this.#logger) {
-      this.#logger.info(message)
-    }
-  }
-
   #isLocal (id) {
     return this.#formatter.isLocal(id)
   }
 
-  async #deliverLocal (id, activity) {
-    const username = this.#formatter.getUserName(id)
-    if (username) {
-      await this.#actorStorage.addToCollection(username, 'inbox', activity)
-    }
-  }
-
   #isLocalActor (parts) {
     return parts.username && !parts.type && !parts.collection
   }

package/lib/app.js

@@ -29,8 +29,12 @@ import { HTTPSignatureAuthenticator } from './httpsignatureauthenticator.js'
 import { Digester } from './digester.js'
 import { runMigrations } from './migrations/index.js'
 import { ActivityDeliverer } from './activitydeliverer.js'
+import { JobQueue } from './jobqueue.js'
+import { JobReaper } from './jobreaper.js'
+import { DeliveryWorker } from './deliveryworker.js'
+import { DistributionWorker } from './distributionworker.js'
 
-export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
+export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8) {
   const logger = Logger({
     level: logLevel
   })
@@ -50,11 +54,13 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
+  const jobQueue = new JobQueue(connection, logger)
   const distributor = new ActivityDistributor(
     client,
     formatter,
     actorStorage,
-    logger
+    logger,
+    jobQueue
   )
   const authorizer = new Authorizer(actorStorage, formatter, client)
   const cache = new ObjectCache({
@@ -74,10 +80,10 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
   )
   const deliverer = new ActivityDeliverer(
     actorStorage,
-    activityHandler,
     formatter,
     logger,
-    client
+    client,
+    jobQueue
   )
 
   // TODO: Make an endpoint for tagged objects
@@ -99,6 +105,41 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     ))
   )
 
+  const deliveryWorkers = new Array(deliveryWorkerCount)
+  const deliveryWorkerRuns = new Array(deliveryWorkerCount)
+
+  for (let i = 0; i < deliveryWorkers.length; i++) {
+    deliveryWorkers[i] = new DeliveryWorker(
+      jobQueue,
+      actorStorage,
+      activityHandler,
+      logger,
+      bots
+    )
+    deliveryWorkerRuns[i] = deliveryWorkers[i].run().catch((err) => {
+      logger.error({ err, workerIndex: i }, 'unexpected error in delivery worker')
+    })
+  }
+
+  const distributionWorkers = new Array(distributionWorkerCount)
+  const distributionWorkerRuns = new Array(distributionWorkerCount)
+
+  for (let i = 0; i < distributionWorkers.length; i++) {
+    distributionWorkers[i] = new DistributionWorker(
+      jobQueue,
+      client,
+      logger
+    )
+    distributionWorkerRuns[i] = distributionWorkers[i].run().catch((err) => {
+      logger.error({ err, workerIndex: i }, 'unexpected error in distribution worker')
+    })
+  }
+
+  const jobReaper = new JobReaper(jobQueue, logger)
+  const jobReaperRun = jobReaper.run().catch((err) => {
+    logger.error({ err }, 'unexpected error in job reaper')
+  })
+
   const app = express()
 
   app.locals = {
@@ -117,7 +158,8 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     bots,
     activityHandler,
     origin,
-    deliverer
+    deliverer,
+    deliveryWorkers
   }
 
   app.use(HTTPLogger({
@@ -196,10 +238,20 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
 
   app.cleanup = async () => {
     logger.info('Closing app')
-    logger.info('Waiting for distributor queue')
-    await distributor.onIdle()
+    for (const worker of deliveryWorkers) {
+      worker.stop()
+    }
+    for (const worker of distributionWorkers) {
+      worker.stop()
+    }
+    jobReaper.stop()
+    jobQueue.abort()
+    await Promise.allSettled(deliveryWorkerRuns)
+    await Promise.allSettled(distributionWorkerRuns)
+    await jobReaperRun
     logger.info('Closing database connection')
     await connection.close()
+    logger.info('Done')
   }
 
   return app

package/lib/authorizer.js

@@ -1,14 +1,26 @@
 import as2 from './activitystreams.js'
 
+const NS = 'https://www.w3.org/ns/activitystreams#'
+
+const PUBLICS = [
+  `${NS}Public`,
+  'as:Public',
+  'Public'
+]
+
+const COLLECTION_TYPES = [
+  `${NS}Collection`,
+  `${NS}OrderedCollection`
+]
+
 export class Authorizer {
-  #PUBLIC = 'https://www.w3.org/ns/activitystreams#Public'
   #actorStorage = null
   #formatter = null
-  #activityPubClient = null
-  constructor (actorStorage, formatter, activityPubClient) {
+  #client = null
+  constructor (actorStorage, formatter, client) {
     this.#actorStorage = actorStorage
     this.#formatter = formatter
-    this.#activityPubClient = activityPubClient
+    this.#client = client
   }
 
   async canRead (actor, object) {
@@ -40,7 +52,7 @@ export class Authorizer {
   async #canReadLocal (actor, object) {
     const recipients = this.#getRecipients(object)
     if (!actor) {
-      return recipients.has(this.#PUBLIC)
+      return PUBLICS.some(id => recipients.has(id))
     }
     const ownerId = (await this.#getOwner(object))?.id
     if (!ownerId) {
@@ -63,25 +75,38 @@ export class Authorizer {
     if (recipients.has(actor.id)) {
       return true
     }
-    if (recipients.has(this.#PUBLIC)) {
+    if (PUBLICS.some(id => recipients.has(id))) {
       return true
     }
-    const followers = this.#formatter.format({ username: ownerName, collection: 'followers' })
-    if (recipients.has(followers) && await this.#actorStorage.isInCollection(ownerName, 'followers', actor)) {
-      return true
+
+    const lcolls = this.#getLocalCollections(recipients)
+
+    for (const id of lcolls) {
+      if (await this.#isInLocalCollection(actor, id)) {
+        return true
+      }
+    }
+
+    const rcolls = await this.#getRemoteCollections(recipients, ownerName)
+
+    for (const id of rcolls) {
+      if (await this.#isInRemoteCollection(actor, id, ownerName)) {
+        return true
+      }
     }
+
     return false
   }
 
   async #canReadRemote (actor, object) {
     const recipients = this.#getRecipients(object)
     if (!actor) {
-      return recipients.has(this.#PUBLIC)
+      return PUBLICS.some(id => recipients.has(id))
     }
     if (recipients.has(actor.id)) {
       return true
     }
-    if (recipients.has(this.#PUBLIC)) {
+    if (PUBLICS.some(id => recipients.has(id))) {
       return true
     }
     // TODO: check if it's to followers, actor is local, and actor
@@ -137,4 +162,82 @@ export class Authorizer {
     }
     return recipientIds
   }
+
+  #getLocalCollections (recipients) {
+    const lcolls = new Set()
+    for (const recipient of recipients) {
+      if (this.#isLocalCollection(recipient)) {
+        lcolls.add(recipient)
+      }
+    }
+    return lcolls
+  }
+
+  #isLocalCollection (recipient) {
+    if (!this.#formatter.isLocal(recipient)) {
+      return false
+    } else {
+      const parts = this.#formatter.unformat(recipient)
+      return !!(parts.username && parts.collection && !parts.type)
+    }
+  }
+
+  #isInLocalCollection (actor, recipient) {
+    const parts = this.#formatter.unformat(recipient)
+    return this.#actorStorage.isInCollection(
+      parts.username,
+      parts.collection,
+      actor
+    )
+  }
+
+  async #getRemoteCollections (recipients, ownerName) {
+    const rcolls = new Set()
+    for (const recipient of recipients) {
+      if (await this.#isRemoteCollection(recipient, ownerName)) {
+        rcolls.add(recipient)
+      }
+    }
+    return rcolls
+  }
+
+  async #isRemoteCollection (recipient, ownerName) {
+    if (this.#formatter.isLocal(recipient)) {
+      return false
+    } else {
+      const obj = await this.#client.get(recipient, ownerName)
+
+      return (Array.isArray(obj.type))
+        ? obj.type.some(item => COLLECTION_TYPES.includes(item))
+        : COLLECTION_TYPES.includes(obj.type)
+    }
+  }
+
+  async #isInRemoteCollection (actor, id, ownerName) {
+    const coll = await this.#client.get(id, ownerName)
+    const collOwner = await this.#getOwner(coll)
+    const collOwnerFull = await this.#client.get(collOwner.id, ownerName)
+
+    // Special case for followers, following collections, since we track
+    // that information locally, too
+
+    if (this.#formatter.isLocal(actor.id)) {
+      const { username } = this.#formatter.unformat(actor.id)
+      if (coll.id === collOwnerFull.followers.first.id) {
+        return await this.#actorStorage.isInCollection(username, 'following', collOwnerFull)
+      } else if (coll.id === collOwnerFull.following.first.id) {
+        return await this.#actorStorage.isInCollection(username, 'followers', collOwnerFull)
+      }
+    }
+
+    // Worst case!
+
+    for await (const item of this.#client.items(coll.id, ownerName)) {
+      if (item.id === actor.id) {
+        return true
+      }
+    }
+
+    return false
+  }
 }

package/lib/botcontext.js

@@ -369,10 +369,12 @@ export class BotContext {
     const owners = obj.attributedTo
       ? Array.from(obj.attributedTo).map((owner) => owner.id)
       : Array.from(obj.actor).map((owner) => owner.id)
+    const summary = `${this.#botId} shared "${await this.#nameOf(obj)}"`
     const activity = await this.#doActivity({
       type: 'Announce',
-      summary: {
-        en: `${this.#botId} shared "${await this.#nameOf(obj)}"`
+      summary,
+      summaryMap: {
+        en: summary
       },
       object: obj.id,
       to: [
@@ -380,7 +382,7 @@ export class BotContext {
           username: this.#botId,
           collection: 'followers'
         }),
-        'https://www.w3.org/ns/activitystreams#Public'
+        'as:Public'
       ],
       cc: owners
     })

package/lib/deliveryworker.js

@@ -0,0 +1,96 @@
+import assert from 'node:assert'
+
+import { nanoid } from 'nanoid'
+
+import as2 from './activitystreams.js'
+import BotMaker from './botmaker.js'
+
+export class DeliveryWorker {
+  static #QUEUE_ID = 'delivery'
+  #queue
+  #actorStorage
+  #activityHandler
+  #logger
+  #running
+  #workerId
+  #bots
+
+  constructor (queue, actorStorage, activityHandler, logger, bots) {
+    this.#queue = queue
+    this.#actorStorage = actorStorage
+    this.#activityHandler = activityHandler
+    this.#workerId = nanoid()
+    this.#logger = logger.child({
+      class: this.constructor.name,
+      worker: this.#workerId
+    })
+    this.#bots = bots
+  }
+
+  async run () {
+    this.#running = true
+    while (this.#running) {
+      let jobId
+      let payload
+      try {
+        this.#logger.debug('dequeueing');
+        ({ jobId, payload } = await this.#queue.dequeue(
+          DeliveryWorker.#QUEUE_ID,
+          this.#workerId
+        ))
+        this.#logger.debug({ jobId, payload }, 'got a job')
+        const activity = await as2.import(payload.activity)
+        assert.ok(payload.botUsername)
+        const bot = await BotMaker.makeBot(this.#bots, payload.botUsername)
+        assert.ok(bot)
+        assert.ok(bot.username)
+        this.#logger.debug(
+          { bot: bot.username, activity: activity.id, jobId },
+          'delivering to bot'
+        )
+        await this.#deliverTo(activity, bot)
+        this.#logger.debug({ bot: bot.username, activity: activity.id, jobId }, 'done')
+        await this.#queue.complete(jobId, this.#workerId)
+      } catch (err) {
+        if (err?.name === 'AbortError') {
+          this.#logger.info('Worker received abort signal')
+          break
+        }
+        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
+        if (jobId) {
+          await this.#queue.release(jobId, this.#workerId)
+        }
+      }
+    }
+  }
+
+  stop () {
+    this.#running = false
+  }
+
+  async #deliverTo (activity, bot) {
+    assert.ok(activity)
+    assert.strictEqual(typeof activity, 'object')
+    assert.ok(bot)
+    assert.strictEqual(typeof bot, 'object')
+    assert.strictEqual(typeof bot.username, 'string')
+    if (await this.#actorStorage.isInCollection(bot.username, 'inbox', activity)) {
+      this.#logger.info(
+        { activity: activity.id, username: bot.username },
+        'skipping redelivery for activity already in the inbox'
+      )
+      return
+    }
+    try {
+      await this.#activityHandler.handleActivity(bot, activity)
+    } catch (err) {
+      this.#logger.warn(
+        { err, activity: activity.id, bot: bot.username },
+        'handler failed for activity'
+      )
+    }
+    this.#logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
+    await this.#actorStorage.addToCollection(bot.username, 'inbox', activity)
+    this.#logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
+  }
+}

package/lib/distributionworker.js

@@ -0,0 +1,79 @@
+import { nanoid } from 'nanoid'
+
+import as2 from './activitystreams.js'
+
+export class DistributionWorker {
+  static #QUEUE_ID = 'distribution'
+  static #MAX_ATTEMPTS = 16
+  #jobQueue
+  #client
+  #logger
+  #running
+  #workerId
+
+  constructor (jobQueue, client, logger) {
+    this.#jobQueue = jobQueue
+    this.#client = client
+    this.#workerId = nanoid()
+    this.#logger = logger.child({
+      class: this.constructor.name,
+      workerId: this.#workerId
+    })
+  }
+
+  async run () {
+    this.#running = true
+    while (this.#running) {
+      let jobId
+      let payload
+      let attempts
+      try {
+        this.#logger.debug('dequeueing');
+        ({ jobId, payload, attempts } = await this.#jobQueue.dequeue(
+          DistributionWorker.#QUEUE_ID,
+          this.#workerId
+        ))
+        this.#logger.debug({ jobId, payload, attempts }, 'got a job')
+        const { inbox, activity, username } = payload
+        const activityObj = await as2.import(activity)
+        try {
+          await this.#client.post(inbox, activityObj, username)
+          this.#logger.info(`Delivered ${activity.id} to ${inbox}`)
+          await this.#jobQueue.complete(jobId, this.#workerId)
+          this.#logger.info({ jobId }, 'completed job')
+        } catch (error) {
+          if (!error.status) {
+            this.#logger.error(`Could not deliver ${activity.id} to ${inbox}: ${error.message}`)
+            this.#logger.error(error.stack)
+          } else if (error.status >= 300 && error.status < 400) {
+            this.#logger.error(`Unexpected redirect code delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
+          } else if (error.status >= 400 && error.status < 500) {
+            this.#logger.error(`Bad request delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
+          } else if (error.status >= 500 && error.status < 600) {
+            if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
+              this.#logger.error(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; giving up after ${attempts} attempts`)
+              await this.#jobQueue.complete(jobId, this.#workerId)
+            } else {
+              const delay = Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
+              this.#logger.warn(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; will retry in ${delay} ms (${attempts} of ${DistributionWorker.#MAX_ATTEMPTS})`)
+              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+            }
+          }
+        }
+      } catch (err) {
+        if (err?.name === 'AbortError') {
+          this.#logger.info('Worker received abort signal')
+          break
+        }
+        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
+        if (jobId) {
+          await this.#jobQueue.release(jobId, this.#workerId)
+        }
+      }
+    }
+  }
+
+  stop () {
+    this.#running = false
+  }
+}

package/lib/jobqueue.js

@@ -0,0 +1,224 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import assert from 'node:assert'
+
+import { nanoid } from 'nanoid'
+import { QueryTypes } from 'sequelize'
+
+const MAX_DELAY = 30000
+const DEFAULT_PRIORITY = 1000000
+
+export class JobQueue {
+  #connection
+  #logger
+  #ac
+  #wakeAc
+
+  constructor (connection, logger) {
+    this.#connection = connection
+    this.#logger = logger.child({ class: this.constructor.name })
+    this.#ac = new AbortController()
+    this.#wakeAc = new AbortController()
+  }
+
+  async enqueue (queueId, payload = {}, options = { priority: DEFAULT_PRIORITY }) {
+    const { priority } = options
+    const jobId = nanoid()
+    const encoded = JSON.stringify(payload)
+    await this.#connection.query(
+      `INSERT INTO job (job_id, queue_id, payload, priority)
+      VALUES (?, ?, ?, ?);`,
+      { replacements: [jobId, queueId, encoded, priority] }
+    )
+    this.#logger.debug({ method: 'enqueue', queueId, jobId }, 'enqueued job')
+    const old = this.#wakeAc
+    this.#wakeAc = new AbortController()
+    old.abort()
+    return jobId
+  }
+
+  async dequeue (queueId, jobRunnerId) {
+    let res
+    let delay = 50
+
+    this.#logger.debug(
+      { method: 'dequeue', queueId, jobRunnerId },
+      'dequeueing job'
+    )
+
+    while (!res) {
+      res = await this.#claimNextJob(queueId, jobRunnerId)
+      if (res) break
+      delay = Math.min(delay * 2, MAX_DELAY)
+      this.#logger.debug({ method: 'dequeue', delay }, 'sleeping')
+      try {
+        await sleep(delay, null, { signal: AbortSignal.any([this.#ac.signal, this.#wakeAc.signal]) })
+      } catch (err) {
+        if (this.#ac.signal.aborted) throw err
+        delay = 50
+      }
+    }
+
+    const jobId = res.job_id
+    const payload = JSON.parse(res.payload)
+    const attempts = res.attempts
+
+    this.#logger.debug({ method: 'dequeue', queueId, jobId }, 'dequeued job')
+    return { jobId, payload, attempts }
+  }
+
+  async complete (jobId, jobRunnerId) {
+    await this.#connection.query(`
+      DELETE FROM job
+      WHERE job_id = ? AND claimed_by = ?;`,
+    { replacements: [jobId, jobRunnerId] }
+    )
+    this.#logger.debug({ method: 'complete', jobRunnerId, jobId }, 'completed job')
+  }
+
+  async release (jobId, jobRunnerId) {
+    await this.#connection.query(`
+      UPDATE job
+      SET claimed_by = NULL,
+          claimed_at = NULL,
+          updated_at = CURRENT_TIMESTAMP
+      WHERE job_id = ? AND claimed_by = ?;`,
+    { replacements: [jobId, jobRunnerId] }
+    )
+    this.#logger.debug({ method: 'release', jobRunnerId, jobId }, 'released job')
+  }
+
+  async retryAfter (jobId, jobRunnerId, delay) {
+    assert.ok(jobId)
+    assert.strictEqual(typeof jobId, 'string')
+    assert.ok(jobRunnerId)
+    assert.strictEqual(typeof jobRunnerId, 'string')
+    assert.ok(delay)
+    assert.strictEqual(typeof delay, 'number')
+
+    const retryAfter = new Date(Date.now() + delay)
+
+    await this.#connection.query(`
+      UPDATE job
+      SET claimed_by = NULL,
+          claimed_at = NULL,
+          updated_at = CURRENT_TIMESTAMP,
+          retry_after = ?
+      WHERE job_id = ? AND claimed_by = ?;`,
+    { replacements: [retryAfter, jobId, jobRunnerId] }
+    )
+    this.#logger.debug(
+      { method: 'retry', jobRunnerId, jobId, delay },
+      'released job'
+    )
+  }
+
+  async onIdle (queueId) {
+    assert.ok(queueId)
+    assert.strictEqual(typeof queueId, 'string')
+
+    let delay = 50
+
+    this.#logger.debug({ method: 'onIdle', queueId }, 'getting queue size')
+
+    let jobCount = await this.#countJobs(queueId)
+
+    this.#logger.debug({ method: 'onIdle', queueId, jobCount }, 'got queue size')
+
+    while (jobCount > 0) {
+      delay = Math.min(delay * 2, MAX_DELAY)
+      this.#logger.debug({ method: 'onIdle', delay }, 'sleeping')
+      await sleep(delay, null, { signal: this.#ac.signal })
+      jobCount = await this.#countJobs(queueId)
+      this.#logger.debug({ method: 'onIdle', queueId, jobCount }, 'got queue size')
+    }
+    this.#logger.debug({ method: 'onIdle', queueId }, 'Now idle')
+  }
+
+  async sweep (timeout) {
+    const cutoff = new Date(Date.now() - timeout)
+    const [, meta] = await this.#connection.query(`
+      UPDATE job
+      SET claimed_by = NULL,
+          claimed_at = NULL,
+          updated_at = CURRENT_TIMESTAMP
+      WHERE claimed_at < ?;`,
+    { replacements: [cutoff] }
+    )
+    const count = meta?.changes ?? meta?.rowCount ?? 0
+    if (count > 0) {
+      this.#logger.warn({ method: 'sweep', count }, 'released stalled jobs')
+    } else {
+      this.#logger.debug({ method: 'sweep' }, 'no stalled jobs found')
+    }
+  }
+
+  abort () {
+    this.#logger.debug('Aborting queue server')
+    this.#ac.abort()
+  }
+
+  async #countJobs (queueId) {
+    this.#logger.debug({ method: '#countJobs', queueId }, 'checking queue size')
+    const rows = await this.#connection.query(`
+      SELECT COUNT(*) as job_count
+      FROM JOB
+      WHERE queue_id = ?;
+    `, { replacements: [queueId] })
+    if (rows[0]) {
+      const size = rows[0][0].job_count
+      this.#logger.debug({ method: '#countJobs', queueId, size }, 'got queue size')
+      return size
+    } else {
+      this.#logger.debug({ method: '#countJobs', queueId }, 'no queue size')
+      return 0
+    }
+  }
+
+  async #claimNextJob (queueId, jobRunnerId) {
+    this.#logger.debug(
+      { method: '#claimNextJob', queueId, jobRunnerId },
+      'claiming next job'
+    )
+    const skipLocked = this.#connection.getDialect() === 'postgres' ? 'FOR UPDATE SKIP LOCKED' : ''
+    const rows = await this.#connection.query(`
+      UPDATE job
+      SET claimed_by = ?,
+          claimed_at = CURRENT_TIMESTAMP,
+          updated_at = CURRENT_TIMESTAMP,
+          attempts = attempts + 1
+      WHERE job_id = (
+        SELECT job_id
+        FROM job
+        WHERE queue_id = ?
+          AND (
+            claimed_by IS NULL
+          )
+          AND (
+            retry_after IS NULL OR
+            retry_after < ?
+          )
+        ORDER BY priority ASC, created_at ASC
+        LIMIT 1
+        ${skipLocked}
+      )
+      RETURNING job_id, payload, attempts;`,
+    {
+      replacements: [jobRunnerId, queueId, new Date()],
+      type: QueryTypes.SELECT
+    }
+    )
+    if (rows[0]) {
+      this.#logger.debug(
+        { method: '#claimNextJob', queueId, jobRunnerId, jobId: rows[0].job_id },
+        'got a job'
+      )
+      return rows[0]
+    } else {
+      this.#logger.debug(
+        { method: '#claimNextJob', queueId, jobRunnerId },
+        'no job found'
+      )
+      return null
+    }
+  }
+}

package/lib/jobreaper.js

@@ -0,0 +1,38 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+
+const DEFAULT_TIMEOUT = 5 * 60 * 1000   // 5 minutes
+const DEFAULT_INTERVAL = 60 * 1000       // 1 minute
+
+export class JobReaper {
+  #jobQueue
+  #logger
+  #timeout
+  #interval
+  #ac
+
+  constructor (jobQueue, logger, { timeout = DEFAULT_TIMEOUT, interval = DEFAULT_INTERVAL } = {}) {
+    this.#jobQueue = jobQueue
+    this.#logger = logger.child({ class: this.constructor.name })
+    this.#timeout = timeout
+    this.#interval = interval
+    this.#ac = new AbortController()
+  }
+
+  async run () {
+    this.#logger.debug('JobReaper started')
+    while (!this.#ac.signal.aborted) {
+      try {
+        await sleep(this.#interval, null, { signal: this.#ac.signal })
+      } catch {
+        break
+      }
+      await this.#jobQueue.sweep(this.#timeout)
+    }
+    this.#logger.debug('JobReaper stopped')
+  }
+
+  stop () {
+    this.#logger.debug('Stopping JobReaper')
+    this.#ac.abort()
+  }
+}

package/lib/migrations/003-jobqueue.js

@@ -0,0 +1,20 @@
+export const id = '003-jobqueue'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE job (
+      job_id char(21) NOT NULL PRIMARY KEY,
+      queue_id varchar(64) NOT NULL,
+      priority INTEGER,
+      payload TEXT,
+      claimed_at TIMESTAMP,
+      claimed_by varchar(64),
+      attempts INTEGER default 0,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+    );
+  `, queryOptions)
+  await connection.query(`
+    CREATE INDEX job_queue_id on job (queue_id);
+  `, queryOptions)
+}

package/lib/migrations/004-retry-after.js

@@ -0,0 +1,8 @@
+export const id = '003-jobqueue'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    ALTER TABLE job
+    ADD COLUMN retry_after TIMESTAMP;
+  `, queryOptions)
+}

package/lib/migrations/index.js

@@ -1,10 +1,19 @@
-import { id as initialId, up as initialUp } from './001-initial.js'
-import { id as lastId, up as lastUp } from './002-last-activity.js'
+import { fileURLToPath } from 'url'
+import { dirname, resolve } from 'path'
+import { readdirSync } from 'fs'
 
-const migrations = [
-  { id: initialId, up: initialUp },
-  { id: lastId, up: lastUp }
-]
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+const migrationFiles = readdirSync(__dirname)
+  .filter(file => /^\d{3}-.*\.js$/.test(file))
+  .sort()
+
+const migrations = await Promise.all(
+  migrationFiles.map(async (file) => {
+    const module = await import(resolve(__dirname, file))
+    return { id: module.id, up: module.up }
+  })
+)
 
 async function runMigrationsInternal (connection, queryOptions = {}) {
   await connection.query(`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.21.0",
+  "version": "0.21.2",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",