@evanp/activitypub-bot 0.34.1 → 0.36.0

package/README.md

@@ -58,6 +58,12 @@ Options:
   --port <number>            Port to listen on
   --bots-config-file <path>  Path to bots config module
   --log-level <level>        Log level (e.g., info, debug)
+  --delivery <number>        Number of background delivery workers
+  --distribution <number>    Number of background distribution workers
+  --fanout <number>          Number of background fanout workers
+  --intake <number>          Number of background intake workers
+  --index-file <path>        HTML page to show at root path
+  --profile-file <path>      HTML page to show for bot profiles
   -h, --help                 Show this help
 ```
 
@@ -109,11 +115,29 @@ The number of background distribution workers to run for this server.  These wor
 
 Falls back to the `DISTRIBUTION` environment variable. Default is 8.
 
+#### --fanout
+
+The number of background fanout workers to run for this server. These workers resolve the recipients for bot activities — expanding follower collections and other addressing — and enqueue individual delivery jobs. If activities are slow to reach recipients, increase this number.
+
+Falls back to the `FANOUT` environment variable. Default is 4.
+
+#### --intake
+
+The number of background intake workers to run for this server. These workers accept activities arriving at the shared inbox and route them to the appropriate local bots. If shared-inbox processing is a bottleneck, increase this number.
+
+Falls back to the `INTAKE` environment variable. Default is 2.
+
 #### --index-file
 
 Path to the HTML file to show for the home page of your server. The activitypub.bot server doesn't support any other files, so any images or CSS stylesheets or JavaScript in this page have to be hosted elsewhere. Or just skip them!
 
-Falls back to the 'INDEX_FILE' environment variable. The default index file is in `web/index.html` and just says that this is an activitypub.bot server with a link to the GitHub repo.
+Falls back to the `INDEX_FILE` environment variable. The default index file is in `web/index.html` and just says that this is an activitypub.bot server with a link to the GitHub repo.
+
+#### --profile-file
+
+Path to the HTML file to show for bot profile pages. Like `--index-file`, any external assets must be hosted elsewhere.
+
+Falls back to the `PROFILE_FILE` environment variable. The default profile file is in `web/profile.html`.
 
 ### Config file
 
@@ -184,9 +208,15 @@ Bots will receive a [BotContext](#botcontext) object at initialization. The BotC
 
 The Bot interface has the following methods.
 
-#### constructor (username)
+#### constructor (username, options)
+
+The constructor; receives the `username` and an optional `options` object. Initialization should probably be deferred to the `initialize()` method. The default implementation stores the username and the following options:
 
-The constructor; receives the `username` by default. Initialization should probably be deferred to the initialize() method. The default implementation stores the username.
+- `fullname` — display name for the bot. Default: `'Bot'`.
+- `description` — bio text for the bot. Default: `'Default bot'`.
+- `icon` — URL string for the bot's avatar image. Default: `null`.
+- `image` — URL string for the bot's header/banner image. Default: `null`.
+- `checkSignature` — whether to require valid HTTP signatures on incoming activities. Default: `true`.
 
 #### async initialize (context)
 
@@ -200,6 +230,18 @@ A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Fun
 
 A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the bio of the bot.
 
+#### get icon ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the avatar image URL of the bot. Returns `null` if not set.
+
+#### get image ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the header/banner image URL of the bot. Returns `null` if not set.
+
+#### get checkSignature ()
+
+A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for whether the bot requires valid HTTP signatures on incoming activities.
+
 #### get username ()
 
 A [getter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get) for the username of the bot. Should match the constructor argument.

package/bin/activitypub-bot.js

@@ -14,6 +14,8 @@ const { values } = parseArgs({
     'log-level': { type: 'string' },
     delivery: { type: 'string' },
     distribution: { type: 'string' },
+    fanout: { type: 'string' },
+    intake: { type: 'string' },
     'index-file': { type: 'string' },
     help: { type: 'boolean', short: 'h' }
   },
@@ -31,6 +33,8 @@ Options:
   --log-level <level>        Log level (e.g., info, debug)
   --delivery <number>        Number of background delivery workers
   --distribution <number>    Number of background distribution workers
+  --fanout <number>          Number of background fanout workers
+  --intake <number>          Number of background intake workers
   --index-file <path>        HTML page to show at root path
   --profile-file <path>      HTML page to show for bot profiles
   -h, --help                 Show this help
@@ -61,6 +65,8 @@ const LOG_LEVEL =
   (process.env.NODE_ENV === 'test' ? 'silent' : 'info')
 const DELIVERY = parseNumber(values.delivery) || parseNumber(process.env.DELIVERY) || 2
 const DISTRIBUTION = parseNumber(values.distribution) || parseNumber(process.env.DISTRIBUTION) || 8
+const FANOUT = parseNumber(values.fanout) || parseNumber(process.env.FANOUT) || 4
+const INTAKE = parseNumber(values.intake) || parseNumber(process.env.INTAKE) || 2
 const INDEX_FILE = values['index-file'] || process.env.INDEX_FILE || DEFAULT_INDEX_FILE
 const PROFILE_FILE = values['profile-file'] || process.env.PROFILE_FILE || DEFAULT_PROFILE_FILE
 
@@ -73,6 +79,8 @@ const app = await makeApp({
   logLevel: LOG_LEVEL,
   deliveryWorkerCount: DELIVERY,
   distributionWorkerCount: DISTRIBUTION,
+  fanoutWorkerCount: FANOUT,
+  intakeWorkerCount: INTAKE,
   indexFileName: INDEX_FILE,
   profileFileName: PROFILE_FILE
 })

package/lib/activitydeliverer.js

@@ -23,6 +23,7 @@ const COLLECTION_TYPES = [
 
 export class ActivityDeliverer {
   static #QUEUE_ID = 'delivery'
+  static #INTAKE_QUEUE_ID = 'intake'
   static #TTL_SEEN_PUBLIC = 3 * 24 * 60 * 60 * 1000 // 3 days
   static #MAX_SEEN_PUBLIC = 1000000
   #actorStorage
@@ -106,10 +107,19 @@ export class ActivityDeliverer {
 
   async onIdle () {
     this.#logger.debug('Awaiting delivery queues')
+    await this.#jobQueue.onIdle(ActivityDeliverer.#INTAKE_QUEUE_ID)
     await this.#jobQueue.onIdle(ActivityDeliverer.#QUEUE_ID)
     this.#logger.debug('Done awaiting delivery queues')
   }
 
+  async intake (activity, subject) {
+    const raw = await activity.export({ useOriginalContext: true })
+    await this.#jobQueue.enqueue(
+      ActivityDeliverer.#INTAKE_QUEUE_ID,
+      { activity: raw, subject }
+    )
+  }
+
   async deliverToAll (activity, bots) {
     await this.#deliverToAll(activity, bots)
   }

package/lib/activitydistributor.js

@@ -14,6 +14,7 @@ const COLLECTION_TYPES = [
 export class ActivityDistributor {
   static #DISTRIBUTION_QUEUE_ID = 'distribution'
   static #DELIVERY_QUEUE_ID = 'delivery'
+  static #FANOUT_QUEUE_ID = 'fanout'
   static #MAX_CACHE_SIZE = 1000000
   static #PUBLIC = [
     'https://www.w3.org/ns/activitystreams#Public',
@@ -45,6 +46,14 @@ export class ActivityDistributor {
   }
 
   async distribute (activity, username) {
+    const raw = await activity.export({ useOriginalContext: true })
+    await this.#jobQueue.enqueue(
+      ActivityDistributor.#FANOUT_QUEUE_ID,
+      { activity: raw, username }
+    )
+  }
+
+  async fanout (activity, username) {
     const stripped = await this.#strip(activity)
     const raw = await stripped.export()
     const actorId = this.#formatter.format({ username })
@@ -102,8 +111,9 @@ export class ActivityDistributor {
   }
 
   async onIdle () {
-    await this.#jobQueue.onIdle(ActivityDistributor.#DELIVERY_QUEUE_ID)
+    await this.#jobQueue.onIdle(ActivityDistributor.#FANOUT_QUEUE_ID)
     await this.#jobQueue.onIdle(ActivityDistributor.#DISTRIBUTION_QUEUE_ID)
+    await this.#jobQueue.onIdle(ActivityDistributor.#DELIVERY_QUEUE_ID)
   }
 
   async * #public (activity, username) {

package/lib/activityhandler.js

@@ -311,14 +311,14 @@ export class ActivityHandler {
       actor: actor.id
     })
     await this.#actorStorage.addToCollection(bot.username, 'followers', actor)
-    this.#logger.info(
-      'Sending accept',
-      { actor: actor.id }
-    )
     const followersId = this.#formatter.format({
       username: bot.username,
       collection: 'followers'
     })
+    this.#logger.debug(
+      'Sending add activity for adding actor to followers',
+      { actor: actor.id, followers: followersId, follow: activity.id }
+    )
     const add = await this.#doActivity(bot, {
       type: 'Add',
       object: actor.id,
@@ -329,6 +329,10 @@ export class ActivityHandler {
     const recipients = await this.#getRecipients(activity)
     this.#addRecipient(recipients, actor, 'to')
     this.#removeRecipient(recipients, await this.#botActor(bot))
+    this.#logger.debug(
+      'Sending accept activity for follow',
+      { bot: this.#botId(bot), actor: actor.id, follow: activity.id }
+    )
     const accept = await this.#doActivity(bot, {
       type: 'Accept',
       object: activity.id,
@@ -337,7 +341,9 @@ export class ActivityHandler {
     this.#logger.debug({ accept: await accept.export() }, 'accepted follow')
     this.#logger.debug({
       msg: 'Notifying bot of new follow',
-      actor: actor.id
+      actor: actor.id,
+      bot: this.#botId(bot),
+      follow: activity.id
     })
     await bot.onFollow(actor, activity)
   }

package/lib/activitypubclient.js

@@ -36,20 +36,23 @@ export class ActivityPubClient {
   #digester = null
   #logger = null
   #limiter
+  #cache
 
-  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter) {
+  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter, cache) {
     assert.strictEqual(typeof keyStorage, 'object')
     assert.strictEqual(typeof urlFormatter, 'object')
     assert.strictEqual(typeof signer, 'object')
     assert.strictEqual(typeof digester, 'object')
     assert.strictEqual(typeof logger, 'object')
     assert.strictEqual(typeof limiter, 'object')
+    assert.strictEqual(typeof cache, 'object')
     this.#keyStorage = keyStorage
     this.#urlFormatter = urlFormatter
     this.#signer = signer
     this.#digester = digester
     this.#logger = logger.child({ class: this.constructor.name })
     this.#limiter = limiter
+    this.#cache = cache
   }
 
   async get (url, username = this.#urlFormatter.hostname) {
@@ -58,85 +61,134 @@ export class ActivityPubClient {
     assert.ok(username)
     assert.equal(typeof username, 'string')
     assert.ok(username !== '*')
-    const res = await this.#getRes(url, username, true)
-    return await this.#handleRes(res, url)
+
+    return await this.#get(url, username, true)
   }
 
   async getKey (url) {
     assert.ok(url)
     assert.equal(typeof url, 'string')
-    let res = await this.#getRes(url, this.#urlFormatter.hostname, false)
-    if ([401, 403, 404].includes(res.status)) {
-      // If we get a 401, 403, or 404, we should try again with the key
-      res = await this.#getRes(url, this.#urlFormatter.hostname, true)
+    let obj
+    try {
+      obj = await this.#get(url, this.#urlFormatter.hostname, false, false)
+    } catch (err) {
+      if (err.status && [401, 403, 404].includes(err.status)) {
+        obj = await this.#get(url, this.#urlFormatter.hostname, true, false)
+      } else {
+        throw err
+      }
     }
-    return await this.#handleRes(res, url)
+    return obj
+  }
+
+  async #get (url, username = this.#urlFormatter.hostname, sign = false, useCache = true) {
+    const json = await this.#getJSON(url, username, sign, useCache)
+    const obj = await this.#getObj(url, json)
+    return obj
   }
 
-  async #getRes (url, username = this.#urlFormatter.hostname, sign = false) {
+  async #getJSON (url, username = this.#urlFormatter.hostname, sign = false, useCache = true) {
     assert.ok(url)
     assert.equal(typeof url, 'string')
     assert.ok(username)
     assert.equal(typeof username, 'string')
+
+    const parsed = new URL(url)
+    const baseUrl = `${parsed.origin}${parsed.pathname}${parsed.search}`
+    let cached
+
+    if (useCache) {
+      cached = await this.#cache.get(baseUrl, username)
+      this.#logger.debug({ baseUrl, username, cached: !!cached, expiry: cached?.expiry }, 'cache lookup')
+
+      if (cached) {
+        this.#logger.debug({ baseUrl }, 'cache hit')
+        const now = Date.now()
+        const expiry = cached.expiry.getTime()
+        if (expiry > now) {
+          this.#logger.debug({ baseUrl, expiry, now }, 'cache fresh, returning cached object')
+          return cached.object
+        } else {
+          this.#logger.debug({ baseUrl, expiry, now }, 'cache stale, revalidating')
+        }
+      }
+    }
+
     const date = new Date().toUTCString()
     const headers = {
       accept: ActivityPubClient.#accept,
       date,
-      'user-agent': ActivityPubClient.#userAgent
+      'user-agent': ActivityPubClient.#userAgent,
+      'if-modified-since': cached?.lastModified?.toUTCString(),
+      'if-none-match': cached?.etag
     }
+    this.#logger.debug({ headers }, 'Sending headers')
     const method = 'GET'
-    this.#logger.debug(`Signing GET request for ${url}`)
+    this.#logger.debug(`Signing GET request for ${baseUrl}`)
     if (sign) {
       headers.signature =
-        await this.#sign({ username, url, method, headers })
+        await this.#sign({ username, url: baseUrl, method, headers })
     }
-    const hostname = (new URL(url)).hostname
-    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    const hostname = parsed.hostname
+    this.#logger.debug({ url: baseUrl, hostname }, 'Waiting for rate limiter')
     await this.#limiter.limit(hostname)
-    this.#logger.debug(`Fetching ${url} with GET`)
-    const result = await fetch(url,
+    this.#logger.debug(`Fetching ${baseUrl} with GET`)
+    const res = await fetch(baseUrl,
       {
         method,
         headers
       }
     )
-    this.#logger.debug({ hostname }, 'updating limiter')
-    await this.#limiter.update(hostname, result.headers)
+    this.#logger.debug({ hostname, status: res.status }, 'response received')
+    await this.#limiter.update(hostname, res.headers)
     this.#logger.debug(`Finished getting ${url}`)
-    return result
-  }
-
-  async #handleRes (res, url) {
-    if (res.status < 200 || res.status > 299) {
+    if (useCache && res.status === 304) {
+      this.#logger.debug({ baseUrl }, '304 Not Modified, returning cached object')
+      return cached.object
+    } else if (res.status < 200 || res.status > 299) {
       const body = await res.text()
-      this.#logger.warn({ status: res.status, body, url }, 'Could not fetch url')
+      this.#logger.warn(
+        { status: res.status, body, url: baseUrl },
+        'Could not fetch url'
+      )
       throw createHttpError(
         res.status,
-        `Could not fetch ${url}`,
+        `Could not fetch ${baseUrl}`,
         { headers: res.headers }
       )
     }
+
     const contentType = res.headers.get('content-type')
     const mimeType = contentType?.split(';')[0].trim()
     if (mimeType !== 'application/json' && !mimeType.endsWith('+json')) {
-      this.#logger.warn({ mimeType, url }, 'Unexpected mime type')
+      this.#logger.warn({ mimeType, url: baseUrl }, 'Unexpected mime type')
       throw new Error(`Got unexpected mime type ${mimeType} for URL ${url}`)
     }
     let json
     try {
       json = await res.json()
     } catch (err) {
-      this.#logger.warn({ url }, 'Error parsing fetch results')
+      this.#logger.warn({ url: baseUrl }, 'Error parsing fetch results')
       throw err
     }
+
+    await this.#cache.set(baseUrl, username, json, res.headers)
+
+    return json
+  }
+
+  async #getObj (url, json) {
+    const parsed = new URL(url)
+    const baseUrl = `${parsed.origin}${parsed.pathname}${parsed.search}`
+
     let obj
     try {
       obj = await as2.import(json)
     } catch (err) {
-      this.#logger.warn({ url, json }, 'Error importing JSON as AS2')
+      this.#logger.warn({ url: baseUrl, json }, 'Error importing JSON as AS2')
       throw err
     }
-    const resolved = (URL.parse(url).hash)
+    const resolved = (parsed.hash)
       ? this.#resolveObject(obj, url)
       : obj
     return resolved

package/lib/actorstorage.js

@@ -6,7 +6,7 @@ const AS2_NS = 'https://www.w3.org/ns/activitystreams#'
 export class ActorStorage {
   #connection = null
   #formatter = null
-  static #MAX_ITEMS_PER_PAGE = 20
+  static #MAX_ITEMS_PER_PAGE = 256
   constructor (connection, formatter) {
     this.#connection = connection
     this.#formatter = formatter

package/lib/app.js

@@ -41,12 +41,21 @@ import { DeliveryWorker } from './deliveryworker.js'
 import { DistributionWorker } from './distributionworker.js'
 import { RateLimiter } from '../lib/ratelimiter.js'
 import DoNothingBot from './bots/donothing.js'
+import { FanoutWorker } from './fanoutworker.js'
+import { IntakeWorker } from './intakeworker.js'
+import { RemoteObjectCache } from './remoteobjectcache.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
 const DEFAULT_PROFILE_FILENAME = resolve(currentDir, '..', 'web', 'profile.html')
 
-export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME }) {
+function createWorkers (logger, count, WorkerClass, ...args) {
+  const workers = Array.from({ length: count }, () => new WorkerClass(...args))
+  const runs = workers.map(w => w.run().catch(err => logger.error({ err }, `unexpected error in ${WorkerClass.name}`)))
+  return { workers, runs }
+}
+
+export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, fanoutWorkerCount = 4, intakeWorkerCount = 2, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME }) {
   const logger = Logger({
     level: logLevel
   })
@@ -63,7 +72,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const keyStorage = new KeyStorage(connection, logger)
   const objectStorage = new ObjectStorage(connection)
   const limiter = new RateLimiter(connection, logger)
-  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter)
+  const remoteObjectCache = new RemoteObjectCache(connection, logger)
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter, remoteObjectCache)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const jobQueue = new JobQueue(connection, logger)
@@ -136,35 +146,10 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
     logger
   ))
 
-  const deliveryWorkers = new Array(deliveryWorkerCount)
-  const deliveryWorkerRuns = new Array(deliveryWorkerCount)
-
-  for (let i = 0; i < deliveryWorkers.length; i++) {
-    deliveryWorkers[i] = new DeliveryWorker(
-      jobQueue,
-      actorStorage,
-      activityHandler,
-      logger,
-      bots
-    )
-    deliveryWorkerRuns[i] = deliveryWorkers[i].run().catch((err) => {
-      logger.error({ err, workerIndex: i }, 'unexpected error in delivery worker')
-    })
-  }
-
-  const distributionWorkers = new Array(distributionWorkerCount)
-  const distributionWorkerRuns = new Array(distributionWorkerCount)
-
-  for (let i = 0; i < distributionWorkers.length; i++) {
-    distributionWorkers[i] = new DistributionWorker(
-      jobQueue,
-      client,
-      logger
-    )
-    distributionWorkerRuns[i] = distributionWorkers[i].run().catch((err) => {
-      logger.error({ err, workerIndex: i }, 'unexpected error in distribution worker')
-    })
-  }
+  const { workers: deliveryWorkers, runs: deliveryWorkerRuns } = createWorkers(logger, deliveryWorkerCount, DeliveryWorker, jobQueue, logger, { actorStorage, activityHandler, bots })
+  const { workers: distributionWorkers, runs: distributionWorkerRuns } = createWorkers(logger, distributionWorkerCount, DistributionWorker, jobQueue, logger, { client })
+  const { workers: fanoutWorkers, runs: fanoutWorkerRuns } = createWorkers(logger, fanoutWorkerCount, FanoutWorker, jobQueue, logger, { distributor })
+  const { workers: intakeWorkers, runs: intakeWorkerRuns } = createWorkers(logger, intakeWorkerCount, IntakeWorker, jobQueue, logger, { deliverer, bots })
 
   const jobReaper = new JobReaper(jobQueue, logger)
   const jobReaperRun = jobReaper.run().catch((err) => {
@@ -273,23 +258,34 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
 
   app.onIdle = async () => {
     logger.debug('Awaiting components')
-    await distributor.onIdle()
-    await deliverer.onIdle()
+    await jobQueue.onIdle('intake')
+    await jobQueue.onIdle('delivery')
+    await jobQueue.onIdle('fanout')
+    await jobQueue.onIdle('delivery')
+    await jobQueue.onIdle('distribution')
     logger.debug('Done awaiting components')
   }
 
   app.cleanup = async () => {
     logger.info('Closing app')
-    for (const worker of deliveryWorkers) {
+    for (const worker of intakeWorkers) {
+      worker.stop()
+    }
+    for (const worker of fanoutWorkers) {
       worker.stop()
     }
     for (const worker of distributionWorkers) {
       worker.stop()
     }
+    for (const worker of deliveryWorkers) {
+      worker.stop()
+    }
     jobReaper.stop()
     jobQueue.abort()
-    await Promise.allSettled(deliveryWorkerRuns)
+    await Promise.allSettled(intakeWorkerRuns)
+    await Promise.allSettled(fanoutWorkerRuns)
     await Promise.allSettled(distributionWorkerRuns)
+    await Promise.allSettled(deliveryWorkerRuns)
     await jobReaperRun
     logger.info('Closing database connection')
     await connection.close()

package/lib/deliveryworker.js

@@ -1,71 +1,36 @@
 import assert from 'node:assert'
 
-import { nanoid } from 'nanoid'
-
 import as2 from './activitystreams.js'
 import BotMaker from './botmaker.js'
+import { Worker } from './worker.js'
 
-export class DeliveryWorker {
-  static #QUEUE_ID = 'delivery'
-  #queue
+export class DeliveryWorker extends Worker {
   #actorStorage
   #activityHandler
-  #logger
-  #running
-  #workerId
   #bots
 
-  constructor (queue, actorStorage, activityHandler, logger, bots) {
-    this.#queue = queue
-    this.#actorStorage = actorStorage
-    this.#activityHandler = activityHandler
-    this.#workerId = nanoid()
-    this.#logger = logger.child({
-      class: this.constructor.name,
-      worker: this.#workerId
-    })
-    this.#bots = bots
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.actorStorage)
+    assert.ok(options.activityHandler)
+    assert.ok(options.bots)
+    this.#actorStorage = options.actorStorage
+    this.#activityHandler = options.activityHandler
+    this.#bots = options.bots
   }
 
-  async run () {
-    this.#running = true
-    while (this.#running) {
-      let jobId
-      let payload
-      try {
-        this.#logger.debug('dequeueing');
-        ({ jobId, payload } = await this.#queue.dequeue(
-          DeliveryWorker.#QUEUE_ID,
-          this.#workerId
-        ))
-        this.#logger.debug({ jobId, payload }, 'got a job')
-        const activity = await as2.import(payload.activity)
-        assert.ok(payload.botUsername)
-        const bot = await BotMaker.makeBot(this.#bots, payload.botUsername)
-        assert.ok(bot)
-        assert.ok(bot.username)
-        this.#logger.debug(
-          { bot: bot.username, activity: activity.id, jobId },
-          'delivering to bot'
-        )
-        await this.#deliverTo(activity, bot)
-        this.#logger.debug({ bot: bot.username, activity: activity.id, jobId }, 'done')
-        await this.#queue.complete(jobId, this.#workerId)
-      } catch (err) {
-        if (err?.name === 'AbortError') {
-          this.#logger.info('Worker received abort signal')
-          break
-        }
-        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
-        if (jobId) {
-          await this.#queue.release(jobId, this.#workerId)
-        }
-      }
-    }
-  }
-
-  stop () {
-    this.#running = false
+  async doJob (payload, attempts) {
+    const activity = await as2.import(payload.activity)
+    assert.ok(payload.botUsername)
+    const bot = await BotMaker.makeBot(this.#bots, payload.botUsername)
+    assert.ok(bot)
+    assert.ok(bot.username)
+    this._logger.debug(
+      { bot: bot.username, activity: activity.id },
+      'delivering to bot'
+    )
+    await this.#deliverTo(activity, bot)
+    this._logger.debug({ bot: bot.username, activity: activity.id }, 'done')
   }
 
   async #deliverTo (activity, bot) {
@@ -75,7 +40,7 @@ export class DeliveryWorker {
     assert.strictEqual(typeof bot, 'object')
     assert.strictEqual(typeof bot.username, 'string')
     if (await this.#actorStorage.isInCollection(bot.username, 'inbox', activity)) {
-      this.#logger.info(
+      this._logger.info(
         { activity: activity.id, username: bot.username },
         'skipping redelivery for activity already in the inbox'
       )
@@ -84,13 +49,13 @@ export class DeliveryWorker {
     try {
       await this.#activityHandler.handleActivity(bot, activity)
     } catch (err) {
-      this.#logger.warn(
+      this._logger.warn(
         { err, activity: activity.id, bot: bot.username },
         'handler failed for activity'
       )
     }
-    this.#logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
+    this._logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
     await this.#actorStorage.addToCollection(bot.username, 'inbox', activity)
-    this.#logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
+    this._logger.debug(`Done adding ${activity.id} to ${bot.username} inbox`)
   }
 }

package/lib/distributionworker.js

@@ -1,123 +1,87 @@
-import { nanoid } from 'nanoid'
+import assert from 'node:assert'
 
 import as2 from './activitystreams.js'
+import { Worker, RecoverableError } from './worker.js'
 
-export class DistributionWorker {
-  static #QUEUE_ID = 'distribution'
+export class DistributionWorker extends Worker {
   static #MAX_ATTEMPTS = 21 // ~24 days
-  #jobQueue
   #client
-  #logger
-  #running
-  #workerId
 
-  constructor (jobQueue, client, logger) {
-    this.#jobQueue = jobQueue
-    this.#client = client
-    this.#workerId = nanoid()
-    this.#logger = logger.child({
-      class: this.constructor.name,
-      workerId: this.#workerId
-    })
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.client)
+    this.#client = options.client
   }
 
-  async run () {
-    this.#running = true
-    while (this.#running) {
-      let jobId
-      let payload
-      let attempts
-      try {
-        this.#logger.debug('dequeueing');
-        ({ jobId, payload, attempts } = await this.#jobQueue.dequeue(
-          DistributionWorker.#QUEUE_ID,
-          this.#workerId
-        ))
-        this.#logger.debug({ jobId, payload, attempts }, 'got a job')
-        const { inbox, activity, username } = payload
-        const activityObj = await as2.import(activity)
-        try {
-          await this.#client.post(inbox, activityObj, username)
-          this.#logger.info(`Delivered ${activity.id} to ${inbox}`)
-          await this.#jobQueue.complete(jobId, this.#workerId)
-          this.#logger.info({ jobId }, 'completed job')
-        } catch (error) {
-          if (!error.status) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity and no HTTP status available')
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          } else if (error.status >= 300 && error.status < 400) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity and unexpected redirect code'
-            )
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          } else if (error.status >= 400 && error.status < 500) {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity due to client error'
-            )
-            if ([408, 425, 429].includes(error.status)) {
-              this.#logger.debug(
-                { error, activity: activity.id, inbox },
-                'Retrying on recoverable status'
-              )
-              const delay = this.#retryDelay(error.headers, attempts)
-              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
-            } else {
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            }
-          } else if (error.status >= 500 && error.status < 600) {
-            if ([501, 505, 508, 510].includes(error.status)) {
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts },
-                'Could not deliver activity due to unrecoverable server error'
-              )
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            } else if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts },
-                'Could not deliver activity due to server error; no more attempts'
-              )
-              await this.#jobQueue.fail(jobId, this.#workerId)
-            } else {
-              const delay = this.#retryDelay(error.headers, attempts)
-              this.#logger.warn(
-                { error, activity: activity.id, inbox, attempts, delay },
-                'Could not deliver activity due to server error; will retry'
-              )
-              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
-            }
-          } else {
-            this.#logger.warn(
-              { error, activity: activity.id, inbox },
-              'Could not deliver activity due to unexpected status range'
-            )
-            await this.#jobQueue.fail(jobId, this.#workerId)
-          }
-        }
-      } catch (err) {
-        if (err?.name === 'AbortError') {
-          this.#logger.info('Worker received abort signal')
-          break
+  async doJob (payload, attempts) {
+    const { inbox, activity, username } = payload
+    const activityObj = await as2.import(activity)
+    try {
+      await this.#client.post(inbox, activityObj, username)
+      this._logger.info(`Delivered ${activity.id} to ${inbox}`)
+    } catch (error) {
+      if (!error.status) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity and no HTTP status available')
+        throw error
+      } else if (error.status >= 300 && error.status < 400) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity and unexpected redirect code'
+        )
+        throw error
+      } else if (error.status >= 400 && error.status < 500) {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity due to client error'
+        )
+        if ([408, 425, 429].includes(error.status)) {
+          this._logger.debug(
+            { error, activity: activity.id, inbox },
+            'Retrying on recoverable status'
+          )
+          const recoverable = new RecoverableError(error.message)
+          recoverable.delay = this.#retryDelay(error.headers, attempts)
+          throw recoverable
+        } else {
+          throw error
         }
-        this.#logger.warn({ err, jobId }, 'Error delivering to bot')
-        if (jobId) {
-          const delay = this.#retryDelay(null, attempts ?? 1)
-          this.#logger.warn(
-            { err, jobId, attempts, delay },
-            'Retrying job after a delay'
+      } else if (error.status >= 500 && error.status < 600) {
+        if ([501, 505, 508, 510].includes(error.status)) {
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts },
+            'Could not deliver activity due to unrecoverable server error'
+          )
+          throw error
+        } else if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts },
+            'Could not deliver activity due to server error; no more attempts'
+          )
+          throw error
+        } else {
+          const recoverable = new RecoverableError(error.message)
+          recoverable.delay = this.#retryDelay(error.headers, attempts)
+          this._logger.warn(
+            { error, activity: activity.id, inbox, attempts, delay: recoverable.delay },
+            'Could not deliver activity due to server error; will retry'
           )
-          await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+          throw recoverable
         }
+      } else {
+        this._logger.warn(
+          { error, activity: activity.id, inbox },
+          'Could not deliver activity due to unexpected status range'
+        )
+        throw error
       }
     }
   }
 
   #retryDelay (headers, attempts) {
     if (headers?.['retry-after']) {
-      this.#logger.debug('using retry-after header')
+      this._logger.debug('using retry-after header')
       const retryAfter = headers['retry-after']
       if (/^\d+$/.test(retryAfter)) {
         return parseInt(retryAfter, 10) * 1000
@@ -125,11 +89,7 @@ export class DistributionWorker {
         return new Date(retryAfter) - Date.now()
       }
     }
-    this.#logger.debug('exponential backoff')
+    this._logger.debug('exponential backoff')
     return Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
   }
-
-  stop () {
-    this.#running = false
-  }
 }

package/lib/fanoutworker.js

@@ -0,0 +1,20 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+import { Worker } from './worker.js'
+
+export class FanoutWorker extends Worker {
+  #distributor
+
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.distributor)
+    this.#distributor = options.distributor
+  }
+
+  async doJob (payload, attempts) {
+    const activity = await as2.import(payload.activity)
+    const { username } = payload
+    await this.#distributor.fanout(activity, username)
+  }
+}

package/lib/intakeworker.js

@@ -0,0 +1,22 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+import { Worker } from './worker.js'
+
+export class IntakeWorker extends Worker {
+  #deliverer
+  #bots
+  constructor (jobQueue, logger, options = {}) {
+    super(jobQueue, logger, options)
+    assert.ok(options.deliverer)
+    assert.ok(options.bots)
+    this.#deliverer = options.deliverer
+    this.#bots = options.bots
+  }
+
+  async doJob (payload, attempts) {
+    const raw = payload.activity
+    const activity = await as2.import(raw)
+    await this.#deliverer.deliverToAll(activity, this.#bots)
+  }
+}

package/lib/migrations/007-remote-object-cache.js

@@ -0,0 +1,20 @@
+export const id = '007-remote-object-cache'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE remote_object_cache (
+      id varchar(512) NOT NULL,
+      username varchar(512) NOT NULL,
+      last_modified TIMESTAMP NULL,
+      etag varchar(256) NULL,
+      expiry TIMESTAMP NULL,
+      data TEXT NOT NULL,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (id, username)
+    );
+  `, queryOptions)
+  await connection.query(`
+    CREATE INDEX remote_object_cache_expiry on remote_object_cache (expiry);
+  `, queryOptions)
+}

package/lib/objectstorage.js

@@ -11,7 +11,7 @@ export class NoSuchObjectError extends Error {
 
 export class ObjectStorage {
   #connection = null
-  static #MAX_ITEMS_PER_PAGE = 20
+  static #MAX_ITEMS_PER_PAGE = 256
 
   constructor (connection) {
     this.#connection = connection

package/lib/remoteobjectcache.js

@@ -0,0 +1,152 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+
+const SEC_NS = 'https://w3id.org/security#'
+const NS = 'https://www.w3.org/ns/activitystreams#'
+const KEY_TYPES = [SEC_NS + 'Key', SEC_NS + 'CryptographicKey']
+const COLLECTION_TYPES =
+  [NS + 'Collection', NS + 'CollectionPage', NS + 'OrderedCollection', NS + 'OrderedCollectionPage']
+
+export class RemoteObjectCache {
+  #connection
+  #logger
+  constructor (connection, logger) {
+    assert.ok(connection)
+    assert.ok(logger)
+    this.#connection = connection
+    this.#logger = logger
+  }
+
+  async get (id, username) {
+    assert.ok(id)
+    assert.strictEqual(typeof id, 'string')
+    assert.ok(username)
+    assert.strictEqual(typeof username, 'string')
+
+    let result
+
+    const [rows] = await this.#connection.query(
+      `SELECT last_modified, etag, expiry, data
+      FROM remote_object_cache
+      WHERE id = ? AND username = ?`,
+      { replacements: [id, username] })
+
+    if (rows.length === 0) {
+      result = null
+    } else {
+      result = {
+        expiry: new Date(rows[0].expiry),
+        lastModified: rows[0].last_modified
+          ? new Date(rows[0].last_modified)
+          : null,
+        etag: rows[0].etag,
+        object: JSON.parse(rows[0].data)
+      }
+    }
+
+    assert.ok(
+      result === null ||
+      (
+        typeof result === 'object' &&
+       'expiry' in result &&
+       'lastModified' in result &&
+       'etag' in result &&
+       'object' in result
+      )
+    )
+
+    return result
+  }
+
+  async set (id, username, object, headers) {
+    assert.ok(id)
+    assert.strictEqual(typeof id, 'string')
+    assert.ok(username)
+    assert.strictEqual(typeof username, 'string')
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    const cacheControl = headers.get('cache-control')
+    if (cacheControl && cacheControl.includes('no-store')) {
+      return
+    }
+
+    const expiry = await this.#getExpiry(object, headers)
+    const lastModified = headers.get('last-modified')
+    const etag = headers.get('etag')
+
+    await this.#connection.query(
+      `INSERT INTO remote_object_cache
+      (id, username, last_modified, etag, expiry, data)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT (id, username)
+      DO UPDATE
+      SET last_modified = EXCLUDED.last_modified,
+      etag = EXCLUDED.etag,
+      expiry = EXCLUDED.expiry,
+      data = EXCLUDED.data,
+      updated_at = CURRENT_TIMESTAMP
+      `,
+      { replacements: [id, username, lastModified, etag, expiry, JSON.stringify(object)] }
+    )
+  }
+
+  async #getExpiry (object, headers) {
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    let expiry = this.#getExpiryFromHeaders(headers)
+    if (!expiry) {
+      expiry = await this.#getExpiryFromObject(object)
+    }
+    return expiry
+  }
+
+  #getExpiryFromHeaders (headers) {
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    const cacheControl = headers.get('cache-control')
+    if (cacheControl) {
+      if (cacheControl.includes('no-cache')) {
+        return new Date(Date.now() - 1)
+      } else if (cacheControl.includes('max-age')) {
+        const match = cacheControl.match(/max-age=(\d+)/)
+        if (match) {
+          return new Date(Date.now() + parseInt(match[1], 10) * 1000)
+        }
+      }
+    }
+    if (headers.has('expires')) {
+      return new Date(headers.get('expires'))
+    }
+    return null
+  }
+
+  async #getExpiryFromObject (object) {
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    const as2obj = await as2.import(object)
+    let offset
+    if (as2obj.isActivity()) {
+      offset = 24 * 60 * 60 * 1000
+    } else if (as2obj.inbox) {
+      offset = 30 * 60 * 1000
+    } else if (KEY_TYPES.includes(as2obj.type)) {
+      offset = -1000
+    } else if (COLLECTION_TYPES.includes(as2obj.type)) {
+      offset = -1000
+    } else {
+      offset = 5 * 60 * 1000
+    }
+    return new Date(Date.now() + offset)
+  }
+}

package/lib/routes/sharedinbox.js

@@ -69,7 +69,7 @@ router.post('/shared/inbox', async (req, res, next) => {
 
   logger.info(`Activity ${activity.id} received at shared inbox`)
 
-  await deliverer.deliverToAll(activity, bots)
+  await deliverer.intake(activity, subject)
 
   res.status(202)
   res.type('text/plain')

package/lib/worker.js

@@ -0,0 +1,86 @@
+import assert from 'node:assert'
+import { nanoid } from 'nanoid'
+
+export class RecoverableError extends Error {
+  delay = 1000
+}
+
+export class Worker {
+  #jobQueue
+  #logger
+  #running
+  #workerId
+
+  constructor (jobQueue, logger, options = {}) {
+    this.#jobQueue = jobQueue
+    this.#workerId = nanoid()
+    this.#logger = logger.child({
+      class: this.constructor.name,
+      worker: this.#workerId
+    })
+  }
+
+  get _logger () {
+    return this.#logger
+  }
+
+  get queueId () {
+    const name = this.constructor.name
+    return name.slice(0, name.length - 6).toLowerCase()
+  }
+
+  async run () {
+    this.#running = true
+    while (this.#running) {
+      let jobId
+      let payload
+      let attempts
+      try {
+        this.#logger.debug('dequeueing');
+        ({ jobId, payload, attempts } = await this.#jobQueue.dequeue(
+          this.queueId,
+          this.#workerId
+        ))
+        this.#logger.debug({ jobId, payload, attempts }, 'got a job')
+      } catch (err) {
+        if (err?.name === 'AbortError') {
+          this.#logger.info('Worker received abort signal')
+          break
+        } else {
+          this.#logger.warn({ err, jobId }, 'error before job, retrying')
+          if (jobId) {
+            const delay = this.#retryDelay(attempts ?? 1)
+            await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+          }
+          continue
+        }
+      }
+
+      assert.ok(jobId)
+      assert.ok(payload)
+
+      try {
+        await this.doJob(payload, attempts)
+        await this.#jobQueue.complete(jobId, this.#workerId)
+      } catch (err) {
+        if (err instanceof RecoverableError) {
+          await this.#jobQueue.retryAfter(jobId, this.#workerId, err.delay)
+        } else {
+          await this.#jobQueue.fail(jobId, this.#workerId)
+        }
+      }
+    }
+  }
+
+  async doJob (payload, attempts) {
+    throw new Error('Must implement doJob')
+  }
+
+  stop () {
+    this.#running = false
+  }
+
+  #retryDelay (attempts) {
+    return Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.34.1",
+  "version": "0.36.0",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",