@evanp/activitypub-bot 0.45.13 → 0.45.15

package/CHANGELOG.md

@@ -9,6 +9,26 @@ and this project adheres to
 
 ## [Unreleased]
 
+## [0.45.15] - 2026-05-11
+
+### Fixed
+
+- Heuristic to check if an integer in X-RateLimit-Reset is probably an offset or
+  probably a Unix epoch in seconds. Anything less than 30 days is treated as an
+  offset; otherwise it's an epoch.
+- Fix job queue to clamp job retry delays between 0 and 30 days.
+- Fix workers to continue operating even if retry or fail calls
+  fail.
+
+## [0.45.14] - 2026-05-10
+
+### Changed
+
+- Depend on inbox forwarding for distribution to remote collections.
+- More robust handling of client request failures in delivery and distribution.
+- More robust handling of intake, fanout throttle errors.
+- Store error stack for failing jobs in queue job entries.
+
 ## [0.45.13] - 2026-05-10
 
 ### Changed

package/lib/activitydeliverer.js

@@ -4,6 +4,7 @@ import * as ttlcachePkg from '@isaacs/ttlcache'
 
 import as2 from './activitystreams.js'
 import BotMaker from './botmaker.js'
+import { ThrottleError } from './requestthrottler.js'
 
 const TTLCache =
   ttlcachePkg.TTLCache ?? ttlcachePkg.default ?? ttlcachePkg
@@ -128,6 +129,7 @@ export class ActivityDeliverer {
     const deliveredTo = new Set()
     const actor = this.getActor(activity)
     const recipients = this.getRecipients(activity)
+    let fullActor
 
     for (const recipient of recipients) {
       this.#logger.debug({ recipient: recipient.id }, 'Checking recipient')
@@ -152,8 +154,18 @@ export class ActivityDeliverer {
           )
         }
       } else {
-        const fullActor = await this.#client.get(actor.id)
-        const fullRecipient = await this.#client.get(recipient.id)
+        fullActor = fullActor ?? await this.#client.get(actor.id)
+        let fullRecipient
+        try {
+          fullRecipient = await this.#client.get(recipient.id)
+        } catch (err) {
+          if (err instanceof ThrottleError) throw err
+          this.#logger.warn(
+            { recipient: recipient.id },
+            'Unreachable remote recipient'
+          )
+          continue
+        }
         if (await this.#isRemoteActor(fullRecipient)) {
           this.#logger.warn({ recipient: recipient.id }, 'Skipping remote actor')
         } else if (await this.#isRemoteFollowersCollection(fullActor, fullRecipient)) {

package/lib/activitydistributor.js

@@ -164,6 +164,14 @@ export class ActivityDistributor {
               yield id
             } else if (this.#isRemoteCollection(obj)) {
               this.#logger.debug({ id }, 'Remote collection')
+              const owner = await this.#getCollectionOwner(obj, username)
+              if (owner && this.#isRecipient(activity, owner)) {
+                this.#logger.debug(
+                  { id, owner: owner.id },
+                  'Leaving remote collection up to inbox forwarding'
+                )
+                continue
+              }
               try {
                 for await (const item of this.#client.items(obj.id, username)) {
                   this.#logger.debug({ id: item.id }, 'Remote collection member')
@@ -194,7 +202,16 @@ export class ActivityDistributor {
       return sharedInbox
     }
 
-    const obj = await this.#client.get(actorId, username)
+    let obj
+    try {
+      obj = await this.#client.get(actorId, username)
+    } catch (err) {
+      this.#logger.warn(
+        { actorId, username, err },
+        'Could not get actor in #getInbox'
+      )
+      return null
+    }
 
     // Get the shared inbox if it exists
 
@@ -232,7 +249,16 @@ export class ActivityDistributor {
       return directInbox
     }
 
-    const obj = await this.#client.get(actorId, username)
+    let obj
+    try {
+      obj = await this.#client.get(actorId, username)
+    } catch (err) {
+      this.#logger.warn(
+        { actorId, username, err },
+        'Could not get actor in #getDirectInbox'
+      )
+      return null
+    }
 
     if (!obj.inbox) {
       return null
@@ -274,4 +300,47 @@ export class ActivityDistributor {
       ? obj.type.some(t => COLLECTION_TYPES.includes(t))
       : COLLECTION_TYPES.includes(obj.type)
   }
+
+  async #getCollectionOwner (obj, username) {
+    if (obj.attributedTo) {
+      return obj.attributedTo.first
+    } else {
+      let ownerId
+      let owner
+
+      if (obj.id.endsWith('/followers')) {
+        ownerId = obj.id.replace('/followers', '')
+      } else if (obj.id.endsWith('/following')) {
+        ownerId = obj.id.replace('/following', '')
+      }
+
+      if (ownerId) {
+        try {
+          owner = await this.#client.get(ownerId, username)
+        } catch (err) {
+          this.#logger.debug(
+            { ownerId },
+            'Owner unreachable'
+          )
+        }
+      }
+
+      return owner
+    }
+  }
+
+  #isRecipient (activity, obj) {
+    const props = ['to', 'cc', 'audience', 'bto', 'bcc']
+    for (const prop of props) {
+      const p = activity.get(prop)
+      if (p) {
+        for (const value of p) {
+          if (value.id === obj.id) {
+            return true
+          }
+        }
+      }
+    }
+    return false
+  }
 }

package/lib/fanoutworker.js

@@ -1,7 +1,8 @@
 import assert from 'node:assert'
 
 import as2 from './activitystreams.js'
-import { Worker } from './worker.js'
+import { Worker, RecoverableError } from './worker.js'
+import { ThrottleError } from './requestthrottler.js'
 
 export class FanoutWorker extends Worker {
   #distributor
@@ -15,6 +16,17 @@ export class FanoutWorker extends Worker {
   async doJob (payload, attempts) {
     const activity = await as2.import(payload.activity)
     const { username } = payload
-    await this.#distributor.fanout(activity, username)
+    try {
+      await this.#distributor.fanout(activity, username)
+    } catch (err) {
+      if (err instanceof ThrottleError) {
+        this._logger.warn(
+          { err, activity: activity.id, username },
+          'Throttled fanout, waiting to retry')
+        throw new RecoverableError(err.message, err.waitTime)
+      } else {
+        throw err
+      }
+    }
   }
 }

package/lib/intakeworker.js

@@ -1,7 +1,8 @@
 import assert from 'node:assert'
 
 import as2 from './activitystreams.js'
-import { Worker } from './worker.js'
+import { Worker, RecoverableError } from './worker.js'
+import { ThrottleError } from './requestthrottler.js'
 
 export class IntakeWorker extends Worker {
   #deliverer
@@ -17,6 +18,17 @@ export class IntakeWorker extends Worker {
   async doJob (payload, attempts) {
     const raw = payload.activity
     const activity = await as2.import(raw)
-    await this.#deliverer.deliverToAll(activity, this.#bots)
+    try {
+      await this.#deliverer.deliverToAll(activity, this.#bots)
+    } catch (err) {
+      if (err instanceof ThrottleError) {
+        this._logger.warn(
+          { err, activity: activity.id },
+          'Throttled intake, waiting to retry')
+        throw new RecoverableError(err.message, err.waitTime)
+      } else {
+        throw err
+      }
+    }
   }
 }

package/lib/jobqueue.js

@@ -6,6 +6,7 @@ import { QueryTypes } from 'sequelize'
 
 const MAX_DELAY = 30000
 const DEFAULT_PRIORITY = 1000000
+const MAX_RETRY = 30 * 24 * 60 * 60 * 1000
 
 export class JobQueue {
   #connection
@@ -87,24 +88,28 @@ export class JobQueue {
     this.#logger.debug({ method: 'release', jobRunnerId, jobId }, 'released job')
   }
 
-  async retryAfter (jobId, jobRunnerId, delay) {
+  async retryAfter (jobId, jobRunnerId, delay, lastError = null) {
     assert.ok(jobId)
     assert.strictEqual(typeof jobId, 'string')
     assert.ok(jobRunnerId)
     assert.strictEqual(typeof jobRunnerId, 'string')
-    assert.ok(delay)
     assert.strictEqual(typeof delay, 'number')
 
-    const retryAfter = new Date(Date.now() + delay)
+    const retryAfter = (isNaN(delay))
+      ? new Date(Date.now() + 1000)
+      : (!isFinite(delay))
+          ? new Date(Date.now() + MAX_RETRY)
+          : new Date(Date.now() + Math.max(0, Math.min(delay, MAX_RETRY)))
 
     await this.#connection.query(`
       UPDATE job
       SET claimed_by = NULL,
           claimed_at = NULL,
           updated_at = CURRENT_TIMESTAMP,
-          retry_after = ?
+          retry_after = ?,
+          last_error = ?
       WHERE job_id = ? AND claimed_by = ?;`,
-    { replacements: [retryAfter, jobId, jobRunnerId] }
+    { replacements: [retryAfter, lastError, jobId, jobRunnerId] }
     )
     this.#logger.debug(
       { method: 'retry', jobRunnerId, jobId, delay },
@@ -157,7 +162,7 @@ export class JobQueue {
     this.#ac.abort()
   }
 
-  async fail (jobId, jobRunnerId) {
+  async fail (jobId, jobRunnerId, lastError = null) {
     await this.#connection.query(`
       INSERT INTO failed_job
       (job_id, queue_id, priority, payload, claimed_at, claimed_by, attempts, retry_after, created_at, updated_at)
@@ -170,6 +175,14 @@ export class JobQueue {
       DELETE FROM job
       WHERE job_id = ? AND claimed_by = ?;`,
     { replacements: [jobId, jobRunnerId] })
+
+    if (lastError) {
+      await this.#connection.query(`
+      UPDATE failed_job
+      SET last_error = ?
+      WHERE job_id = ?;`,
+      { replacements: [lastError, jobId] })
+    }
     this.#logger.debug({ method: 'complete', jobRunnerId, jobId }, 'completed job')
   }
 

package/lib/migrations/011-job-last-error.js

@@ -0,0 +1,12 @@
+export const id = '011-job-last-error'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    ALTER TABLE job
+    ADD COLUMN last_error TEXT;
+  `, queryOptions)
+  await connection.query(`
+    ALTER TABLE failed_job
+    ADD COLUMN last_error TEXT;
+  `, queryOptions)
+}

package/lib/requestthrottler.js

@@ -2,6 +2,7 @@ import { setTimeout as sleep } from 'node:timers/promises'
 import assert from 'node:assert'
 
 const BETA = 0.75
+const EPOCH_THRESHOLD = 30 * 24 * 60 * 60
 
 export class ThrottleError extends Error {
   constructor (message, waitTime) {
@@ -51,7 +52,11 @@ export class RequestThrottler {
       let reset
       if (resetHeader.match(/^\d+$/)) {
         resetSeconds = parseInt(resetHeader)
-        reset = new Date(Date.now() + (resetSeconds * 1000))
+        if (resetSeconds < EPOCH_THRESHOLD) {
+          reset = new Date(Date.now() + (resetSeconds * 1000))
+        } else {
+          reset = new Date(resetSeconds * 1000)
+        }
       } else {
         reset = new Date(resetHeader)
         resetSeconds = reset - Date.now()

package/lib/worker.js

@@ -55,7 +55,7 @@ export class Worker {
           this.#logger.warn({ err, jobId }, 'error before job, retrying')
           if (jobId) {
             const delay = this.#retryDelay(attempts ?? 1)
-            await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+            await this.#jobQueue.retryAfter(jobId, this.#workerId, delay, err.stack)
           }
           continue
         }
@@ -68,10 +68,17 @@ export class Worker {
         await this.doJob(payload, attempts)
         await this.#jobQueue.complete(jobId, this.#workerId)
       } catch (err) {
-        if (err instanceof RecoverableError) {
-          await this.#jobQueue.retryAfter(jobId, this.#workerId, err.delay)
-        } else {
-          await this.#jobQueue.fail(jobId, this.#workerId)
+        try {
+          if (err instanceof RecoverableError) {
+            await this.#jobQueue.retryAfter(jobId, this.#workerId, err.delay, err.stack)
+          } else {
+            await this.#jobQueue.fail(jobId, this.#workerId, err.stack)
+          }
+        } catch (queueError) {
+          this.#logger.error(
+            { err: queueError, original: err, jobId, workerId: this.#workerId },
+            'job retry/fail error'
+          )
         }
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.45.13",
+  "version": "0.45.15",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",