@evanp/activitypub-bot 0.32.0 → 0.32.2

package/lib/activitypubclient.js

@@ -108,7 +108,11 @@ export class ActivityPubClient {
     if (res.status < 200 || res.status > 299) {
       const body = await res.text()
       this.#logger.warn({ status: res.status, body, url }, 'Could not fetch url')
-      throw createHttpError(res.status, `Could not fetch ${url}`)
+      throw createHttpError(
+        res.status,
+        `Could not fetch ${url}`,
+        { headers: res.headers }
+      )
     }
     const contentType = res.headers.get('content-type')
     const mimeType = contentType?.split(';')[0].trim()
@@ -172,7 +176,11 @@ export class ActivityPubClient {
     await this.#limiter.update(hostname, res.headers)
     this.#logger.debug(`Done fetching POST for ${url}`)
     if (res.status < 200 || res.status > 299) {
-      throw createHttpError(res.status, await res.text())
+      throw createHttpError(
+        res.status,
+        await res.text(),
+        { headers: res.headers }
+      )
     }
   }
 

package/lib/distributionworker.js

@@ -4,7 +4,7 @@ import as2 from './activitystreams.js'
 
 export class DistributionWorker {
   static #QUEUE_ID = 'distribution'
-  static #MAX_ATTEMPTS = 16
+  static #MAX_ATTEMPTS = 21 // ~24 days
   #jobQueue
   #client
   #logger
@@ -43,21 +43,64 @@ export class DistributionWorker {
           this.#logger.info({ jobId }, 'completed job')
         } catch (error) {
           if (!error.status) {
-            this.#logger.error(`Could not deliver ${activity.id} to ${inbox}: ${error.message}`)
-            this.#logger.error(error.stack)
+            this.#logger.warn(
+              { error, activity: activity.id, inbox },
+              'Could not deliver activity and no HTTP status available')
+            await this.#jobQueue.fail(jobId, this.#workerId)
           } else if (error.status >= 300 && error.status < 400) {
-            this.#logger.error(`Unexpected redirect code delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
+            this.#logger.warn(
+              { error, activity: activity.id, inbox },
+              'Could not deliver activity and unexpected redirect code'
+            )
+            await this.#jobQueue.fail(jobId, this.#workerId)
           } else if (error.status >= 400 && error.status < 500) {
-            this.#logger.error(`Bad request delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}`)
+            this.#logger.warn(
+              { error, activity: activity.id, inbox },
+              'Could not deliver activity due to client error'
+            )
+            if (error.status === 429) {
+              this.#logger.debug(
+                { error, activity: activity.id, inbox },
+                'Retrying on 429 status'
+              )
+              let delay
+              if (error.headers && error.headers['retry-after']) {
+                this.#logger.debug('using retry-after header')
+                const retryAfter = error.headers['retry-after']
+                if (/^\d+$/.test(retryAfter)) {
+                  delay = parseInt(retryAfter, 10) * 1000
+                } else {
+                  delay = new Date(retryAfter) - Date.now()
+                }
+              } else {
+                this.#logger.debug('exponential backoff')
+                delay = Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
+              }
+              await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
+            } else {
+              await this.#jobQueue.fail(jobId, this.#workerId)
+            }
           } else if (error.status >= 500 && error.status < 600) {
             if (attempts >= DistributionWorker.#MAX_ATTEMPTS) {
-              this.#logger.error(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; giving up after ${attempts} attempts`)
-              await this.#jobQueue.complete(jobId, this.#workerId)
+              this.#logger.warn(
+                { error, activity: activity.id, inbox, attempts },
+                'Could not deliver activity due to server error; no more attempts'
+              )
+              await this.#jobQueue.fail(jobId, this.#workerId)
             } else {
               const delay = Math.round((2 ** (attempts - 1) * 1000) * (0.5 + Math.random()))
-              this.#logger.warn(`Server error delivering ${activity.id} to ${inbox}: ${error.status} ${error.message}; will retry in ${delay} ms (${attempts} of ${DistributionWorker.#MAX_ATTEMPTS})`)
+              this.#logger.warn(
+                { error, activity: activity.id, inbox, attempts, delay },
+                'Could not deliver activity due to server error; will retry'
+              )
               await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
             }
+          } else {
+            this.#logger.warn(
+              { error, activity: activity.id, inbox },
+              'Could not deliver activity due to unexpected status range'
+            )
+            await this.#jobQueue.fail(jobId, this.#workerId)
           }
         }
       } catch (err) {
@@ -67,7 +110,12 @@ export class DistributionWorker {
         }
         this.#logger.warn({ err, jobId }, 'Error delivering to bot')
         if (jobId) {
-          await this.#jobQueue.release(jobId, this.#workerId)
+          const delay = Math.round((2 ** ((attempts ?? 1) - 1) * 1000) * (0.5 + Math.random()))
+          this.#logger.warn(
+            { err, jobId, attempts, delay },
+            'Retrying job after a delay'
+          )
+          await this.#jobQueue.retryAfter(jobId, this.#workerId, delay)
         }
       }
     }

package/lib/jobqueue.js

@@ -157,6 +157,22 @@ export class JobQueue {
     this.#ac.abort()
   }
 
+  async fail (jobId, jobRunnerId) {
+    await this.#connection.query(`
+      INSERT INTO failed_job
+      (job_id, queue_id, priority, payload, claimed_at, claimed_by, attempts, retry_after, created_at, updated_at)
+      SELECT job_id, queue_id, priority, payload, claimed_at, claimed_by, attempts, retry_after, created_at, updated_at
+      FROM job
+      WHERE job_id = ?
+      AND claimed_by = ?;`,
+    { replacements: [jobId, jobRunnerId] })
+    await this.#connection.query(`
+      DELETE FROM job
+      WHERE job_id = ? AND claimed_by = ?;`,
+    { replacements: [jobId, jobRunnerId] })
+    this.#logger.debug({ method: 'complete', jobRunnerId, jobId }, 'completed job')
+  }
+
   async #countJobs (queueId) {
     this.#logger.debug({ method: '#countJobs', queueId }, 'checking queue size')
     const rows = await this.#connection.query(`

package/lib/migrations/006-failed-jobs.js

@@ -0,0 +1,21 @@
+export const id = '006-failed-jobs'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE failed_job (
+      job_id char(21) NOT NULL PRIMARY KEY,
+      queue_id varchar(64) NOT NULL,
+      priority INTEGER,
+      payload TEXT,
+      claimed_at TIMESTAMP,
+      claimed_by varchar(64),
+      attempts INTEGER default 0,
+      retry_after TIMESTAMP,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+    );
+  `, queryOptions)
+  await connection.query(`
+    CREATE INDEX failed_job_queue_id on job (queue_id);
+  `, queryOptions)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.32.0",
+  "version": "0.32.2",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",