@evanp/activitypub-bot 0.26.2 → 0.27.1

package/lib/activitypubclient.js

@@ -33,13 +33,21 @@ export class ActivityPubClient {
   #signer = null
   #digester = null
   #logger = null
-
-  constructor (keyStorage, urlFormatter, signer, digester, logger) {
+  #limiter
+
+  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter) {
+    assert.strictEqual(typeof keyStorage, 'object')
+    assert.strictEqual(typeof urlFormatter, 'object')
+    assert.strictEqual(typeof signer, 'object')
+    assert.strictEqual(typeof digester, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    assert.strictEqual(typeof limiter, 'object')
     this.#keyStorage = keyStorage
     this.#urlFormatter = urlFormatter
     this.#signer = signer
     this.#digester = digester
     this.#logger = logger.child({ class: this.constructor.name })
+    this.#limiter = limiter
   }
 
   async get (url, username = null) {
@@ -75,6 +83,9 @@ export class ActivityPubClient {
       headers.signature =
         await this.#sign({ username, url, method, headers })
     }
+    const hostname = (new URL(url)).hostname
+    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    await this.#limiter.limit(hostname)
     this.#logger.debug(`Fetching ${url} with GET`)
     const result = await fetch(url,
       {
@@ -82,6 +93,8 @@ export class ActivityPubClient {
         headers
       }
     )
+    this.#logger.debug({ hostname }, 'updating limiter')
+    await this.#limiter.update(hostname, result.headers)
     this.#logger.debug(`Finished getting ${url}`)
     return result
   }
@@ -115,6 +128,9 @@ export class ActivityPubClient {
     assert.ok(headers)
     this.#logger.debug(`Signing POST for ${url}`)
     headers.signature = await this.#sign({ username, url, method, headers })
+    const hostname = (new URL(url)).hostname
+    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    await this.#limiter.limit(hostname)
     this.#logger.debug(`Fetching POST for ${url}`)
     const res = await fetch(url,
       {
@@ -123,6 +139,8 @@ export class ActivityPubClient {
         body
       }
     )
+    this.#logger.debug({ hostname }, 'updating limiter')
+    await this.#limiter.update(hostname, res.headers)
     this.#logger.debug(`Done fetching POST for ${url}`)
     if (res.status < 200 || res.status > 299) {
       throw createHttpError(res.status, await res.text())

package/lib/app.js

@@ -37,6 +37,7 @@ import { JobQueue } from './jobqueue.js'
 import { JobReaper } from './jobreaper.js'
 import { DeliveryWorker } from './deliveryworker.js'
 import { DistributionWorker } from './distributionworker.js'
+import { RateLimiter } from '../lib/ratelimiter.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
@@ -57,8 +58,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const botDataStorage = new BotDataStorage(connection)
   const keyStorage = new KeyStorage(connection, logger)
   const objectStorage = new ObjectStorage(connection)
-  const client =
-    new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
+  const limiter = new RateLimiter(connection, logger)
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const jobQueue = new JobQueue(connection, logger)

package/lib/migrations/005-rate-limit.js

@@ -0,0 +1,13 @@
+export const id = '005-rate-limit'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE rate_limit (
+      host varchar(256) NOT NULL PRIMARY KEY,
+      remaining INTEGER default 0,
+      reset TIMESTAMP,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+    );
+  `, queryOptions)
+}

package/lib/ratelimiter.js

@@ -0,0 +1,109 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import assert from 'node:assert'
+
+export class RateLimiter {
+  #connection
+  #logger
+
+  constructor (connection, logger) {
+    assert.strictEqual(typeof connection, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    this.#connection = connection
+    this.#logger = logger
+  }
+
+  async limit (host, maxWaitTime = 30000) {
+    assert.strictEqual(typeof host, 'string')
+    assert.strictEqual(typeof maxWaitTime, 'number')
+    const waitTime = await this.#getWaitTime(host)
+    if (waitTime > maxWaitTime) {
+      throw new Error(`Wait time is too long; ${waitTime} > ${maxWaitTime}`)
+    }
+    if (waitTime > 0) {
+      await this.#decrement(host)
+      await sleep(waitTime)
+    }
+  }
+
+  async update (host, headers) {
+    assert.strictEqual(typeof host, 'string')
+    assert.strictEqual(typeof headers, 'object')
+
+    const resetHeader = headers.get('x-ratelimit-reset')
+    const remainingHeader = headers.get('x-ratelimit-remaining')
+
+    if (resetHeader && remainingHeader) {
+      const remaining = parseInt(remainingHeader)
+      const resetSeconds = parseInt(resetHeader)
+      const reset = new Date(Date.now() + (resetSeconds * 1000))
+      this.#logger.debug({ reset, remaining, host }, 'updating')
+      await this.#connection.query(
+        `INSERT INTO rate_limit (host, remaining, reset)
+        VALUES (?, ?, ?)
+        ON CONFLICT (host) DO UPDATE
+        SET remaining = EXCLUDED.remaining,
+            reset = EXCLUDED.reset,
+            updated_at = CURRENT_TIMESTAMP`,
+        { replacements: [host, remaining, reset] }
+      )
+    }
+  }
+
+  async #getWaitTime (host) {
+    assert.strictEqual(typeof host, 'string')
+
+    const [result] = await this.#connection.query(
+      'SELECT remaining, reset FROM rate_limit WHERE host = ?',
+      { replacements: [host] }
+    )
+
+    if (result.length === 0) {
+      return 0
+    }
+
+    const { remaining, reset } = result[0]
+    const resetDate = new Date(reset)
+    const now = new Date()
+
+    if (now > resetDate) {
+      this.#logger.debug(
+        { remaining, resetDate, host },
+        'past epoch'
+      )
+      return 0
+    }
+
+    if (remaining === 0) {
+      this.#logger.debug(
+        { remaining, resetDate, host },
+        'no more requests remaining'
+      )
+      return Math.round(resetDate - now)
+    }
+
+    this.#logger.debug(
+      { remaining, resetDate, host },
+      'no more requests remaining'
+    )
+
+    const waitTime = Math.round((resetDate - now) / (remaining * 1.0))
+
+    this.#logger.debug(
+      { remaining, resetDate, host, waitTime },
+      'no more requests remaining'
+    )
+    return waitTime
+  }
+
+  async #decrement (host) {
+    assert.strictEqual(typeof host, 'string')
+
+    await this.#connection.query(
+      `UPDATE rate_limit
+       SET remaining = remaining - 1,
+          updated_at = CURRENT_TIMESTAMP
+      WHERE host = ?`,
+      { replacements: [host] }
+    )
+  }
+}

package/lib/routes/server.js

@@ -5,32 +5,35 @@ const router = express.Router()
 
 router.get('/', async (req, res) => {
   const fullUrl = `${req.protocol}://${req.get('host')}${req.originalUrl}`
+  const sendJson = async () => {
+    const { formatter } = req.app.locals
+    const server = await as2.import({
+      '@context': [
+        'https://www.w3.org/ns/activitystreams',
+        'https://w3id.org/security/v1'
+      ],
+      id: formatter.format({ server: true }),
+      type: 'Service',
+      publicKey: formatter.format({ server: true, type: 'publickey' }),
+      url: {
+        type: 'Link',
+        mediaType: 'text/html',
+        href: fullUrl
+      }
+    })
+    const body = await server.export({ useOriginalContext: true })
+    res.status(200)
+    res.type(as2.mediaType)
+    res.json(body)
+  }
   res.format({
     html: async () => {
       const { indexFileName } = req.app.locals
       res.sendFile(indexFileName)
     },
-    json: async () => {
-      const { formatter } = req.app.locals
-      const server = await as2.import({
-        '@context': [
-          'https://www.w3.org/ns/activitystreams',
-          'https://w3id.org/security/v1'
-        ],
-        id: formatter.format({ server: true }),
-        type: 'Service',
-        publicKey: formatter.format({ server: true, type: 'publickey' }),
-        url: {
-          type: 'Link',
-          mediaType: 'text/html',
-          href: fullUrl
-        }
-      })
-      const body = await server.export({ useOriginalContext: true })
-      res.status(200)
-      res.type(as2.mediaType)
-      res.json(body)
-    }
+    'application/activity+json': sendJson,
+    'application/ld+json': sendJson,
+    json: sendJson
   })
 })
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.26.2",
+  "version": "0.27.1",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",
@@ -43,7 +43,7 @@
     "sequelize": "^6.37.7"
   },
   "devDependencies": {
-    "@evanp/activitypub-nock": "^0.6.0",
+    "@evanp/activitypub-nock": "^0.7.0",
     "eslint": "^8.57.1",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-import": "^2.29.1",