@evanp/activitypub-bot 0.26.3 → 0.28.2

package/lib/activitypubclient.js

@@ -33,13 +33,21 @@ export class ActivityPubClient {
   #signer = null
   #digester = null
   #logger = null
-
-  constructor (keyStorage, urlFormatter, signer, digester, logger) {
+  #limiter
+
+  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter) {
+    assert.strictEqual(typeof keyStorage, 'object')
+    assert.strictEqual(typeof urlFormatter, 'object')
+    assert.strictEqual(typeof signer, 'object')
+    assert.strictEqual(typeof digester, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    assert.strictEqual(typeof limiter, 'object')
     this.#keyStorage = keyStorage
     this.#urlFormatter = urlFormatter
     this.#signer = signer
     this.#digester = digester
     this.#logger = logger.child({ class: this.constructor.name })
+    this.#limiter = limiter
   }
 
   async get (url, username = null) {
@@ -75,6 +83,9 @@ export class ActivityPubClient {
       headers.signature =
         await this.#sign({ username, url, method, headers })
     }
+    const hostname = (new URL(url)).hostname
+    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    await this.#limiter.limit(hostname)
     this.#logger.debug(`Fetching ${url} with GET`)
     const result = await fetch(url,
       {
@@ -82,6 +93,8 @@ export class ActivityPubClient {
         headers
       }
     )
+    this.#logger.debug({ hostname }, 'updating limiter')
+    await this.#limiter.update(hostname, result.headers)
     this.#logger.debug(`Finished getting ${url}`)
     return result
   }
@@ -115,6 +128,9 @@ export class ActivityPubClient {
     assert.ok(headers)
     this.#logger.debug(`Signing POST for ${url}`)
     headers.signature = await this.#sign({ username, url, method, headers })
+    const hostname = (new URL(url)).hostname
+    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    await this.#limiter.limit(hostname)
     this.#logger.debug(`Fetching POST for ${url}`)
     const res = await fetch(url,
       {
@@ -123,6 +139,8 @@ export class ActivityPubClient {
         body
       }
     )
+    this.#logger.debug({ hostname }, 'updating limiter')
+    await this.#limiter.update(hostname, res.headers)
     this.#logger.debug(`Done fetching POST for ${url}`)
     if (res.status < 200 || res.status > 299) {
       throw createHttpError(res.status, await res.text())

package/lib/app.js

@@ -37,6 +37,7 @@ import { JobQueue } from './jobqueue.js'
 import { JobReaper } from './jobreaper.js'
 import { DeliveryWorker } from './deliveryworker.js'
 import { DistributionWorker } from './distributionworker.js'
+import { RateLimiter } from '../lib/ratelimiter.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
@@ -57,8 +58,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const botDataStorage = new BotDataStorage(connection)
   const keyStorage = new KeyStorage(connection, logger)
   const objectStorage = new ObjectStorage(connection)
-  const client =
-    new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
+  const limiter = new RateLimiter(connection, logger)
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const jobQueue = new JobQueue(connection, logger)

package/lib/migrations/005-rate-limit.js

@@ -0,0 +1,13 @@
+export const id = '005-rate-limit'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE rate_limit (
+      host varchar(256) NOT NULL PRIMARY KEY,
+      remaining INTEGER default 0,
+      reset TIMESTAMP,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+    );
+  `, queryOptions)
+}

package/lib/ratelimiter.js

@@ -0,0 +1,109 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import assert from 'node:assert'
+
+export class RateLimiter {
+  #connection
+  #logger
+
+  constructor (connection, logger) {
+    assert.strictEqual(typeof connection, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    this.#connection = connection
+    this.#logger = logger
+  }
+
+  async limit (host, maxWaitTime = 30000) {
+    assert.strictEqual(typeof host, 'string')
+    assert.strictEqual(typeof maxWaitTime, 'number')
+    const waitTime = await this.#getWaitTime(host)
+    if (waitTime > maxWaitTime) {
+      throw new Error(`Wait time is too long; ${waitTime} > ${maxWaitTime}`)
+    }
+    if (waitTime > 0) {
+      await this.#decrement(host)
+      await sleep(waitTime)
+    }
+  }
+
+  async update (host, headers) {
+    assert.strictEqual(typeof host, 'string')
+    assert.strictEqual(typeof headers, 'object')
+
+    const resetHeader = headers.get('x-ratelimit-reset')
+    const remainingHeader = headers.get('x-ratelimit-remaining')
+
+    if (resetHeader && remainingHeader) {
+      const remaining = parseInt(remainingHeader)
+      const resetSeconds = parseInt(resetHeader)
+      const reset = new Date(Date.now() + (resetSeconds * 1000))
+      this.#logger.debug({ reset, remaining, host }, 'updating')
+      await this.#connection.query(
+        `INSERT INTO rate_limit (host, remaining, reset)
+        VALUES (?, ?, ?)
+        ON CONFLICT (host) DO UPDATE
+        SET remaining = EXCLUDED.remaining,
+            reset = EXCLUDED.reset,
+            updated_at = CURRENT_TIMESTAMP`,
+        { replacements: [host, remaining, reset] }
+      )
+    }
+  }
+
+  async #getWaitTime (host) {
+    assert.strictEqual(typeof host, 'string')
+
+    const [result] = await this.#connection.query(
+      'SELECT remaining, reset FROM rate_limit WHERE host = ?',
+      { replacements: [host] }
+    )
+
+    if (result.length === 0) {
+      return 0
+    }
+
+    const { remaining, reset } = result[0]
+    const resetDate = new Date(reset)
+    const now = new Date()
+
+    if (now > resetDate) {
+      this.#logger.debug(
+        { remaining, resetDate, host },
+        'past epoch'
+      )
+      return 0
+    }
+
+    if (remaining === 0) {
+      this.#logger.debug(
+        { remaining, resetDate, host },
+        'no more requests remaining'
+      )
+      return Math.round(resetDate - now)
+    }
+
+    this.#logger.debug(
+      { remaining, resetDate, host },
+      'no more requests remaining'
+    )
+
+    const waitTime = Math.round((resetDate - now) / (remaining * 1.0))
+
+    this.#logger.debug(
+      { remaining, resetDate, host, waitTime },
+      'no more requests remaining'
+    )
+    return waitTime
+  }
+
+  async #decrement (host) {
+    assert.strictEqual(typeof host, 'string')
+
+    await this.#connection.query(
+      `UPDATE rate_limit
+       SET remaining = remaining - 1,
+          updated_at = CURRENT_TIMESTAMP
+      WHERE host = ?`,
+      { replacements: [host] }
+    )
+  }
+}

package/lib/routes/server.js

@@ -4,37 +4,32 @@ import as2 from '../activitystreams.js'
 const router = express.Router()
 
 router.get('/', async (req, res) => {
-  const fullUrl = `${req.protocol}://${req.get('host')}${req.originalUrl}`
-  const sendJson = async () => {
-    const { formatter } = req.app.locals
-    const server = await as2.import({
-      '@context': [
-        'https://www.w3.org/ns/activitystreams',
-        'https://w3id.org/security/v1'
-      ],
-      id: formatter.format({ server: true }),
-      type: 'Service',
-      publicKey: formatter.format({ server: true, type: 'publickey' }),
-      url: {
-        type: 'Link',
-        mediaType: 'text/html',
-        href: fullUrl
-      }
-    })
-    const body = await server.export({ useOriginalContext: true })
-    res.status(200)
-    res.type(as2.mediaType)
-    res.json(body)
-  }
-  res.format({
-    html: async () => {
-      const { indexFileName } = req.app.locals
-      res.sendFile(indexFileName)
-    },
-    'application/activity+json': sendJson,
-    'application/ld+json': sendJson,
-    json: sendJson
+  const { indexFileName } = req.app.locals
+  res.type('html')
+  res.sendFile(indexFileName)
+})
+
+router.get('/actor', async (req, res) => {
+  const homepage = `${req.protocol}://${req.get('host')}/`
+  const { formatter } = req.app.locals
+  const server = await as2.import({
+    '@context': [
+      'https://www.w3.org/ns/activitystreams',
+      'https://w3id.org/security/v1'
+    ],
+    id: formatter.format({ server: true }),
+    type: 'Service',
+    publicKey: formatter.format({ server: true, type: 'publickey' }),
+    url: {
+      type: 'Link',
+      mediaType: 'text/html',
+      href: homepage
+    }
   })
+  const body = await server.export({ useOriginalContext: true })
+  res.status(200)
+  res.type(as2.mediaType)
+  res.json(body)
 })
 
 router.get('/publickey', async (req, res) => {

package/lib/urlformatter.js

@@ -41,7 +41,7 @@ export class UrlFormatter {
     }
     // For the base case, we want a trailing slash.
     if (url === this.#origin) {
-      url = `${url}/`
+      url = `${url}/actor`
     }
     return url
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.26.3",
+  "version": "0.28.2",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",
@@ -43,7 +43,7 @@
     "sequelize": "^6.37.7"
   },
   "devDependencies": {
-    "@evanp/activitypub-nock": "^0.6.0",
+    "@evanp/activitypub-nock": "^0.7.0",
     "eslint": "^8.57.1",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-import": "^2.29.1",
@@ -55,7 +55,7 @@
   "optionalDependencies": {
     "mysql2": "^3.9.1",
     "pg": "^8.16.3",
-    "sqlite3": "^5.1.7"
+    "sqlite3": "^6.0.1"
   },
   "engines": {
     "node": "^20 || ^22 || ^24 || ^25"