@evanp/activitypub-bot 0.39.6 → 0.40.1

package/bin/activitypub-bot.js

@@ -17,6 +17,7 @@ const { values } = parseArgs({
     fanout: { type: 'string' },
     intake: { type: 'string' },
     'index-file': { type: 'string' },
+    'allow-private': { type: 'boolean' },
     help: { type: 'boolean', short: 'h' }
   },
   allowPositionals: false
@@ -37,6 +38,7 @@ Options:
   --intake <number>          Number of background intake workers
   --index-file <path>        HTML page to show at root path
   --profile-file <path>      HTML page to show for bot profiles
+  --allow-private            flag to allow private network requests
   -h, --help                 Show this help
 `)
   process.exit(0)
@@ -48,6 +50,13 @@ const parseNumber = (value) => {
   const parsed = Number.parseInt(value, 10)
   return Number.isNaN(parsed) ? undefined : parsed
 }
+function parseBoolean (value) {
+  if (value == null || value === '') return undefined
+  const normalized = value.trim().toLowerCase()
+  if (['1', 'true', 'yes', 'on'].includes(normalized)) return true
+  if (['0', 'false', 'no', 'off'].includes(normalized)) return false
+  return undefined
+}
 
 const baseDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_BOTS_CONFIG_FILE = resolve(baseDir, '..', 'bots', 'index.js')
@@ -69,6 +78,10 @@ const FANOUT = parseNumber(values.fanout) || parseNumber(process.env.FANOUT) ||
 const INTAKE = parseNumber(values.intake) || parseNumber(process.env.INTAKE) || 2
 const INDEX_FILE = values['index-file'] || process.env.INDEX_FILE || DEFAULT_INDEX_FILE
 const PROFILE_FILE = values['profile-file'] || process.env.PROFILE_FILE || DEFAULT_PROFILE_FILE
+const ALLOW_PRIVATE = values['allow-private'] ||
+  ('ALLOW_PRIVATE' in process.env)
+  ? parseBoolean(process.env.ALLOW_PRIVATE)
+  : false
 
 const bots = (await import(BOTS_CONFIG_FILE)).default
 
@@ -82,7 +95,8 @@ const app = await makeApp({
   fanoutWorkerCount: FANOUT,
   intakeWorkerCount: INTAKE,
   indexFileName: INDEX_FILE,
-  profileFileName: PROFILE_FILE
+  profileFileName: PROFILE_FILE,
+  allowPrivateNetworkRequests: ALLOW_PRIVATE
 })
 
 const server = app.listen(parseInt(PORT), () => {

package/lib/activitypubclient.js

@@ -2,8 +2,10 @@ import assert from 'node:assert'
 import fs from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
+import dns from 'node:dns/promises'
 
 import fetch from 'node-fetch'
+import ipaddr from 'ipaddr.js'
 
 import as2 from './activitystreams.js'
 import { SignaturePolicyStorage } from './signaturepolicystorage.js'
@@ -48,6 +50,14 @@ export class ActivityPubClientError extends Error {
   }
 }
 
+export class ProtocolError extends Error {
+  constructor (url) {
+    super(`URL ${url} uses a disallowed protocol`)
+    this.name = 'ProtocolError'
+    this.url = url
+  }
+}
+
 export class ActivityPubClient {
   static #githubUrl = 'https://github.com/evanp/activitypub-bot'
   static #userAgent = `activitypub.bot/${version} (${ActivityPubClient.#githubUrl})`
@@ -61,18 +71,19 @@ export class ActivityPubClient {
   #signer = null
   #digester = null
   #logger = null
-  #limiter
+  #throttler
   #cache
   #messageSigner
   #policyStorage
+  #agent
 
-  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter, cache, messageSigner, policyStorage) {
+  constructor (keyStorage, urlFormatter, signer, digester, logger, throttler, cache, messageSigner, policyStorage, agent = null) {
     assert.strictEqual(typeof keyStorage, 'object')
     assert.strictEqual(typeof urlFormatter, 'object')
     assert.strictEqual(typeof signer, 'object')
     assert.strictEqual(typeof digester, 'object')
     assert.strictEqual(typeof logger, 'object')
-    assert.strictEqual(typeof limiter, 'object')
+    assert.strictEqual(typeof throttler, 'object')
     assert.strictEqual(typeof cache, 'object')
     assert.strictEqual(typeof messageSigner, 'object')
     assert.strictEqual(typeof policyStorage, 'object')
@@ -81,10 +92,15 @@ export class ActivityPubClient {
     this.#signer = signer
     this.#digester = digester
     this.#logger = logger.child({ class: this.constructor.name })
-    this.#limiter = limiter
+    this.#throttler = throttler
     this.#cache = cache
     this.#messageSigner = messageSigner
     this.#policyStorage = policyStorage
+    this.#agent = agent
+  }
+
+  get allowPrivateNetworkRequests () {
+    return this.#agent == null
   }
 
   async get (url, username = this.#urlFormatter.hostname) {
@@ -179,10 +195,10 @@ export class ActivityPubClient {
       }
     }
     const hostname = parsed.hostname
-    this.#logger.debug({ url: baseUrl, hostname }, 'Waiting for rate limiter')
-    await this.#limiter.limit(hostname)
+    this.#logger.debug({ url: baseUrl, hostname }, 'Waiting for throttler')
+    await this.#throttler.throttle(hostname)
     this.#logger.debug({ url: baseUrl }, 'Fetching with GET')
-    let res = await fetch(baseUrl,
+    let res = await this.#fetch(baseUrl,
       {
         method,
         headers
@@ -200,7 +216,7 @@ export class ActivityPubClient {
       delete headers['signature-input']
       headers.signature =
         await this.#sign({ username, url: baseUrl, method, headers })
-      res = await fetch(baseUrl,
+      res = await this.#fetch(baseUrl,
         {
           method,
           headers
@@ -208,7 +224,7 @@ export class ActivityPubClient {
       )
     }
 
-    await this.#limiter.update(hostname, res.headers)
+    await this.#throttler.update(hostname, res.headers)
     this.#logger.debug({ url }, 'Finished GET')
     if (useCache && res.status === 304) {
       this.#logger.debug({ baseUrl }, '304 Not Modified, returning cached object')
@@ -316,10 +332,10 @@ export class ActivityPubClient {
       throw new Error(`Unexpected signature policy ${storedPolicy}`)
     }
     const hostname = parsed.hostname
-    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
-    await this.#limiter.limit(hostname)
+    this.#logger.debug({ url, hostname }, 'Waiting for throttler')
+    await this.#throttler.throttle(hostname)
     this.#logger.debug({ url }, 'Fetching POST')
-    let res = await fetch(url,
+    let res = await this.#fetch(url,
       {
         method,
         headers,
@@ -339,7 +355,7 @@ export class ActivityPubClient {
       }
       headers.signature =
         await this.#sign({ username, url, method, headers })
-      res = await fetch(url,
+      res = await this.#fetch(url,
         {
           method,
           headers,
@@ -347,8 +363,8 @@ export class ActivityPubClient {
         }
       )
     }
-    this.#logger.debug({ hostname }, 'updating limiter')
-    await this.#limiter.update(hostname, res.headers)
+    this.#logger.debug({ hostname }, 'updating throttler')
+    await this.#throttler.update(hostname, res.headers)
     this.#logger.debug({ url }, 'Done fetching POST')
     if (res.status < 200 || res.status > 299) {
       const body = await res.text()
@@ -503,4 +519,37 @@ export class ActivityPubClient {
     }
     return results
   }
+
+  async #fetch (url, options) {
+    if (!(await this.#checkProtocol(url))) {
+      throw new ProtocolError(url)
+    }
+    const fullOptions = {
+      ...options,
+      agent: this.#agent ?? undefined,
+      timeout: 10000,
+      size: 1024 * 1024,
+      follow: 10
+    }
+    return await fetch(url, fullOptions)
+  }
+
+  async #checkProtocol (url) {
+    const parsed = (new URL(url))
+    switch (parsed.protocol) {
+      case 'https:':
+        return true
+      case 'http:': {
+        if (this.#agent) {
+          return false
+        }
+        const { address } = await dns.lookup(parsed.hostname)
+        const addr = ipaddr.parse(address)
+        const range = addr.range()
+        return range !== 'unicast'
+      }
+      default:
+        return false
+    }
+  }
 }

package/lib/app.js

@@ -7,6 +7,8 @@ import { Sequelize } from 'sequelize'
 import express from 'express'
 import Logger from 'pino'
 import HTTPLogger from 'pino-http'
+import { RedisStore } from 'rate-limit-redis'
+import { createClient } from 'redis'
 
 import { ActivityDistributor } from './activitydistributor.js'
 import { ActivityPubClient, ActivityPubClientError } from './activitypubclient.js'
@@ -40,13 +42,15 @@ import { JobQueue } from './jobqueue.js'
 import { JobReaper } from './jobreaper.js'
 import { DeliveryWorker } from './deliveryworker.js'
 import { DistributionWorker } from './distributionworker.js'
-import { RateLimiter } from '../lib/ratelimiter.js'
+import { RequestThrottler } from '../lib/requestthrottler.js'
 import DoNothingBot from './bots/donothing.js'
 import { FanoutWorker } from './fanoutworker.js'
 import { IntakeWorker } from './intakeworker.js'
 import { RemoteObjectCache } from './remoteobjectcache.js'
 import { HTTPMessageSignature } from './httpmessagesignature.js'
 import { SignaturePolicyStorage } from './signaturepolicystorage.js'
+import { SafeAgent } from './safeagent.js'
+import { rateLimit } from 'express-rate-limit'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
@@ -59,7 +63,7 @@ function createWorkers (logger, count, WorkerClass, ...args) {
   return { workers, runs }
 }
 
-export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, fanoutWorkerCount = 4, intakeWorkerCount = 2, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME }) {
+export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent', deliveryWorkerCount = 2, distributionWorkerCount = 8, fanoutWorkerCount = 4, intakeWorkerCount = 2, indexFileName = DEFAULT_INDEX_FILENAME, profileFileName = DEFAULT_PROFILE_FILENAME, allowPrivateNetworkRequests = false, redisUrl }) {
   const logger = Logger({
     level: logLevel
   })
@@ -76,10 +80,13 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const botDataStorage = new BotDataStorage(connection)
   const keyStorage = new KeyStorage(connection, logger)
   const objectStorage = new ObjectStorage(connection)
-  const limiter = new RateLimiter(connection, logger)
+  const throttler = new RequestThrottler(connection, logger)
   const remoteObjectCache = new RemoteObjectCache(connection, logger)
   const policyStore = new SignaturePolicyStorage(connection, logger)
-  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter, remoteObjectCache, messageSigner, policyStore)
+  const agent = (allowPrivateNetworkRequests)
+    ? null
+    : new SafeAgent()
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, throttler, remoteObjectCache, messageSigner, policyStore, agent)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, messageSigner, digester, logger)
   const jobQueue = new JobQueue(connection, logger)
@@ -117,6 +124,13 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   // TODO: Make an endpoint for tagged objects
   const transformer = new Transformer(origin + '/tag/', client)
 
+  let redisClient
+
+  if (redisUrl) {
+    redisClient = createClient({ url: redisUrl })
+    await redisClient.connect()
+  }
+
   await Promise.all(
     Object.entries(bots).map(([key, bot]) => bot.initialize(
       new BotContext(
@@ -188,6 +202,65 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
 
   app.disable('x-powered-by')
 
+  const rateLimitHandler = (req, res) => {
+    res.status(429)
+      .set('Content-Type', 'application/problem+json')
+      .send({
+        type: 'about:blank',
+        status: 429,
+        title: 'Too Many Requests',
+        detail: 'You made too many requests.'
+      })
+  }
+
+  app.use(rateLimit({
+    identifier: 'steady-get',
+    windowMs: 60 * 1000,
+    limit: 2400,
+    standardHeaders: 'draft-8',
+    legacyHeaders: false,
+    skip: (req, res) => !['HEAD', 'GET'].includes(req.method),
+    handler: rateLimitHandler,
+    store: (redisClient)
+      ? new RedisStore({
+        prefix: `${origin}:steady-get:`,
+        sendCommand: (...args) => redisClient.sendCommand(args)
+      })
+      : undefined
+  }))
+
+  app.use(rateLimit({
+    identifier: 'burst-get',
+    windowMs: 10 * 1000,
+    limit: 1000,
+    standardHeaders: 'draft-8',
+    legacyHeaders: false,
+    skip: (req, res) => !['HEAD', 'GET'].includes(req.method),
+    handler: rateLimitHandler,
+    store: (redisClient)
+      ? new RedisStore({
+        prefix: `${origin}:burst-get:`,
+        sendCommand: (...args) => redisClient.sendCommand(args)
+      })
+      : undefined
+  }))
+
+  app.use(rateLimit({
+    identifier: 'post',
+    windowMs: 60 * 1000,
+    limit: 600,
+    standardHeaders: 'draft-8',
+    legacyHeaders: false,
+    skip: (req, res) => req.method !== 'POST',
+    handler: rateLimitHandler,
+    store: (redisClient)
+      ? new RedisStore({
+        prefix: `${origin}:post:`,
+        sendCommand: (...args) => redisClient.sendCommand(args)
+      })
+      : undefined
+  }))
+
   app.use(async (req, res, next) => {
     let id = req.get('x-request-id')
     if (!id || !id.match(UUID_REGEXP)) {
@@ -328,6 +401,10 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
     await Promise.allSettled(distributionWorkerRuns)
     await Promise.allSettled(deliveryWorkerRuns)
     await jobReaperRun
+    if (redisClient) {
+      logger.info('Closing Redis connection')
+      await redisClient.disconnect()
+    }
     logger.info('Closing database connection')
     await connection.close()
     logger.info('Done')

package/lib/migrations/009-timestamp-to-timestampz.js

@@ -0,0 +1,50 @@
+export const id = '009-timestamp-to-timestampz'
+
+const COLUMNS = [
+  ['actorcollection', 'createdat'],
+  ['actorcollection', 'updatedat'],
+  ['actorcollectionpage', 'createdat'],
+  ['objects', 'createdat'],
+  ['objects', 'updatedat'],
+  ['collections', 'createdat'],
+  ['collections', 'updatedat'],
+  ['pages', 'createdat'],
+  ['botdata', 'createdat'],
+  ['botdata', 'updatedat'],
+  ['new_remotekeys', 'createdat'],
+  ['new_remotekeys', 'updatedat'],
+  ['lastactivity', 'createdat'],
+  ['lastactivity', 'updatedat'],
+  ['job', 'claimed_at'],
+  ['job', 'retry_after'],
+  ['job', 'created_at'],
+  ['job', 'updated_at'],
+  ['rate_limit', 'reset'],
+  ['rate_limit', 'created_at'],
+  ['rate_limit', 'updated_at'],
+  ['failed_job', 'claimed_at'],
+  ['failed_job', 'retry_after'],
+  ['failed_job', 'created_at'],
+  ['failed_job', 'updated_at'],
+  ['remote_object_cache', 'last_modified'],
+  ['remote_object_cache', 'expiry'],
+  ['remote_object_cache', 'created_at'],
+  ['remote_object_cache', 'updated_at'],
+  ['signature_policy', 'expiry'],
+  ['signature_policy', 'created_at'],
+  ['signature_policy', 'updated_at']
+]
+
+export async function up (connection, queryOptions = {}) {
+  if (connection.getDialect() !== 'postgres') {
+    return
+  }
+  for (const [table, column] of COLUMNS) {
+    await connection.query(
+      `ALTER TABLE "${table}"
+       ALTER COLUMN "${column}" TYPE TIMESTAMPTZ
+       USING "${column}" AT TIME ZONE 'UTC'`,
+      queryOptions
+    )
+  }
+}

package/lib/{ratelimiter.js → requestthrottler.js}

@@ -3,7 +3,7 @@ import assert from 'node:assert'
 
 const BETA = 0.75
 
-export class RateLimiter {
+export class RequestThrottler {
   #connection
   #logger
 
@@ -14,7 +14,7 @@ export class RateLimiter {
     this.#logger = logger.child({ class: this.constructor.name })
   }
 
-  async limit (host, maxWaitTime = 30000) {
+  async throttle (host, maxWaitTime = 30000) {
     assert.strictEqual(typeof host, 'string')
     assert.strictEqual(typeof maxWaitTime, 'number')
     const waitTime = await this.#getWaitTime(host, maxWaitTime)

package/lib/routes/webfinger.js

@@ -86,6 +86,10 @@ router.get('/.well-known/webfinger', async (req, res, next) => {
   if (!resource) {
     return next(new ProblemDetailsError(400, 'resource parameter is required'))
   }
+  if (Array.isArray(resource)) {
+    assert.ok(resource.length > 1)
+    throw new ProblemDetailsError(400, '"resource" parameter required exactly once')
+  }
   const colon = resource.indexOf(':')
   const protocol = resource.slice(0, colon)
 

package/lib/safeagent.js

@@ -0,0 +1,33 @@
+import https from 'node:https'
+import dns from 'node:dns'
+
+import ipaddr from 'ipaddr.js'
+
+function isPrivateIP (address) {
+  if (!ipaddr.isValid(address)) return false
+  const addr = ipaddr.parse(address)
+  const range = addr.range()
+  return range !== 'unicast'
+}
+
+class PrivateNetworkError extends Error {
+  constructor (address) {
+    super(`Private network address ${address}`)
+    this.name = 'PrivateNetworkError'
+    this.address = address
+  }
+}
+
+export class SafeAgent extends https.Agent {
+  createConnection (options, callback) {
+    dns.lookup(options.hostname, (err, address) => {
+      if (err) {
+        return callback(err)
+      }
+      if (isPrivateIP(address)) {
+        return callback(new PrivateNetworkError(address))
+      }
+      super.createConnection({ ...options, hostname: address }, callback)
+    })
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.39.6",
+  "version": "0.40.1",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",
@@ -9,6 +9,7 @@
   },
   "scripts": {
     "test": "NODE_ENV=test node --test",
+    "test:serial": "NODE_ENV=test node --test --test-concurrency=1",
     "start": "npx activitypub-bot",
     "lint": "eslint ."
   },
@@ -32,12 +33,16 @@
     "@isaacs/ttlcache": "^2.1.4",
     "activitystrea.ms": "^3.3.0",
     "express": "^5.2.1",
+    "express-rate-limit": "^8.3.2",
     "humanhash": "^1.0.4",
+    "ipaddr.js": "^2.3.0",
     "lru-cache": "^11.1.0",
     "nanoid": "^5.1.5",
     "node-fetch": "^3.3.2",
     "pino": "^10.1.1",
     "pino-http": "^11.0.0",
+    "rate-limit-redis": "^4.3.1",
+    "redis": "^5.11.0",
     "sequelize": "^6.37.7"
   },
   "devDependencies": {