@evanp/activitypub-bot 0.45.19 → 0.46.0

package/CHANGELOG.md

@@ -9,6 +9,32 @@ and this project adheres to
 
 ## [Unreleased]
 
+## [0.46.0] - 2026-05-25
+
+### Added
+
+- Support for nodeinfo 2.1 and 2.2.
+- ActorStorage method for checking active users.
+- nodeinfo properties for monthly active and half-yearly active users.
+- SafeFetcher service class for all outbound HTTP requests
+
+### Changed
+
+- Name of package in nodeinfo from `activitypub-bot` to `activitypub-dot-bot` to
+  avoid a naming conflict with another package.
+- Use URLFormatter in Transformer and BotContext to short-circuit Webfinger self-request
+- Use SafeFetcher in ActivityPubClient
+- Use SafeFetcher in Transformer
+- Use SafeFetcher in BotContext
+- Encode HTML entities in Transformer.transform()
+
+## [0.45.20] - 2026-05-14
+
+### Fixed
+
+- Handle some pathological formats for request throttle resets -- epoch in
+  ms and offset in ms.
+
 ## [0.45.19] - 2026-05-14
 
 ### Changed

package/README.md

@@ -248,6 +248,12 @@ Alternatively, you can implement a whole class of bots with the [BotFactory](#bo
 
 Bots will receive a [BotContext](#botcontext) object at initialization. The BotContext is the main way to access data or execute activities.
 
+It's important to remember that the bot may run on many different physical
+machines in a multi-server deployment. Don't keep volatile state for the bot in
+memory; it will get out of sync between servers, and will be lost between server
+restarts. Instead, use the `getData()`/`setData()` (see below) interfaces to
+save named data items into the database.
+
 ### Bot
 
 The Bot interface has the following methods.

package/lib/activitypubclient.js

@@ -2,10 +2,6 @@ import assert from 'node:assert'
 import fs from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
-import dns from 'node:dns/promises'
-
-import fetch from 'node-fetch'
-import ipaddr from 'ipaddr.js'
 
 import as2 from './activitystreams.js'
 import { SignaturePolicyStorage } from './signaturepolicystorage.js'
@@ -52,14 +48,6 @@ export class ActivityPubClientError extends Error {
   }
 }
 
-export class ProtocolError extends Error {
-  constructor (url) {
-    super(`URL ${url} uses a disallowed protocol`)
-    this.name = 'ProtocolError'
-    this.url = url
-  }
-}
-
 export class ActivityPubClient {
   static #githubUrl = 'https://github.com/evanp/activitypub-bot'
   static #userAgent = `activitypub.bot/${version} (${ActivityPubClient.#githubUrl})`
@@ -77,9 +65,9 @@ export class ActivityPubClient {
   #cache
   #messageSigner
   #policyStorage
-  #agent
+  #fetcher
 
-  constructor (keyStorage, urlFormatter, signer, digester, logger, throttler, cache, messageSigner, policyStorage, agent = null) {
+  constructor (keyStorage, urlFormatter, signer, digester, logger, throttler, cache, messageSigner, policyStorage, fetcher) {
     assert.strictEqual(typeof keyStorage, 'object')
     assert.strictEqual(typeof urlFormatter, 'object')
     assert.strictEqual(typeof signer, 'object')
@@ -89,6 +77,7 @@ export class ActivityPubClient {
     assert.strictEqual(typeof cache, 'object')
     assert.strictEqual(typeof messageSigner, 'object')
     assert.strictEqual(typeof policyStorage, 'object')
+    assert.strictEqual(typeof fetcher, 'object')
     this.#keyStorage = keyStorage
     this.#urlFormatter = urlFormatter
     this.#signer = signer
@@ -98,11 +87,11 @@ export class ActivityPubClient {
     this.#cache = cache
     this.#messageSigner = messageSigner
     this.#policyStorage = policyStorage
-    this.#agent = agent
+    this.#fetcher = fetcher
   }
 
   get allowPrivateNetworkRequests () {
-    return this.#agent == null
+    return this.#fetcher.allowPrivateNetworkRequests
   }
 
   async get (url, username = this.#urlFormatter.hostname) {
@@ -200,7 +189,7 @@ export class ActivityPubClient {
     this.#logger.debug({ url: baseUrl, hostname }, 'Waiting for throttler')
     await this.#throttler.throttle(hostname)
     this.#logger.debug({ url: baseUrl }, 'Fetching with GET')
-    let res = await this.#fetch(baseUrl,
+    let res = await this.#fetcher.fetch(baseUrl,
       {
         method,
         headers
@@ -218,7 +207,7 @@ export class ActivityPubClient {
       delete headers['signature-input']
       headers.signature =
         await this.#sign({ username, url: baseUrl, method, headers })
-      res = await this.#fetch(baseUrl,
+      res = await this.#fetcher.fetch(baseUrl,
         {
           method,
           headers
@@ -342,7 +331,7 @@ export class ActivityPubClient {
     this.#logger.debug({ url, hostname }, 'Waiting for throttler')
     await this.#throttler.throttle(hostname)
     this.#logger.debug({ url }, 'Fetching POST')
-    let res = await this.#fetch(url,
+    let res = await this.#fetcher.fetch(url,
       {
         method,
         headers,
@@ -363,7 +352,7 @@ export class ActivityPubClient {
       }
       headers.signature =
         await this.#sign({ username, url, method, headers })
-      res = await this.#fetch(url,
+      res = await this.#fetcher.fetch(url,
         {
           method,
           headers,
@@ -530,37 +519,4 @@ export class ActivityPubClient {
     }
     return results
   }
-
-  async #fetch (url, options) {
-    if (!(await this.#checkProtocol(url))) {
-      throw new ProtocolError(url)
-    }
-    const fullOptions = {
-      ...options,
-      agent: this.#agent ?? undefined,
-      signal: options?.signal ?? AbortSignal.timeout(10000),
-      size: 1024 * 1024,
-      follow: 10
-    }
-    return await fetch(url, fullOptions)
-  }
-
-  async #checkProtocol (url) {
-    const parsed = (new URL(url))
-    switch (parsed.protocol) {
-      case 'https:':
-        return true
-      case 'http:': {
-        if (this.#agent) {
-          return false
-        }
-        const { address } = await dns.lookup(parsed.hostname)
-        const addr = ipaddr.parse(address)
-        const range = addr.range()
-        return range !== 'unicast'
-      }
-      default:
-        return false
-    }
-  }
 }

package/lib/actorstorage.js

@@ -3,6 +3,8 @@ import assert from 'node:assert'
 
 const AS2_NS = 'https://www.w3.org/ns/activitystreams#'
 
+const MS_PER_DAY = 24 * 60 * 60 * 1000
+
 export class ActorStorage {
   #connection = null
   #formatter = null
@@ -330,6 +332,24 @@ export class ActorStorage {
     )
   }
 
+  async activeUsers (days) {
+    const [result] = await this.#connection.query(
+      `SELECT COUNT(DISTINCT username) as active_users
+      FROM actorcollectionpage
+      WHERE property = 'outbox'
+      AND createdat > ?;`,
+      { replacements: [new Date(Date.now() - days * MS_PER_DAY)] }
+    )
+    if (result.length > 0) {
+      const row = result[0]
+      return (typeof row.active_users === 'string')
+        ? parseInt(row.active_users, 10)
+        : row.active_users
+    } else {
+      return 0
+    }
+  }
+
   async #getCollectionInfo (username, property) {
     const [result] = await this.#connection.query(
       `SELECT

package/lib/app.js

@@ -51,7 +51,7 @@ import { IntakeWorker } from './intakeworker.js'
 import { RemoteObjectCache } from './remoteobjectcache.js'
 import { HTTPMessageSignature } from './httpmessagesignature.js'
 import { SignaturePolicyStorage } from './signaturepolicystorage.js'
-import { SafeAgent } from './safeagent.js'
+import { SafeFetcher } from './safefetcher.js'
 import { EndpointCache } from './endpointcache.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
@@ -98,10 +98,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const throttler = new RequestThrottler(connection, logger)
   const remoteObjectCache = new RemoteObjectCache(connection, logger)
   const policyStore = new SignaturePolicyStorage(connection, logger)
-  const agent = (allowPrivateNetworkRequests)
-    ? null
-    : new SafeAgent()
-  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, throttler, remoteObjectCache, messageSigner, policyStore, agent)
+  const fetcher = new SafeFetcher({ allowPrivate: allowPrivateNetworkRequests })
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, throttler, remoteObjectCache, messageSigner, policyStore, fetcher)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, messageSigner, digester, logger)
   const jobQueue = new JobQueue(connection, logger)
@@ -134,7 +132,9 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   )
 
   // TODO: Make an endpoint for tagged objects
-  const transformer = new Transformer(origin + '/tag/', client)
+  const transformer = new Transformer(
+    origin + '/tag/', client, fetcher, formatter
+  )
 
   let redisClient
 
@@ -157,7 +157,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
         formatter,
         transformer,
         logger,
-        bots
+        bots,
+        fetcher
       )
     ))
   )
@@ -179,7 +180,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
     formatter,
     transformer,
     logger,
-    bots
+    bots,
+    fetcher
   ))
 
   memoryStatus('bots initialized')

package/lib/botcontext.js

@@ -1,7 +1,6 @@
 import assert from 'node:assert'
 
 import { nanoid } from 'nanoid'
-import fetch from 'node-fetch'
 
 import as2 from './activitystreams.js'
 
@@ -28,6 +27,7 @@ export class BotContext {
   #transformer = null
   #logger = null
   #bots
+  #fetcher
 
   get botId () {
     return this.#botId
@@ -47,8 +47,20 @@ export class BotContext {
     formatter,
     transformer,
     logger,
-    bots
+    bots,
+    fetcher
   ) {
+    assert.strictEqual(typeof botId, 'string')
+    assert.strictEqual(typeof botDataStorage, 'object')
+    assert.strictEqual(typeof objectStorage, 'object')
+    assert.strictEqual(typeof actorStorage, 'object')
+    assert.strictEqual(typeof client, 'object')
+    assert.strictEqual(typeof distributor, 'object')
+    assert.strictEqual(typeof formatter, 'object')
+    assert.strictEqual(typeof transformer, 'object')
+    assert.strictEqual(typeof logger, 'object')
+    assert.strictEqual(typeof bots, 'object')
+    assert.strictEqual(typeof fetcher, 'object')
     this.#botId = botId
     this.#botDataStorage = botDataStorage
     this.#objectStorage = objectStorage
@@ -59,6 +71,7 @@ export class BotContext {
     this.#transformer = transformer
     this.#bots = bots
     this.#logger = logger.child({ class: 'BotContext', botId })
+    this.#fetcher = fetcher
   }
 
   // copy constructor
@@ -73,7 +86,9 @@ export class BotContext {
       this.#distributor,
       this.#formatter,
       this.#transformer,
-      this.#logger
+      this.#logger,
+      this.#bots,
+      this.#fetcher
     )
   }
 
@@ -377,10 +392,13 @@ export class BotContext {
   }
 
   async toActorId (webfinger) {
-    const [, domain] = webfinger.split('@')
+    const [username, domain] = webfinger.split('@')
+    if (domain.toLowerCase() === this.#formatter.hostname) {
+      return this.#formatter.format({ username })
+    }
     const url = `https://${domain}/.well-known/webfinger` +
       `?resource=acct:${webfinger}`
-    const res = await fetch(url)
+    const res = await this.#fetcher.fetch(url)
     if (res.status !== 200) {
       throw new Error(`Status ${res.status} fetching ${url}`)
     }

package/lib/microsyntax.js

@@ -1,20 +1,37 @@
 import assert from 'node:assert'
-import fetch from 'node-fetch'
 
 const AS2 = 'https://www.w3.org/ns/activitystreams#'
 
 export class Transformer {
   #tagNamespace = null
   #client = null
-  constructor (tagNamespace, client) {
+  #fetcher = null
+  #formatter = null
+
+  constructor (tagNamespace, client, fetcher, formatter) {
+    assert.strictEqual(typeof tagNamespace, 'string')
+    assert.strictEqual(typeof client, 'object')
+    assert.strictEqual(typeof fetcher, 'object')
+    assert.strictEqual(typeof formatter, 'object')
+
     this.#tagNamespace = tagNamespace
     this.#client = client
+    this.#fetcher = fetcher
+    this.#formatter = formatter
+
     assert.ok(this.#tagNamespace)
     assert.ok(this.#client)
+    assert.ok(this.#fetcher)
+    assert.ok(this.#formatter)
   }
 
   async transform (text) {
     let html = text
+      .replace(/&/g, '&')
+      .replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;')
+      .replace(/"/g, '&quot;')
+      .replace(/'/g, '&apos;')
     let tag = [];
     ({ html, tag } = this.#replaceUrls(html, tag));
     ({ html, tag } = this.#replaceHashtags(html, tag));
@@ -71,11 +88,14 @@ export class Transformer {
   async #homePage (webfinger) {
     if (!this.#client) return null
     const [username, domain] = webfinger.split('@')
+    if (domain.toLowerCase() === this.#formatter.hostname) {
+      return this.#formatter.format({ username, type: 'profile' })
+    }
     const url = `https://${domain}/.well-known/webfinger?` +
       `resource=acct:${username}@${domain}`
     let json = null
     try {
-      const response = await fetch(url,
+      const response = await this.#fetcher.fetch(url,
         { headers: { Accept: 'application/jrd+json' } })
       if (response.status !== 200) return null
       json = await response.json()

package/lib/migrations/012-actor-outbox-created-at-index.js

@@ -0,0 +1,9 @@
+export const id = '012-actor-outbox-created-at-index'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE INDEX actorcollectionpage_outbox_createdat
+    ON actorcollectionpage (createdat)
+    WHERE property = 'outbox';
+  `, queryOptions)
+}

package/lib/requestthrottler.js

@@ -2,7 +2,8 @@ import { setTimeout as sleep } from 'node:timers/promises'
 import assert from 'node:assert'
 
 const BETA = 0.75
-const EPOCH_THRESHOLD = 30 * 24 * 60 * 60
+const OFFSET_THRESHOLD = 30 * 24 * 60 * 60
+const DEFAULT_RESET = 30 * 1000
 
 export class ThrottleError extends Error {
   constructor (message, waitTime) {
@@ -43,25 +44,33 @@ export class RequestThrottler {
     assert.strictEqual(typeof host, 'string')
     assert.strictEqual(typeof headers, 'object')
 
+    const retryAfterHeader = headers.get('retry-after')
     const resetHeader = headers.get('x-ratelimit-reset')
     const remainingHeader = headers.get('x-ratelimit-remaining')
 
-    if (resetHeader && remainingHeader) {
+    if (retryAfterHeader) {
+      const remaining = 0
+      const reset = this.#headerToReset(retryAfterHeader)
+      this.#logger.debug(
+        { reset, remaining, host, retryAfterHeader },
+        'updating'
+      )
+      await this.#connection.query(
+        `INSERT INTO rate_limit (host, remaining, reset)
+        VALUES (?, ?, ?)
+        ON CONFLICT (host) DO UPDATE
+        SET remaining = EXCLUDED.remaining,
+            reset = EXCLUDED.reset,
+            updated_at = CURRENT_TIMESTAMP`,
+        { replacements: [host, remaining, reset] }
+      )
+    } else if (resetHeader && remainingHeader) {
       const remaining = parseInt(remainingHeader)
-      let resetSeconds
-      let reset
-      if (resetHeader.match(/^\d+$/)) {
-        resetSeconds = parseInt(resetHeader)
-        if (resetSeconds < EPOCH_THRESHOLD) {
-          reset = new Date(Date.now() + (resetSeconds * 1000))
-        } else {
-          reset = new Date(resetSeconds * 1000)
-        }
-      } else {
-        reset = new Date(resetHeader)
-        resetSeconds = reset - Date.now()
-      }
-      this.#logger.debug({ reset, remaining, host }, 'updating')
+      const reset = this.#headerToReset(resetHeader)
+      this.#logger.debug(
+        { reset, remaining, host, resetHeader, remainingHeader },
+        'updating'
+      )
       await this.#connection.query(
         `INSERT INTO rate_limit (host, remaining, reset)
         VALUES (?, ?, ?)
@@ -165,4 +174,39 @@ export class RequestThrottler {
       { replacements: [host] }
     )
   }
+
+  #headerToReset (header) {
+    let reset
+    const now = Date.now()
+    if (header.match(/^\d+$/)) {
+      const num = parseInt(header)
+
+      if (num > now - (3600 * 1000)) {
+        // epoch in ms
+        reset = new Date(num)
+      } else if (num > (now / 1000) - 3600) {
+        // epoch in s
+        reset = new Date(num * 1000)
+      } else if (num < OFFSET_THRESHOLD) {
+        // offset in s
+        reset = new Date(now + (num * 1000))
+      } else if (num < OFFSET_THRESHOLD * 1000) {
+        // offset in ms
+        reset = new Date(now + num)
+      } else {
+        // no good guesses; default
+        reset = new Date(now + DEFAULT_RESET)
+      }
+    } else {
+      reset = new Date(header)
+      if (Number.isNaN(reset.getTime())) {
+        this.#logger.warn(
+          { header },
+          'Error parsing header for request throttling'
+        )
+        reset = new Date(now + DEFAULT_RESET)
+      }
+    }
+    return reset
+  }
 }

package/lib/routes/nodeinfo.js

@@ -9,7 +9,7 @@ import { ProblemDetailsError } from '../errors.js'
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
 
-const packageName = 'activitypub-bot'
+const packageName = 'activitypub-dot-bot'
 
 const { version: packageVersion } = JSON.parse(
   fs.readFileSync(path.join(__dirname, '..', '..', 'package.json'), 'utf8')
@@ -17,7 +17,7 @@ const { version: packageVersion } = JSON.parse(
 
 const router = Router()
 
-const VERSIONS = ['2.0']
+const VERSIONS = ['2.0', '2.1', '2.2']
 
 router.get('/.well-known/nodeinfo', async (req, res, next) => {
   const { origin } = req.app.locals
@@ -33,7 +33,7 @@ router.get('/.well-known/nodeinfo', async (req, res, next) => {
 
 router.get('/nodeinfo/:nodeinfoVersion', async (req, res, next) => {
   const { nodeinfoVersion } = req.params
-  const { keyStorage } = req.app.locals
+  const { keyStorage, actorStorage } = req.app.locals
   if (!VERSIONS.includes(nodeinfoVersion)) {
     throw new ProblemDetailsError(404, 'unsupported nodeinfo version')
   }
@@ -51,7 +51,9 @@ router.get('/nodeinfo/:nodeinfoVersion', async (req, res, next) => {
     openRegistrations: false,
     usage: {
       users: {
-        total: await keyStorage.count()
+        total: await keyStorage.count(),
+        activeMonth: await actorStorage.activeUsers(30),
+        activeHalfyear: await actorStorage.activeUsers(180)
       }
     },
     metadata: {}

package/lib/safefetcher.js

@@ -0,0 +1,112 @@
+import fetch from 'node-fetch'
+import dns from 'node:dns'
+import dnsPromises from 'node:dns/promises'
+import https from 'node:https'
+
+import ipaddr from 'ipaddr.js'
+
+function isPrivateIP (address) {
+  if (!ipaddr.isValid(address)) return false
+  const addr = ipaddr.parse(address)
+  const range = addr.range()
+  return range !== 'unicast'
+}
+
+class PrivateNetworkError extends Error {
+  constructor (address) {
+    super(`Private network address ${address}`)
+    this.name = 'PrivateNetworkError'
+    this.address = address
+  }
+}
+
+class SafeAgent extends https.Agent {
+  constructor (options = {}) {
+    super({
+      keepAlive: true,
+      keepAliveMsecs: 1000,
+      maxSockets: 64,
+      maxFreeSockets: 256,
+      ...options
+    })
+  }
+
+  createConnection (options, callback) {
+    dns.lookup(options.hostname, (err, address) => {
+      if (err) {
+        return callback(err)
+      }
+      if (isPrivateIP(address)) {
+        return callback(new PrivateNetworkError(address))
+      }
+      try {
+        const socket = super.createConnection({
+          ...options,
+          host: address,
+          servername: options.servername || options.hostname
+        })
+        callback(null, socket)
+      } catch (e) {
+        callback(e)
+      }
+    })
+  }
+}
+
+export class ProtocolError extends Error {
+  constructor (url) {
+    super(`URL ${url} uses a disallowed protocol`)
+    this.name = 'ProtocolError'
+    this.url = url
+  }
+}
+
+export class SafeFetcher {
+  #allowPrivate
+  #agent
+
+  constructor (options = {}) {
+    const { allowPrivate } = options
+    this.#allowPrivate = !!allowPrivate
+    this.#agent = (this.#allowPrivate)
+      ? null
+      : new SafeAgent()
+  }
+
+  get allowPrivateNetworkRequests () {
+    return this.#allowPrivate
+  }
+
+  async fetch (url, options) {
+    if (!(await this.#checkProtocol(url))) {
+      throw new ProtocolError(url)
+    }
+    const fullOptions = {
+      ...options,
+      agent: this.#agent ?? undefined,
+      signal: options?.signal ?? AbortSignal.timeout(10000),
+      size: 1024 * 1024,
+      follow: 10
+    }
+    return await fetch(url, fullOptions)
+  }
+
+  async #checkProtocol (url) {
+    const parsed = (new URL(url))
+    switch (parsed.protocol) {
+      case 'https:':
+        return true
+      case 'http:': {
+        if (this.#agent) {
+          return false
+        }
+        const { address } = await dnsPromises.lookup(parsed.hostname)
+        const addr = ipaddr.parse(address)
+        const range = addr.range()
+        return range !== 'unicast'
+      }
+      default:
+        return false
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.45.19",
+  "version": "0.46.0",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",

package/lib/safeagent.js

@@ -1,52 +0,0 @@
-import https from 'node:https'
-import dns from 'node:dns'
-
-import ipaddr from 'ipaddr.js'
-
-function isPrivateIP (address) {
-  if (!ipaddr.isValid(address)) return false
-  const addr = ipaddr.parse(address)
-  const range = addr.range()
-  return range !== 'unicast'
-}
-
-class PrivateNetworkError extends Error {
-  constructor (address) {
-    super(`Private network address ${address}`)
-    this.name = 'PrivateNetworkError'
-    this.address = address
-  }
-}
-
-export class SafeAgent extends https.Agent {
-  constructor (options = {}) {
-    super({
-      keepAlive: true,
-      keepAliveMsecs: 1000,
-      maxSockets: 64,
-      maxFreeSockets: 256,
-      ...options
-    })
-  }
-
-  createConnection (options, callback) {
-    dns.lookup(options.hostname, (err, address) => {
-      if (err) {
-        return callback(err)
-      }
-      if (isPrivateIP(address)) {
-        return callback(new PrivateNetworkError(address))
-      }
-      try {
-        const socket = super.createConnection({
-          ...options,
-          host: address,
-          servername: options.servername || options.hostname
-        })
-        callback(null, socket)
-      } catch (e) {
-        callback(e)
-      }
-    })
-  }
-}