@evanp/activitypub-bot 0.35.0 → 0.36.1

package/lib/activitypubclient.js

@@ -36,20 +36,23 @@ export class ActivityPubClient {
   #digester = null
   #logger = null
   #limiter
+  #cache
 
-  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter) {
+  constructor (keyStorage, urlFormatter, signer, digester, logger, limiter, cache) {
     assert.strictEqual(typeof keyStorage, 'object')
     assert.strictEqual(typeof urlFormatter, 'object')
     assert.strictEqual(typeof signer, 'object')
     assert.strictEqual(typeof digester, 'object')
     assert.strictEqual(typeof logger, 'object')
     assert.strictEqual(typeof limiter, 'object')
+    assert.strictEqual(typeof cache, 'object')
     this.#keyStorage = keyStorage
     this.#urlFormatter = urlFormatter
     this.#signer = signer
     this.#digester = digester
     this.#logger = logger.child({ class: this.constructor.name })
     this.#limiter = limiter
+    this.#cache = cache
   }
 
   async get (url, username = this.#urlFormatter.hostname) {
@@ -58,85 +61,138 @@ export class ActivityPubClient {
     assert.ok(username)
     assert.equal(typeof username, 'string')
     assert.ok(username !== '*')
-    const res = await this.#getRes(url, username, true)
-    return await this.#handleRes(res, url)
+
+    return await this.#get(url, username, true)
   }
 
   async getKey (url) {
     assert.ok(url)
     assert.equal(typeof url, 'string')
-    let res = await this.#getRes(url, this.#urlFormatter.hostname, false)
-    if ([401, 403, 404].includes(res.status)) {
-      // If we get a 401, 403, or 404, we should try again with the key
-      res = await this.#getRes(url, this.#urlFormatter.hostname, true)
+    let obj
+    try {
+      obj = await this.#get(url, this.#urlFormatter.hostname, false, false)
+    } catch (err) {
+      if (err.status && [401, 403, 404].includes(err.status)) {
+        obj = await this.#get(url, this.#urlFormatter.hostname, true, false)
+      } else {
+        throw err
+      }
     }
-    return await this.#handleRes(res, url)
+    return obj
+  }
+
+  async #get (url, username = this.#urlFormatter.hostname, sign = false, useCache = true) {
+    const json = await this.#getJSON(url, username, sign, useCache)
+    const obj = await this.#getObj(url, json)
+    return obj
   }
 
-  async #getRes (url, username = this.#urlFormatter.hostname, sign = false) {
+  async #getJSON (url, username = this.#urlFormatter.hostname, sign = false, useCache = true) {
     assert.ok(url)
     assert.equal(typeof url, 'string')
     assert.ok(username)
     assert.equal(typeof username, 'string')
+
+    const parsed = new URL(url)
+    const baseUrl = `${parsed.origin}${parsed.pathname}${parsed.search}`
+    let cached
+
+    if (useCache) {
+      cached = await this.#cache.get(baseUrl, username)
+      this.#logger.debug({ baseUrl, username, cached: !!cached, expiry: cached?.expiry }, 'cache lookup')
+
+      if (cached) {
+        this.#logger.debug({ baseUrl }, 'cache hit')
+        const now = Date.now()
+        const expiry = cached.expiry.getTime()
+        if (expiry > now) {
+          this.#logger.debug({ baseUrl, expiry, now }, 'cache fresh, returning cached object')
+          return cached.object
+        } else {
+          this.#logger.debug({ baseUrl, expiry, now }, 'cache stale, revalidating')
+        }
+      }
+    }
+
     const date = new Date().toUTCString()
     const headers = {
       accept: ActivityPubClient.#accept,
       date,
       'user-agent': ActivityPubClient.#userAgent
     }
+    if (cached?.lastModified) {
+      headers['if-modified-since'] = cached.lastModified.toUTCString()
+    }
+    if (cached?.etag) {
+      headers['if-none-match'] = cached.etag
+    }
+    this.#logger.debug({ headers }, 'Sending headers')
     const method = 'GET'
-    this.#logger.debug(`Signing GET request for ${url}`)
+    this.#logger.debug(`Signing GET request for ${baseUrl}`)
     if (sign) {
       headers.signature =
-        await this.#sign({ username, url, method, headers })
+        await this.#sign({ username, url: baseUrl, method, headers })
     }
-    const hostname = (new URL(url)).hostname
-    this.#logger.debug({ url, hostname }, 'Waiting for rate limiter')
+    const hostname = parsed.hostname
+    this.#logger.debug({ url: baseUrl, hostname }, 'Waiting for rate limiter')
     await this.#limiter.limit(hostname)
-    this.#logger.debug(`Fetching ${url} with GET`)
-    const result = await fetch(url,
+    this.#logger.debug(`Fetching ${baseUrl} with GET`)
+    const res = await fetch(baseUrl,
       {
         method,
         headers
       }
     )
-    this.#logger.debug({ hostname }, 'updating limiter')
-    await this.#limiter.update(hostname, result.headers)
+    this.#logger.debug({ hostname, status: res.status }, 'response received')
+    await this.#limiter.update(hostname, res.headers)
     this.#logger.debug(`Finished getting ${url}`)
-    return result
-  }
-
-  async #handleRes (res, url) {
-    if (res.status < 200 || res.status > 299) {
+    if (useCache && res.status === 304) {
+      this.#logger.debug({ baseUrl }, '304 Not Modified, returning cached object')
+      return cached.object
+    } else if (res.status < 200 || res.status > 299) {
       const body = await res.text()
-      this.#logger.warn({ status: res.status, body, url }, 'Could not fetch url')
+      this.#logger.warn(
+        { status: res.status, body, url: baseUrl },
+        'Could not fetch url'
+      )
       throw createHttpError(
         res.status,
-        `Could not fetch ${url}`,
+        `Could not fetch ${baseUrl}`,
         { headers: res.headers }
       )
     }
+
     const contentType = res.headers.get('content-type')
     const mimeType = contentType?.split(';')[0].trim()
     if (mimeType !== 'application/json' && !mimeType.endsWith('+json')) {
-      this.#logger.warn({ mimeType, url }, 'Unexpected mime type')
+      this.#logger.warn({ mimeType, url: baseUrl }, 'Unexpected mime type')
       throw new Error(`Got unexpected mime type ${mimeType} for URL ${url}`)
     }
     let json
     try {
       json = await res.json()
     } catch (err) {
-      this.#logger.warn({ url }, 'Error parsing fetch results')
+      this.#logger.warn({ url: baseUrl }, 'Error parsing fetch results')
       throw err
     }
+
+    await this.#cache.set(baseUrl, username, json, res.headers)
+
+    return json
+  }
+
+  async #getObj (url, json) {
+    const parsed = new URL(url)
+    const baseUrl = `${parsed.origin}${parsed.pathname}${parsed.search}`
+
     let obj
     try {
       obj = await as2.import(json)
     } catch (err) {
-      this.#logger.warn({ url, json }, 'Error importing JSON as AS2')
+      this.#logger.warn({ url: baseUrl, json }, 'Error importing JSON as AS2')
       throw err
     }
-    const resolved = (URL.parse(url).hash)
+    const resolved = (parsed.hash)
       ? this.#resolveObject(obj, url)
       : obj
     return resolved

package/lib/app.js

@@ -43,6 +43,7 @@ import { RateLimiter } from '../lib/ratelimiter.js'
 import DoNothingBot from './bots/donothing.js'
 import { FanoutWorker } from './fanoutworker.js'
 import { IntakeWorker } from './intakeworker.js'
+import { RemoteObjectCache } from './remoteobjectcache.js'
 
 const currentDir = dirname(fileURLToPath(import.meta.url))
 const DEFAULT_INDEX_FILENAME = resolve(currentDir, '..', 'web', 'index.html')
@@ -71,7 +72,8 @@ export async function makeApp ({ databaseUrl, origin, bots, logLevel = 'silent',
   const keyStorage = new KeyStorage(connection, logger)
   const objectStorage = new ObjectStorage(connection)
   const limiter = new RateLimiter(connection, logger)
-  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter)
+  const remoteObjectCache = new RemoteObjectCache(connection, logger)
+  const client = new ActivityPubClient(keyStorage, formatter, signer, digester, logger, limiter, remoteObjectCache)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const jobQueue = new JobQueue(connection, logger)

package/lib/migrations/007-remote-object-cache.js

@@ -0,0 +1,20 @@
+export const id = '007-remote-object-cache'
+
+export async function up (connection, queryOptions = {}) {
+  await connection.query(`
+    CREATE TABLE remote_object_cache (
+      id varchar(512) NOT NULL,
+      username varchar(512) NOT NULL,
+      last_modified TIMESTAMP NULL,
+      etag varchar(256) NULL,
+      expiry TIMESTAMP NULL,
+      data TEXT NOT NULL,
+      created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (id, username)
+    );
+  `, queryOptions)
+  await connection.query(`
+    CREATE INDEX remote_object_cache_expiry on remote_object_cache (expiry);
+  `, queryOptions)
+}

package/lib/remoteobjectcache.js

@@ -0,0 +1,152 @@
+import assert from 'node:assert'
+
+import as2 from './activitystreams.js'
+
+const SEC_NS = 'https://w3id.org/security#'
+const NS = 'https://www.w3.org/ns/activitystreams#'
+const KEY_TYPES = [SEC_NS + 'Key', SEC_NS + 'CryptographicKey']
+const COLLECTION_TYPES =
+  [NS + 'Collection', NS + 'CollectionPage', NS + 'OrderedCollection', NS + 'OrderedCollectionPage']
+
+export class RemoteObjectCache {
+  #connection
+  #logger
+  constructor (connection, logger) {
+    assert.ok(connection)
+    assert.ok(logger)
+    this.#connection = connection
+    this.#logger = logger
+  }
+
+  async get (id, username) {
+    assert.ok(id)
+    assert.strictEqual(typeof id, 'string')
+    assert.ok(username)
+    assert.strictEqual(typeof username, 'string')
+
+    let result
+
+    const [rows] = await this.#connection.query(
+      `SELECT last_modified, etag, expiry, data
+      FROM remote_object_cache
+      WHERE id = ? AND username = ?`,
+      { replacements: [id, username] })
+
+    if (rows.length === 0) {
+      result = null
+    } else {
+      result = {
+        expiry: new Date(rows[0].expiry),
+        lastModified: rows[0].last_modified
+          ? new Date(rows[0].last_modified)
+          : null,
+        etag: rows[0].etag,
+        object: JSON.parse(rows[0].data)
+      }
+    }
+
+    assert.ok(
+      result === null ||
+      (
+        typeof result === 'object' &&
+       'expiry' in result &&
+       'lastModified' in result &&
+       'etag' in result &&
+       'object' in result
+      )
+    )
+
+    return result
+  }
+
+  async set (id, username, object, headers) {
+    assert.ok(id)
+    assert.strictEqual(typeof id, 'string')
+    assert.ok(username)
+    assert.strictEqual(typeof username, 'string')
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    const cacheControl = headers.get('cache-control')
+    if (cacheControl && cacheControl.includes('no-store')) {
+      return
+    }
+
+    const expiry = await this.#getExpiry(object, headers)
+    const lastModified = headers.get('last-modified')
+    const etag = headers.get('etag')
+
+    await this.#connection.query(
+      `INSERT INTO remote_object_cache
+      (id, username, last_modified, etag, expiry, data)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT (id, username)
+      DO UPDATE
+      SET last_modified = EXCLUDED.last_modified,
+      etag = EXCLUDED.etag,
+      expiry = EXCLUDED.expiry,
+      data = EXCLUDED.data,
+      updated_at = CURRENT_TIMESTAMP
+      `,
+      { replacements: [id, username, lastModified, etag, expiry, JSON.stringify(object)] }
+    )
+  }
+
+  async #getExpiry (object, headers) {
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    let expiry = this.#getExpiryFromHeaders(headers)
+    if (!expiry) {
+      expiry = await this.#getExpiryFromObject(object)
+    }
+    return expiry
+  }
+
+  #getExpiryFromHeaders (headers) {
+    assert.ok(headers)
+    assert.strictEqual(typeof headers, 'object')
+    assert.ok(typeof headers.get === 'function')
+
+    const cacheControl = headers.get('cache-control')
+    if (cacheControl) {
+      if (cacheControl.includes('no-cache')) {
+        return new Date(Date.now() - 1)
+      } else if (cacheControl.includes('max-age')) {
+        const match = cacheControl.match(/max-age=(\d+)/)
+        if (match) {
+          return new Date(Date.now() + parseInt(match[1], 10) * 1000)
+        }
+      }
+    }
+    if (headers.has('expires')) {
+      return new Date(headers.get('expires'))
+    }
+    return null
+  }
+
+  async #getExpiryFromObject (object) {
+    assert.ok(object)
+    assert.strictEqual(typeof object, 'object')
+    const as2obj = await as2.import(object)
+    let offset
+    if (as2obj.isActivity()) {
+      offset = 24 * 60 * 60 * 1000
+    } else if (as2obj.inbox) {
+      offset = 30 * 60 * 1000
+    } else if (KEY_TYPES.includes(as2obj.type)) {
+      offset = -1000
+    } else if (COLLECTION_TYPES.includes(as2obj.type)) {
+      offset = -1000
+    } else {
+      offset = 5 * 60 * 1000
+    }
+    return new Date(Date.now() + offset)
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evanp/activitypub-bot",
-  "version": "0.35.0",
+  "version": "0.36.1",
   "description": "server-side ActivityPub bot framework",
   "type": "module",
   "main": "lib/index.js",