@evanp/activitypub-bot 0.8.0

package/lib/objectcache.js

@@ -0,0 +1,48 @@
+import TTLCache from '@isaacs/ttlcache'
+
+export class ObjectCache {
+  #objects = null
+  #members = null
+  constructor ({ longTTL, shortTTL, maxItems }) {
+    this.#objects = new TTLCache({ ttl: shortTTL, max: maxItems })
+    this.#members = new TTLCache({ ttl: shortTTL, max: maxItems })
+    this.longTTL = longTTL
+    this.shortTTL = shortTTL
+    this.maxItems = maxItems
+  }
+
+  async initialize () {
+  }
+
+  async get (id) {
+    return this.#objects.get(id)
+  }
+
+  async save (object) {
+    return this.#objects.set(object.id, object, { ttl: this.longTTL })
+  }
+
+  async saveReceived (object) {
+    return this.#objects.set(object.id, object, { ttl: this.shortTTL })
+  }
+
+  async clear (object) {
+    return this.#objects.delete(object.id)
+  }
+
+  membershipKey (collection, object) {
+    return `${collection.id}:${object.id}`
+  }
+
+  async saveMembership (collection, object, isMember = true) {
+    return this.#members.set(this.membershipKey(collection, object), isMember, { ttl: this.longTTL })
+  }
+
+  async saveMembershipReceived (collection, object, isMember = true) {
+    return this.#members.set(this.membershipKey(collection, object), isMember, { ttl: this.shortTTL })
+  }
+
+  async isMember (collection, object) {
+    return this.#members.get(this.membershipKey(collection, object))
+  }
+}

package/lib/objectstorage.js

@@ -0,0 +1,319 @@
+import assert from 'node:assert'
+import as2 from './activitystreams.js'
+
+export class NoSuchObjectError extends Error {
+  constructor (id) {
+    const message = `No such object: ${id}`
+    super(message)
+    this.name = 'NoSuchObjectError'
+  }
+}
+
+export class ObjectStorage {
+  #connection = null
+  static #MAX_ITEMS_PER_PAGE = 20
+
+  constructor (connection) {
+    this.#connection = connection
+  }
+
+  async initialize () {
+    await this.#connection.query(`
+            CREATE TABLE IF NOT EXISTS objects (
+                id VARCHAR(512) PRIMARY KEY,
+                data TEXT NOT NULL,
+                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+            )
+        `)
+    await this.#connection.query(`
+            CREATE TABLE IF NOT EXISTS collections (
+                id VARCHAR(512) NOT NULL,
+                property VARCHAR(512) NOT NULL,
+                first INTEGER NOT NULL,
+                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (id, property)
+            )
+        `)
+    await this.#connection.query(`
+            CREATE TABLE IF NOT EXISTS pages (
+                id VARCHAR(512) NOT NULL,
+                property VARCHAR(64) NOT NULL,
+                item VARCHAR(512) NOT NULL,
+                page INTEGER NOT NULL,
+                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (id, property, item)
+            )
+        `)
+
+    await this.#connection.query(
+      `CREATE INDEX IF NOT EXISTS pages_username_property_page
+      ON pages (id, property, page);`
+    )
+  }
+
+  async create (object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(object, 'object is required')
+    assert.ok(object.id, 'object.id is required')
+    const id = object.id
+    const data = await object.write()
+    assert.ok(data, 'object is required')
+    assert.ok(typeof data === 'string', 'data must be a string')
+    await this.#connection.query(
+      'INSERT INTO objects (id, data) VALUES (?, ?)',
+      { replacements: [id, data] }
+    )
+  }
+
+  async read (id) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    const data = await this.#connection.query(
+      'SELECT data FROM objects WHERE id = ?',
+      { replacements: [id] }
+    )
+    if (data[0].length === 0) {
+      throw new NoSuchObjectError(id)
+    }
+    return await as2.import(JSON.parse(data[0][0].data))
+  }
+
+  async update (object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(object, 'object is required')
+    assert.ok(object.id, 'object.id is required')
+    const id = object.id
+    const data = await object.write()
+    assert.ok(data, 'object is required')
+    assert.ok(typeof data === 'string', 'data must be a string')
+    await this.#connection.query(
+      'UPDATE objects SET data = ?, updatedAt = CURRENT_TIMESTAMP WHERE id = ?',
+      { replacements: [data, id] }
+    )
+  }
+
+  async delete (object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(object, 'object is required')
+    assert.ok(object.id, 'object.id is required')
+    const id = object.id
+    await this.#connection.query('DELETE FROM objects WHERE id = ?', {
+      replacements: [id]
+    })
+  }
+
+  async #collectionInfo (id, property) {
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    let totalItems = 0
+    let first = 1
+    let createdAt = null
+    const row = await this.#connection.query(
+      'SELECT first, createdAt FROM collections WHERE id = ? AND property = ?',
+      { replacements: [id, property] }
+    )
+    if (row[0].length > 0) {
+      first = row[0][0].first
+      createdAt = row[0][0].createdAt
+    }
+    const count = await this.#connection.query(
+      'SELECT COUNT(*) FROM pages WHERE id = ? AND property = ?',
+      { replacements: [id, property] }
+    )
+    if (count[0].length > 0) {
+      totalItems = count[0][0]['COUNT(*)']
+    }
+    assert.equal(typeof totalItems, 'number', 'totalItems must be a number')
+    assert.ok(totalItems >= 0, 'totalItems must be greater than or equal to 0')
+    assert.equal(typeof first, 'number', 'first must be a number')
+    assert.ok(first >= 1, 'first must be greater than or equal to 1')
+    return [totalItems, first, createdAt]
+  }
+
+  async getCollection (id, property) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.ok(property, 'property is required')
+    const [totalItems, first, createdAt] = await this.#collectionInfo(
+      id,
+      property
+    )
+    const context = [
+      'https://www.w3.org/ns/activitystreams'
+    ]
+    if (property === 'thread') {
+      context.push('https://purl.archive.org/socialweb/thread/1.0')
+    } else {
+      context.push('https://w3id.org/fep/5711')
+    }
+    const inverse = (property === 'thread') ? 'root' : `${property}Of`
+    const collection = await as2.import({
+      '@context': context,
+      id: `${id}/${property}`,
+      type: 'OrderedCollection',
+      totalItems,
+      first: `${id}/${property}/${first}`,
+      last: `${id}/${property}/1`,
+      published: createdAt,
+      [inverse]: id
+    })
+    assert.ok(collection, 'collection is required')
+    assert.equal(typeof collection, 'object', 'collection must be an object')
+    return collection
+  }
+
+  async getCollectionPage (id, property, page) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(page, 'pageNo is required')
+    assert.equal(typeof page, 'number', 'page must be a number')
+    assert.ok(page >= 1, 'page must be greater than or equal to 1')
+
+    const [, first] = await this.#collectionInfo(
+      id,
+      property
+    )
+
+    if (page > first) {
+      throw new NoSuchObjectError(`${id}/${property}/${page}`)
+    }
+
+    let items = []
+
+    const rows = await this.#connection.query(
+      'SELECT item FROM pages WHERE id = ? AND property = ? and page = ? ORDER BY createdAt DESC, item DESC',
+      { replacements: [id, property, page] }
+    )
+
+    if (rows[0].length > 0) {
+      items = rows[0].map((row) => row.item)
+    }
+
+    const pageObject = await as2.import({
+      id: `${id}/${property}/${page}`,
+      type: 'OrderedCollectionPage',
+      partOf: `${id}/${property}`,
+      next: page === 1 ? null : `${id}/${property}/${page - 1}`,
+      prev: page === first ? null : `${id}/${property}/${page + 1}`,
+      items
+    })
+
+    assert.ok(pageObject, 'collection is required')
+    assert.equal(typeof pageObject, 'object', 'collection must be an object')
+
+    return pageObject
+  }
+
+  async addToCollection (id, property, object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+
+    const [, first, createdAt] = await this.#collectionInfo(
+      id,
+      property
+    )
+
+    if (createdAt === null) {
+      await this.#connection.query(
+        'INSERT INTO collections (id, property, first) VALUES (?, ?, 1)',
+        { replacements: [id, property] }
+      )
+    }
+
+    const count = await this.#itemCount(id, property, first)
+
+    const page = count >= ObjectStorage.#MAX_ITEMS_PER_PAGE ? first + 1 : first
+
+    if (page > first) {
+      await this.#connection.query(
+        'UPDATE collections SET first = ?, updatedAt = CURRENT_TIMESTAMP WHERE id = ? AND property = ?',
+        { replacements: [page, id, property] }
+      )
+    }
+
+    await this.#connection.query(
+      'INSERT INTO pages (id, property, item, page) VALUES (?, ?, ?, ?)',
+      { replacements: [id, property, object.id, page] }
+    )
+  }
+
+  async #itemCount (id, property, page) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(page, 'page is required')
+    assert.equal(typeof page, 'number', 'page must be a number')
+    assert.ok(page >= 1, 'page must be greater than or equal to 1')
+    const rows = await this.#connection.query(
+      'SELECT COUNT(*) FROM pages WHERE id = ? AND property = ? AND page = ?',
+      { replacements: [id, property, page] }
+    )
+    return rows[0][0]['COUNT(*)']
+  }
+
+  async removeFromCollection (id, property, object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+
+    await this.#connection.query(
+      'DELETE FROM pages WHERE id = ? AND property = ? AND item = ?',
+      { replacements: [id, property, object.id] }
+    )
+  }
+
+  async isInCollection (id, property, object) {
+    assert.ok(this.#connection, 'ObjectStorage not initialized')
+    assert.ok(id, 'id is required')
+    assert.equal(typeof id, 'string', 'id must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+    assert.ok(object.id, 'object.id is required')
+    assert.equal(typeof object.id, 'string', 'object.id must be a string')
+    const [result] = await this.#connection.query(
+      `SELECT COUNT(*) as item_count
+      FROM pages
+      WHERE id = ? AND property = ? AND item = ?`,
+      { replacements: [id, property, object.id] }
+    )
+    return result[0].item_count > 0
+  }
+
+  async * items (id, property) {
+    assert.ok(id, 'username is required')
+    assert.equal(typeof id, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+
+    const result = await this.#connection.query(
+      `SELECT item
+      FROM pages
+      WHERE id = ? AND property = ?
+      ORDER BY page DESC, createdAt DESC, item DESC;`,
+      { replacements: [id, property] }
+    )
+    for (const row of result[0]) {
+      yield as2.import({ id: row.item })
+    }
+  }
+}

package/lib/remotekeystorage.js

@@ -0,0 +1,116 @@
+const SEC_NS = 'https://w3id.org/security#'
+
+export class RemoteKeyStorage {
+  #client = null
+  #connection = null
+  #logger = null
+  constructor (client, connection, logger) {
+    this.#client = client
+    this.#connection = connection
+    this.#logger = logger.child({ class: this.constructor.name })
+  }
+
+  async initialize () {
+    await this.#connection.query(
+      `CREATE TABLE IF NOT EXISTS new_remotekeys (
+        id VARCHAR(512) PRIMARY KEY,
+        owner VARCHAR(512) NOT NULL,
+        publicKeyPem TEXT NOT NULL,
+        createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+      )`
+    )
+    try {
+      await this.#connection.query(
+        `INSERT OR IGNORE INTO new_remotekeys (id, owner, publicKeyPem)
+         SELECT id, owner, publicKeyPem
+         FROM remotekeys`
+      )
+    } catch (error) {
+      this.#logger.debug(
+        { error, method: 'RemoteKeyStorage.initialize' },
+        'failed to copy remotekeys to new_remotekeys table'
+      )
+    }
+  }
+
+  async getPublicKey (id, useCache = true) {
+    this.debug(`getPublicKey(${id})`)
+    if (useCache) {
+      this.debug('using cache')
+      const cached = await this.#getCachedPublicKey(id)
+      if (cached) {
+        this.debug(`getPublicKey(${id}) - cached`)
+        return cached
+      }
+    } else {
+      this.debug('skipping cache')
+    }
+    const remote = await this.#getRemotePublicKey(id)
+    if (!remote) {
+      this.debug(`getPublicKey(${id}) - remote not found`)
+      return null
+    }
+    await this.#cachePublicKey(id, remote.owner, remote.publicKeyPem)
+    return remote
+  }
+
+  async #getCachedPublicKey (id) {
+    this.debug(`#getCachedPublicKey(${id})`)
+    const [result] = await this.#connection.query(
+      'SELECT publicKeyPem, owner FROM new_remotekeys WHERE id = ?',
+      { replacements: [id] }
+    )
+    if (result.length > 0) {
+      this.debug(`cache hit for ${id}`)
+      return {
+        publicKeyPem: result[0].publicKeyPem,
+        owner: result[0].owner
+      }
+    } else {
+      this.debug(`cache miss for ${id}`)
+      return null
+    }
+  }
+
+  async #getRemotePublicKey (id) {
+    this.debug(`#getRemotePublicKey(${id})`)
+    const response = await this.#client.getKey(id)
+    if (!response) {
+      return null
+    }
+    this.debug(`getRemotePublicKey(${id}) - response: ${await response.id}`)
+    let owner = null
+    let publicKeyPem = null
+    if (response.get(SEC_NS + 'publicKeyPem')) {
+      this.debug(`getRemotePublicKey(${id}) - publicKeyPem`)
+      owner = response.get(SEC_NS + 'owner')?.first?.id
+      publicKeyPem = response.get(SEC_NS + 'publicKeyPem')?.first
+    } else if (response.get(SEC_NS + 'publicKey')) {
+      this.debug(`getRemotePublicKey(${id}) - publicKey`)
+      const publicKey = response.get(SEC_NS + 'publicKey').first
+      if (publicKey) {
+        owner = publicKey.get(SEC_NS + 'owner')?.first?.id
+        publicKeyPem = publicKey.get(SEC_NS + 'publicKeyPem')?.first
+      }
+    }
+    if (!owner || !publicKeyPem) {
+      return null
+    }
+    return { owner, publicKeyPem }
+  }
+
+  async #cachePublicKey (id, owner, publicKeyPem) {
+    await this.#connection.query(
+      'INSERT INTO new_remotekeys (id, owner, publicKeyPem) VALUES (?, ?, ?) ' + ' ON CONFLICT(id) DO UPDATE ' +
+      'SET owner=excluded.owner, publicKeyPem = excluded.publicKeyPem;',
+      { replacements: [id, owner, publicKeyPem] }
+    )
+  }
+
+  debug (...args) {
+    if (this.#logger) {
+      this.#logger.debug(...args)
+    }
+  }
+}

package/lib/routes/collection.js

@@ -0,0 +1,92 @@
+import express from 'express'
+import as2 from '../activitystreams.js'
+import createHttpError from 'http-errors'
+
+const router = express.Router()
+
+async function filterAsync (array, asyncPredicate) {
+  // 1. Kick off all predicate calls in parallel:
+  const checks = array.map(item => asyncPredicate(item))
+
+  // 2. Wait for all to settle into [true, false, …]:
+  const booleans = await Promise.all(checks)
+
+  // 3. Pick only those whose boolean was true:
+  return array.filter((_, idx) => booleans[idx])
+}
+
+router.get('/user/:username/:collection', async (req, res, next) => {
+  const { username, collection } = req.params
+  const { actorStorage, bots } = req.app.locals
+  if (!(username in bots)) {
+    return next(createHttpError(404, `User ${username} not found`))
+  }
+  if (collection === 'inbox') {
+    return next(createHttpError(403, `No access to ${collection} collection`))
+  }
+  if (!['outbox', 'liked', 'followers', 'following'].includes(req.params.collection)) {
+    return next(createHttpError(404,
+      `No such collection ${collection} for user ${username}`))
+  }
+  const coll = await actorStorage.getCollection(username, collection)
+  res.status(200)
+  res.type(as2.mediaType)
+  res.end(await coll.prettyWrite({ useOriginalContext: true }))
+})
+
+router.get('/user/:username/:collection/:n(\\d+)', async (req, res, next) => {
+  const { username, collection, n } = req.params
+  const { actorStorage, bots, authorizer, objectStorage, formatter, client } = req.app.locals
+
+  if (!(username in bots)) {
+    return next(createHttpError(404, `User ${username} not found`))
+  }
+  if (collection === 'inbox') {
+    return next(createHttpError(403, `No access to ${collection} collection`))
+  }
+  if (!['outbox', 'liked', 'followers', 'following'].includes(collection)) {
+    return next(createHttpError(404,
+      `No such collection ${collection} for user ${username}`))
+  }
+  if (!await actorStorage.hasPage(username, collection, parseInt(n))) {
+    return next(createHttpError(404, `No such page ${n} for collection ${collection} for user ${username}`))
+  }
+
+  let exported = null
+
+  try {
+    const id = req.auth?.subject
+    const remote = (id) ? await as2.import({ id }) : null
+    const page = await actorStorage.getCollectionPage(
+      username,
+      collection,
+      parseInt(n)
+    )
+    exported = await page.export({ useOriginalContext: true })
+
+    if (['outbox', 'liked'].includes(collection)) {
+      exported.items = await filterAsync(exported.items, async (id) => {
+        const object = (formatter.isLocal(id))
+          ? await objectStorage.read(id)
+          : await client.get(id)
+        if (!object) {
+          req.log.warn({ id }, 'could not load object')
+          return false
+        }
+        req.log.debug({ id, object }, 'loaded object')
+        return await authorizer.canRead(remote, object)
+      })
+    }
+  } catch (error) {
+    req.log.error(
+      { err: error, username, collection, n },
+      'error loading collection page'
+    )
+    return next(createHttpError(500, 'Error loading collection page'))
+  }
+  res.status(200)
+  res.type(as2.mediaType)
+  res.end(JSON.stringify(exported))
+})
+
+export default router

package/lib/routes/health.js

@@ -0,0 +1,24 @@
+import express from 'express'
+import createHttpError from 'http-errors'
+
+const router = express.Router()
+
+router.get('/livez', async (req, res) => {
+  res.status(200)
+  res.type('text/plain')
+  res.end('OK')
+})
+
+router.get('/readyz', async (req, res, next) => {
+  const connection = req.app.locals.connection
+  try {
+    await connection.query('SELECT 1')
+    res.status(200)
+    res.type('text/plain')
+    res.end('OK')
+  } catch (err) {
+    return next(createHttpError(503, 'Service Unavailable'))
+  }
+})
+
+export default router

package/lib/routes/inbox.js

@@ -0,0 +1,83 @@
+import express from 'express'
+import as2 from '../activitystreams.js'
+import createHttpError from 'http-errors'
+
+const router = express.Router()
+
+function isActivity (object) {
+  return true
+}
+
+function getActor (activity) {
+  return activity.actor?.first
+}
+
+router.post('/user/:username/inbox', async (req, res, next) => {
+  const { username } = req.params
+  const { bots, actorStorage, activityHandler } = req.app.locals
+  const { subject } = req.auth
+  const { logger } = req.app.locals
+
+  const bot = bots[username]
+  if (!bot) {
+    return next(createHttpError(404, 'Not Found'))
+  }
+
+  if (!subject) {
+    return next(createHttpError(401, 'Unauthorized'))
+  }
+
+  if (!req.body) {
+    return next(createHttpError(400, 'No request body provided'))
+  }
+
+  let activity
+
+  try {
+    activity = await as2.import(req.body)
+  } catch (err) {
+    logger.warn('Failed to import activity', err)
+    logger.debug('Request body', req.body)
+    return next(createHttpError(400, 'Invalid request body'))
+  }
+
+  if (!isActivity(activity)) {
+    return next(createHttpError(400, 'Request body is not an activity'))
+  }
+
+  const actor = getActor(activity)
+
+  if (!actor) {
+    return next(createHttpError(400, 'No actor found in activity'))
+  }
+
+  if (!actor.id) {
+    return next(createHttpError(400, 'No actor id found in activity'))
+  }
+
+  if (actor.id !== subject) {
+    return next(createHttpError(403, `${subject} is not the actor ${actor.id}`))
+  }
+
+  if (await actorStorage.isInCollection(username, 'blocked', actor)) {
+    return next(createHttpError(403, 'Forbidden'))
+  }
+
+  if (await actorStorage.isInCollection(bot.username, 'inbox', activity)) {
+    return next(createHttpError(400, 'Activity already delivered'))
+  }
+
+  try {
+    await activityHandler.handleActivity(bot, activity)
+  } catch (err) {
+    return next(err)
+  }
+
+  await actorStorage.addToCollection(bot.username, 'inbox', activity)
+
+  res.status(200)
+  res.type('text/plain')
+  res.send('OK')
+})
+
+export default router