@evanp/activitypub-bot 0.8.0

package/lib/activitypubclient.js

@@ -0,0 +1,126 @@
+import as2 from './activitystreams.js'
+import fetch from 'node-fetch'
+import assert from 'node:assert'
+import createHttpError from 'http-errors'
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const { version } = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+export class ActivityPubClient {
+  static #githubUrl = 'https://github.com/evanp/activitypub-bot'
+  static #userAgent = `activitypub.bot/${version} (${ActivityPubClient.#githubUrl})`
+  static #accept =
+    ['application/activity+json',
+      'application/ld+json',
+      'application/json'].join(',')
+
+  #keyStorage = null
+  #urlFormatter = null
+  #signer = null
+  #digester = null
+  #logger = null
+
+  constructor (keyStorage, urlFormatter, signer, digester, logger) {
+    this.#keyStorage = keyStorage
+    this.#urlFormatter = urlFormatter
+    this.#signer = signer
+    this.#digester = digester
+    this.#logger = logger.child({ class: this.constructor.name })
+  }
+
+  async get (url, username = null) {
+    assert.ok(url)
+    assert.equal(typeof url, 'string')
+    const res = await this.#getRes(url, username, true)
+    return await this.#handleRes(res, url)
+  }
+
+  async getKey (url) {
+    assert.ok(url)
+    assert.equal(typeof url, 'string')
+    let res = await this.#getRes(url, null, false)
+    if ([401, 403, 404].includes(res.status)) {
+      // If we get a 401, 403, or 404, we should try again with the key
+      res = await this.#getRes(url, null, true)
+    }
+    return await this.#handleRes(res, url)
+  }
+
+  async #getRes (url, username = null, sign = false) {
+    assert.ok(url)
+    assert.equal(typeof url, 'string')
+    const date = new Date().toUTCString()
+    const headers = {
+      accept: ActivityPubClient.#accept,
+      date,
+      'user-agent': ActivityPubClient.#userAgent
+    }
+    const method = 'GET'
+    if (sign) {
+      headers.signature =
+        await this.#sign({ username, url, method, headers })
+    }
+    return await fetch(url,
+      {
+        method,
+        headers
+      }
+    )
+  }
+
+  async #handleRes (res, url) {
+    if (res.status < 200 || res.status > 299) {
+      throw createHttpError(res.status, `Could not fetch ${url}`)
+    }
+    const json = await res.json()
+    const obj = await as2.import(json)
+    return obj
+  }
+
+  async post (url, obj, username) {
+    assert.ok(url)
+    assert.equal(typeof url, 'string')
+    assert.ok(obj)
+    assert.equal(typeof obj, 'object')
+    assert.ok(username)
+    assert.equal(typeof username, 'string')
+    const body = await obj.write()
+    const headers = {
+      date: new Date().toUTCString(),
+      'user-agent': ActivityPubClient.#userAgent,
+      'content-type': 'application/activity+json',
+      digest: await this.#digester.digest(body)
+    }
+    const method = 'POST'
+    assert.ok(headers)
+    headers.signature = await this.#sign({ username, url, method, headers })
+    const res = await fetch(url,
+      {
+        method,
+        headers,
+        body
+      }
+    )
+    if (res.status < 200 || res.status > 299) {
+      throw createHttpError(res.status, await res.text())
+    }
+  }
+
+  async #sign ({ username, url, method, headers }) {
+    assert.ok(url)
+    assert.ok(method)
+    assert.ok(headers)
+    const privateKey = await this.#keyStorage.getPrivateKey(username)
+    const keyId = (username)
+      ? this.#urlFormatter.format({ username, type: 'publickey' })
+      : this.#urlFormatter.format({ server: true, type: 'publickey' })
+    return this.#signer.sign({ privateKey, keyId, url, method, headers })
+  }
+}

package/lib/activitystreams.js

@@ -0,0 +1,41 @@
+import as2 from 'activitystrea.ms'
+
+as2.registerContext('https://w3id.org/fep/5711', {
+  '@context': {
+    inv: 'https://w3id.org/fep/5711#',
+    likesOf: {
+      '@id': 'inv:likesOf',
+      '@type': '@id'
+    },
+    sharesOf: {
+      '@id': 'inv:sharesOf',
+      '@type': '@id'
+    },
+    repliesOf: {
+      '@id': 'inv:repliesOf',
+      '@type': '@id'
+    },
+    inboxOf: {
+      '@id': 'inv:inboxOf',
+      '@type': '@id'
+    },
+    outboxOf: {
+      '@id': 'inv:outboxOf',
+      '@type': '@id'
+    },
+    followersOf: {
+      '@id': 'inv:followersOf',
+      '@type': '@id'
+    },
+    followingOf: {
+      '@id': 'inv:followingOf',
+      '@type': '@id'
+    },
+    likedOf: {
+      '@id': 'inv:likedOf',
+      '@type': '@id'
+    }
+  }
+})
+
+export default as2

package/lib/actorstorage.js

@@ -0,0 +1,300 @@
+import as2 from './activitystreams.js'
+import assert from 'node:assert'
+
+export class ActorStorage {
+  #connection = null
+  #formatter = null
+  static #MAX_ITEMS_PER_PAGE = 20
+  constructor (connection, formatter) {
+    this.#connection = connection
+    this.#formatter = formatter
+  }
+
+  async initialize () {
+    await this.#connection.query(`
+      CREATE TABLE IF NOT EXISTS actorcollection (
+        username varchar(512) NOT NULL,
+        property varchar(512) NOT NULL,
+        first INTEGER NOT NULL,
+        totalItems INTEGER NOT NULL DEFAULT 0,
+        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (username, property)
+      );`
+    )
+    await this.#connection.query(`
+      CREATE TABLE IF NOT EXISTS actorcollectionpage (
+        username varchar(512) NOT NULL,
+        property varchar(512) NOT NULL,
+        item varchar(512) NOT NULL,
+        page INTEGER NOT NULL,
+        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (username, property, item)
+      );`
+    )
+    await this.#connection.query(
+      `CREATE INDEX IF NOT EXISTS actorcollectionpage_username_property_page
+      ON actorcollectionpage (username, property, page);`
+    )
+  }
+
+  async getActor (username, props = {}) {
+    assert.ok(username)
+    assert.equal(typeof username, 'string')
+    const type = ('type' in props)
+      ? (Array.isArray(props.type))
+          ? [...props.type, 'Service']
+          : [props.type, 'Service']
+      : ['Service']
+    // XXX: spread props first so type is not overwritten
+    return await as2.import({
+      ...props,
+      id: this.#formatter.format({ username }),
+      type,
+      preferredUsername: username,
+      inbox: this.#formatter.format({ username, collection: 'inbox' }),
+      outbox: this.#formatter.format({ username, collection: 'outbox' }),
+      followers: this.#formatter.format({ username, collection: 'followers' }),
+      following: this.#formatter.format({ username, collection: 'following' }),
+      liked: this.#formatter.format({ username, collection: 'liked' }),
+      to: 'as:Public'
+    })
+  }
+
+  async getActorById (id) {
+    assert.ok(id)
+    assert.equal(typeof id, 'string')
+    const username = this.#formatter.getUserName(id)
+    return await this.getActor(username)
+  }
+
+  async getCollection (username, property) {
+    assert.ok(username)
+    assert.equal(typeof username, 'string')
+    assert.ok(property)
+    assert.equal(typeof property, 'string')
+    const [totalItems, first, createdAt, updatedAt] =
+      await this.#getCollectionInfo(
+        username,
+        property
+      )
+    return await as2.import({
+      '@context': [
+        'https://www.w3.org/ns/activitystreams',
+        'https://w3id.org/fep/5711'
+      ],
+      id: this.#formatter.format({ username, collection: property }),
+      type: 'OrderedCollection',
+      attributedTo: this.#formatter.format({ username }),
+      [`${property}Of`]: this.#formatter.format({ username }),
+      to: 'as:Public',
+      summaryMap: {
+        en: 'The ' + property + ' collection for user ' + username
+      },
+      totalItems,
+      first: this.#formatter.format({
+        username,
+        collection: property,
+        page: first
+      }),
+      last: this.#formatter.format({ username, collection: property, page: 1 }),
+      published: createdAt,
+      updated: updatedAt
+    })
+  }
+
+  async getCollectionPage (username, property, page) {
+    assert.ok(username)
+    assert.equal(typeof username, 'string')
+    assert.ok(property)
+    assert.equal(typeof property, 'string')
+    assert.ok(page)
+    assert.equal(typeof page, 'number')
+    assert.ok(page > 0)
+    const [, first] = await this.#getCollectionInfo(username, property)
+    if (page > first) {
+      throw new Error('page out of range')
+    }
+    const result = await this.#connection.query(
+      `SELECT item
+      FROM actorcollectionpage
+      WHERE username = ? AND property = ? AND page = ?
+      ORDER BY createdAt DESC, item DESC
+      ;`,
+      { replacements: [username, property, page] })
+    const items = []
+    for (const row of result[0]) {
+      items.push(row.item)
+    }
+    return await as2.import({
+      id: this.#formatter.format({ username, collection: property, page }),
+      type: 'OrderedCollectionPage',
+      partOf: this.#formatter.format({ username, collection: property }),
+      attributedTo: this.#formatter.format({ username }),
+      to: 'as:Public',
+      summaryMap: {
+        en: `Page ${page} of the ${property} collection for user ${username}`
+      },
+      next: (page === 1)
+        ? null
+        : this.#formatter.format({ username, collection: property, page: page - 1 }),
+      prev: (page === first)
+        ? null
+        : this.#formatter.format({ username, collection: property, page: page + 1 }),
+      items
+    })
+  }
+
+  async addToCollection (username, property, object) {
+    assert.ok(this.#connection, 'ActorStorage not initialized')
+    assert.ok(username, 'username is required')
+    assert.equal(typeof username, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+
+    const [, first, createdAt] = await this.#getCollectionInfo(
+      username,
+      property
+    )
+
+    if (createdAt === null) {
+      await this.#connection.query(
+        `INSERT INTO actorcollection (username, property, first, totalItems)
+         VALUES (?, ?, 1, 0)`,
+        { replacements: [username, property] }
+      )
+    }
+
+    const count = await this.#itemCount(username, property, first)
+
+    const page = (count >= ActorStorage.#MAX_ITEMS_PER_PAGE)
+      ? first + 1
+      : first
+
+    if (page > first) {
+      await this.#connection.query(
+        `UPDATE actorcollection
+        SET first = ?, updatedAt = CURRENT_TIMESTAMP
+        WHERE username = ? AND property = ?`,
+        { replacements: [page, username, property] }
+      )
+    }
+
+    await this.#connection.query(
+      `INSERT INTO actorcollectionpage (username, property, item, page)
+      VALUES (?, ?, ?, ?)`,
+      { replacements: [username, property, object.id, page] }
+    )
+
+    await this.#connection.query(
+      `UPDATE actorcollection
+      SET totalItems = totalItems + 1, updatedAt = CURRENT_TIMESTAMP
+      WHERE username = ? AND property = ?`,
+      { replacements: [username, property] }
+    )
+  }
+
+  async removeFromCollection (username, property, object) {
+    assert.ok(username, 'username is required')
+    assert.equal(typeof username, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+
+    if (!await this.isInCollection(username, property, object)) {
+      return 0
+    }
+
+    await this.#connection.query(
+      `DELETE FROM actorcollectionpage
+      WHERE username = ? AND property = ? AND item = ?`,
+      { replacements: [username, property, object.id] }
+    )
+
+    await this.#connection.query(
+      `UPDATE actorcollection
+      SET totalItems = totalItems - 1, updatedAt = CURRENT_TIMESTAMP
+      WHERE username = ? AND property = ?`,
+      { replacements: [username, property] }
+    )
+
+    return 1
+  }
+
+  async * items (username, property) {
+    assert.ok(username, 'username is required')
+    assert.equal(typeof username, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+
+    const result = await this.#connection.query(
+      `SELECT item
+      FROM actorcollectionpage
+      WHERE username = ? AND property = ?
+      ORDER BY page DESC, createdAt DESC, item DESC;`,
+      { replacements: [username, property] }
+    )
+    for (const row of result[0]) {
+      yield as2.import({ id: row.item })
+    }
+  }
+
+  async isInCollection (username, property, object) {
+    assert.ok(username, 'username is required')
+    assert.equal(typeof username, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+    const [result] = await this.#connection.query(
+      `SELECT COUNT(*) as item_count
+      FROM actorcollectionpage
+      WHERE username = ? AND property = ? AND item = ?`,
+      { replacements: [username, property, object.id] }
+    )
+    return result[0].item_count > 0
+  }
+
+  async hasPage (username, property, page) {
+    const [, first] = await this.#getCollectionInfo(
+      username,
+      property
+    )
+    return page <= first
+  }
+
+  async #getCollectionInfo (username, property) {
+    const [result] = await this.#connection.query(
+      `SELECT first, totalItems, createdAt, updatedAt
+      FROM actorcollection
+      WHERE username = ? AND property = ?;`,
+      { replacements: [username, property] }
+    )
+    if (result.length > 0) {
+      const row = result[0]
+      return [row.totalItems, row.first, row.createdAt, row.updatedAt]
+    } else {
+      return [0, 1, null, null]
+    }
+  }
+
+  async #itemCount (username, property, page) {
+    assert.ok(this.#connection, 'ActorStorage not initialized')
+    assert.ok(username, 'username is required')
+    assert.equal(typeof username, 'string', 'username must be a string')
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(page, 'page is required')
+    assert.equal(typeof page, 'number', 'page must be a number')
+    assert.ok(page >= 1, 'page must be greater than or equal to 1')
+    const rows = await this.#connection.query(
+      `SELECT COUNT(*) as item_count FROM actorcollectionpage
+      WHERE username = ? AND property = ? AND page = ?`,
+      { replacements: [username, property, page] }
+    )
+    return rows[0][0].item_count
+  }
+}

package/lib/app.js

@@ -0,0 +1,173 @@
+import { Sequelize } from 'sequelize'
+import express from 'express'
+import Logger from 'pino'
+import HTTPLogger from 'pino-http'
+import http from 'node:http'
+import { ActivityDistributor } from './activitydistributor.js'
+import { ActivityPubClient } from './activitypubclient.js'
+import { ActorStorage } from './actorstorage.js'
+import { BotDataStorage } from './botdatastorage.js'
+import { KeyStorage } from './keystorage.js'
+import { ObjectStorage } from './objectstorage.js'
+import { UrlFormatter } from './urlformatter.js'
+import { HTTPSignature } from './httpsignature.js'
+import { Authorizer } from './authorizer.js'
+import { RemoteKeyStorage } from './remotekeystorage.js'
+import { ActivityHandler } from './activityhandler.js'
+import { ObjectCache } from '../lib/objectcache.js'
+import serverRouter from './routes/server.js'
+import userRouter from './routes/user.js'
+import objectRouter from './routes/object.js'
+import collectionRouter from './routes/collection.js'
+import inboxRouter from './routes/inbox.js'
+import healthRouter from './routes/health.js'
+import webfingerRouter from './routes/webfinger.js'
+import { BotContext } from './botcontext.js'
+import { Transformer } from './microsyntax.js'
+import { HTTPSignatureAuthenticator } from './httpsignatureauthenticator.js'
+import { Digester } from './digester.js'
+
+export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
+  const logger = Logger({
+    level: logLevel
+  })
+  logger.debug('Logger initialized')
+  const connection = new Sequelize(databaseUrl, { logging: false })
+  const formatter = new UrlFormatter(origin)
+  const signer = new HTTPSignature(logger)
+  const digester = new Digester(logger)
+  const actorStorage = new ActorStorage(connection, formatter)
+  await actorStorage.initialize()
+  const botDataStorage = new BotDataStorage(connection)
+  await botDataStorage.initialize()
+  const keyStorage = new KeyStorage(connection, logger)
+  await keyStorage.initialize()
+  const objectStorage = new ObjectStorage(connection)
+  await objectStorage.initialize()
+  const client =
+    new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
+  const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
+  await remoteKeyStorage.initialize()
+  const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
+  const distributor = new ActivityDistributor(
+    client,
+    formatter,
+    actorStorage,
+    logger
+  )
+  const authorizer = new Authorizer(actorStorage, formatter, client)
+  const cache = new ObjectCache({
+    longTTL: 3600 * 1000,
+    shortTTL: 300 * 1000,
+    maxItems: 1000
+  })
+  const activityHandler = new ActivityHandler(
+    actorStorage,
+    objectStorage,
+    distributor,
+    formatter,
+    cache,
+    authorizer,
+    logger,
+    client
+  )
+  // TODO: Make an endpoint for tagged objects
+  const transformer = new Transformer(origin + '/tag/', client)
+  await Promise.all(
+    Object.values(bots).map(bot => bot.initialize(
+      new BotContext(
+        bot.username,
+        botDataStorage,
+        objectStorage,
+        actorStorage,
+        client,
+        distributor,
+        formatter,
+        transformer,
+        logger
+      )
+    ))
+  )
+
+  const app = express()
+
+  app.locals = {
+    connection,
+    formatter,
+    actorStorage,
+    botDataStorage,
+    keyStorage,
+    objectStorage,
+    remoteKeyStorage,
+    client,
+    distributor,
+    signature,
+    logger,
+    authorizer,
+    bots,
+    activityHandler,
+    origin
+  }
+
+  app.use(HTTPLogger({
+    logger,
+    level: logLevel
+  }))
+
+  app.use(express.json({
+    type: [
+      'application/activity+json',
+      'application/ld+json',
+      'application/json'
+    ],
+    verify: (req, res, buf, encoding) => {
+      req.rawBodyText = buf.toString(encoding || 'utf8')
+    }
+  }))
+
+  app.use(signature.authenticate.bind(signature))
+
+  app.use('/', serverRouter)
+  app.use('/', userRouter)
+  app.use('/', collectionRouter)
+  app.use('/', objectRouter)
+  app.use('/', inboxRouter)
+  app.use('/', healthRouter)
+  app.use('/', webfingerRouter)
+
+  app.use((err, req, res, next) => {
+    const { logger } = req.app.locals
+    let status = 500
+    if (err.status) {
+      status = err.status
+    }
+    const title = (http.STATUS_CODES[status])
+      ? http.STATUS_CODES[status]
+      : 'Unknown Status'
+
+    if (status >= 500 && status < 600) {
+      logger.error(err)
+    } else if (status >= 400 && status < 500) {
+      logger.warn(err)
+    } else {
+      logger.debug(err)
+    }
+    res.status(status)
+    res.type('application/problem+json')
+    res.json({ type: 'about:blank', title, status, detail: err.message })
+  })
+
+  app.onIdle = async () => {
+    await distributor.onIdle()
+  }
+
+  app.cleanup = async () => {
+    logger.info('Closing app')
+    logger.info('Waiting for distributor queue')
+    await distributor.onIdle()
+    logger.info('Closing database connection')
+    await connection.close()
+  }
+
+  return app
+}