@evanp/activitypub-bot 0.8.0 → 0.11.0

package/lib/activitydeliverer.js

@@ -0,0 +1,260 @@
+import BotMaker from './botmaker.js'
+import assert from 'node:assert'
+import as2 from './activitystreams.js'
+
+const NS = 'https://www.w3.org/ns/activitystreams#'
+
+const PUBLIC = [
+  `${NS}Public`,
+  'as:Public',
+  'Public'
+]
+
+const COLLECTION_TYPES = [
+  `${NS}Collection`,
+  `${NS}OrderedCollection`
+]
+
+export class ActivityDeliverer {
+  #actorStorage
+  #activityHandler
+  #formatter
+  #logger
+  #client
+
+  constructor (actorStorage, activityHandler, formatter, logger, client) {
+    this.#actorStorage = actorStorage
+    this.#activityHandler = activityHandler
+    this.#formatter = formatter
+    this.#client = client
+    this.#logger = logger.child({ class: this.constructor.name })
+  }
+
+  isActivity (object) {
+    return true
+  }
+
+  getActor (activity) {
+    return activity.actor?.first
+  }
+
+  getRecipients (obj) {
+    let r = []
+    for (const prop of ['to', 'cc', 'audience']) {
+      const val = obj.get(prop)
+      if (val) {
+        r = r.concat(Array.from(val))
+      }
+    }
+    return r
+  }
+
+  async deliverTo (activity, bot) {
+    try {
+      await this.#activityHandler.handleActivity(bot, activity)
+    } catch (err) {
+      this.#logger.warn(err)
+    }
+    this.#logger.debug(`Adding ${activity.id} to ${bot.username} inbox`)
+    await this.#actorStorage.addToCollection(bot.username, 'inbox', activity)
+  }
+
+  async deliverToAll (activity, bots) {
+    const deliveredTo = new Set()
+    const actor = this.getActor(activity)
+    const recipients = this.getRecipients(activity)
+
+    for (const recipient of recipients) {
+      if (this.#isPublic(recipient)) {
+        await this.#deliverPublic(activity, bots)
+      } else if (this.#isLocal(recipient)) {
+        const parts = this.#formatter.unformat(recipient.id)
+        if (this.#isLocalActor(parts)) {
+          await this.#deliverLocalActor(activity, recipient, bots, deliveredTo)
+        } else if (this.#isLocalFollowersCollection(parts)) {
+          await this.#deliverLocalFollowersCollection(activity, parts.username, bots, deliveredTo)
+        } else if (this.#isLocalFollowingCollection(parts)) {
+          await this.#deliverLocalFollowingCollection(activity, parts.username, bots, deliveredTo)
+        } else {
+          this.#logger.warn(
+            `Unrecognized recipient for remote delivery: ${recipient.id}`
+          )
+        }
+      } else {
+        const fullActor = await this.#client.get(actor.id)
+        const fullRecipient = await this.#client.get(recipient.id)
+        if (await this.#isRemoteActor(fullRecipient)) {
+          this.#logger.warn(`Skipping remote actor ${recipient.id}`)
+        } else if (await this.#isRemoteFollowersCollection(fullActor, fullRecipient)) {
+          await this.#deliverRemoteFollowersCollection(activity, fullRecipient, fullActor, deliveredTo, bots)
+        } else if (await this.#isRemoteFollowingCollection(fullActor, fullRecipient)) {
+          await this.#deliverRemoteFollowingCollection(activity, fullRecipient, fullActor, deliveredTo, bots)
+        } else if (await this.#isRemoteCollection(fullRecipient)) {
+          await this.#deliverRemoteCollection(activity, fullRecipient, deliveredTo, bots)
+        } else {
+          this.#logger.warn(`Unrecognized recipient: ${recipient.id}`)
+        }
+      }
+    }
+  }
+
+  async #isRemoteFollowersCollection (actor, object) {
+    assert.strictEqual(typeof actor, 'object')
+    assert.strictEqual(typeof actor.id, 'string')
+    assert.strictEqual(typeof object, 'object')
+    assert.strictEqual(typeof object.id, 'string')
+
+    return (actor.followers?.first?.id === object.id)
+  }
+
+  async #isRemoteFollowingCollection (actor, object) {
+    assert.strictEqual(typeof actor, 'object')
+    assert.strictEqual(typeof actor.id, 'string')
+    assert.strictEqual(typeof object, 'object')
+    assert.strictEqual(typeof object.id, 'string')
+
+    return (actor.following?.first?.id === object.id)
+  }
+
+  async #getLocalFollowers (actor) {
+    return await this.#actorStorage.getUsernamesWith('following', actor)
+  }
+
+  async #getLocalFollowing (actor) {
+    return await this.#actorStorage.getUsernamesWith('followers', actor)
+  }
+
+  async #deliverLocalActor (activity, recipient, bots, deliveredTo) {
+    const { username } = this.#formatter.unformat(recipient.id)
+    if (!deliveredTo.has(username)) {
+      const bot = await BotMaker.makeBot(bots, username)
+      if (!bot) {
+        this.#logger.warn(`sharedInbox direct delivery for unknown bot ${username}`)
+      }
+      await this.deliverTo(activity, bot)
+      deliveredTo.add(username)
+    }
+  }
+
+  async #deliverRemoteFollowersCollection (activity, recipient, actor, deliveredTo, bots) {
+    const followers = await this.#getLocalFollowers(actor)
+    for (const username of followers) {
+      if (!deliveredTo.has(username)) {
+        const bot = await BotMaker.makeBot(bots, username)
+        if (!bot) {
+          this.#logger.warn(`sharedInbox direct delivery for unknown bot ${username}`)
+          continue
+        }
+        await this.deliverTo(activity, bot)
+        deliveredTo.add(username)
+      }
+    }
+  }
+
+  async #deliverRemoteFollowingCollection (activity, recipient, actor, deliveredTo, bots) {
+    const following = await this.#getLocalFollowing(actor)
+    for (const username of following) {
+      if (!deliveredTo.has(username)) {
+        const bot = await BotMaker.makeBot(bots, username)
+        if (!bot) {
+          this.#logger.warn(`sharedInbox direct delivery for unknown bot ${username}`)
+          continue
+        }
+        await this.deliverTo(activity, bot)
+        deliveredTo.add(username)
+      }
+    }
+  }
+
+  #isPublic (recipient) {
+    return PUBLIC.includes(recipient.id)
+  }
+
+  #isLocal (recipient) {
+    return this.#formatter.isLocal(recipient.id)
+  }
+
+  #isLocalActor (parts) {
+    return parts.username && !parts.collection && !parts.type
+  }
+
+  #isLocalFollowersCollection (parts) {
+    return parts.username && (parts.collection === 'followers')
+  }
+
+  #isLocalFollowingCollection (parts) {
+    return parts.username && (parts.collection === 'following')
+  }
+
+  async #isRemoteCollection (recipient) {
+    assert.strictEqual(typeof recipient, 'object')
+    assert.strictEqual(typeof recipient.id, 'string')
+
+    return (Array.isArray(recipient.type))
+      ? recipient.type.some(item => COLLECTION_TYPES.includes(item))
+      : COLLECTION_TYPES.includes(recipient.type)
+  }
+
+  async #isRemoteActor (recipient) {
+    assert.strictEqual(typeof recipient, 'object')
+    assert.strictEqual(typeof recipient.id, 'string')
+
+    return !!recipient.inbox?.first?.id
+  }
+
+  async #deliverPublic (activity, bots) {
+    await Promise.all(Object.values(bots).map(bot => bot.onPublic(activity)))
+  }
+
+  async #deliverLocalFollowersCollection (activity, username, bots, deliveredTo) {
+    const id = this.#formatter.format({ username })
+    const followed = await as2.import({ id })
+    const followers = await this.#actorStorage.getUsernamesWith('following', followed)
+    for (const follower of followers) {
+      if (!deliveredTo.has(follower)) {
+        const bot = await BotMaker.makeBot(bots, follower)
+        if (!bot) {
+          this.#logger.warn(`sharedInbox delivery for unknown bot ${follower}`)
+          continue
+        }
+        await this.deliverTo(activity, bot)
+        deliveredTo.add(follower)
+      }
+    }
+  }
+
+  async #deliverLocalFollowingCollection (activity, username, bots, deliveredTo) {
+    const id = this.#formatter.format({ username })
+    const following = await as2.import({ id })
+    const followeds = await this.#actorStorage.getUsernamesWith('followers', following)
+    for (const followed of followeds) {
+      if (!deliveredTo.has(followed)) {
+        const bot = await BotMaker.makeBot(bots, followed)
+        if (!bot) {
+          this.#logger.warn(`sharedInbox delivery for unknown bot ${followed}`)
+          continue
+        }
+        await this.deliverTo(activity, bot)
+        deliveredTo.add(followed)
+      }
+    }
+  }
+
+  async #deliverRemoteCollection (activity, recipient, deliveredTo, bots) {
+    for await (const item of this.#client.items(recipient.id)) {
+      this.#logger.debug(`item: ${JSON.stringify(item)}`)
+      if (this.#isLocal(item)) {
+        const parts = this.#formatter.unformat(item.id)
+        if (this.#isLocalActor(parts)) {
+          const bot = await BotMaker.makeBot(bots, parts.username)
+          if (!bot) {
+            this.#logger.warn(`sharedInbox delivery for unknown bot ${parts.username}`)
+            continue
+          }
+          await this.deliverTo(activity, bot)
+          deliveredTo.add(parts.username)
+        }
+      }
+    }
+  }
+}

package/lib/activityhandler.js

@@ -314,6 +314,11 @@ export class ActivityHandler {
       object: activity,
       to: actor
     }))
+    this.#logger.debug({
+      msg: 'Notifying bot of new follow',
+      actor: actor.id
+    })
+    await bot.onFollow(actor, activity)
   }
 
   async #handleAccept (bot, activity) {
@@ -516,6 +521,7 @@ export class ActivityHandler {
       }),
       ...recipients
     }))
+    await bot.onLike(object, activity)
   }
 
   async #handleAnnounce (bot, activity) {
@@ -592,6 +598,14 @@ export class ActivityHandler {
       }),
       ...recipients
     }))
+    try {
+      await bot.onAnnounce(object, activity)
+    } catch (err) {
+      this.#logger.warn(
+        `Error notifying bot of announce: ${err.message}`,
+        { err, username: bot.username, activity: activity.id }
+      )
+    }
   }
 
   async #handleBlock (bot, activity) {

package/lib/activitypubclient.js

@@ -13,6 +13,13 @@ const { version } = JSON.parse(
   fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
 )
 
+const NS = 'https://www.w3.org/ns/activitystreams#'
+
+const COLLECTION_TYPES = [
+  `${NS}Collection`,
+  `${NS}OrderedCollection`
+]
+
 export class ActivityPubClient {
   static #githubUrl = 'https://github.com/evanp/activitypub-bot'
   static #userAgent = `activitypub.bot/${version} (${ActivityPubClient.#githubUrl})`
@@ -63,16 +70,20 @@ export class ActivityPubClient {
       'user-agent': ActivityPubClient.#userAgent
     }
     const method = 'GET'
+    this.#logger.debug(`Signing GET request for ${url}`)
     if (sign) {
       headers.signature =
         await this.#sign({ username, url, method, headers })
     }
-    return await fetch(url,
+    this.#logger.debug(`Fetching ${url} with GET`)
+    const result = await fetch(url,
       {
         method,
         headers
       }
     )
+    this.#logger.debug(`Finished getting ${url}`)
+    return result
   }
 
   async #handleRes (res, url) {
@@ -100,7 +111,9 @@ export class ActivityPubClient {
     }
     const method = 'POST'
     assert.ok(headers)
+    this.#logger.debug(`Signing POST for ${url}`)
     headers.signature = await this.#sign({ username, url, method, headers })
+    this.#logger.debug(`Fetching POST for ${url}`)
     const res = await fetch(url,
       {
         method,
@@ -108,6 +121,7 @@ export class ActivityPubClient {
         body
       }
     )
+    this.#logger.debug(`Done fetching POST for ${url}`)
     if (res.status < 200 || res.status > 299) {
       throw createHttpError(res.status, await res.text())
     }
@@ -123,4 +137,41 @@ export class ActivityPubClient {
       : this.#urlFormatter.format({ server: true, type: 'publickey' })
     return this.#signer.sign({ privateKey, keyId, url, method, headers })
   }
+
+  #isCollection (obj) {
+    return (Array.isArray(obj.type))
+      ? obj.type.some(item => COLLECTION_TYPES.includes(item))
+      : COLLECTION_TYPES.includes(obj.type)
+  }
+
+  async * items (id, username = null) {
+    const coll = await this.get(id, username)
+
+    this.#logger.debug(`Got object ${id}`)
+
+    if (!this.#isCollection(coll)) {
+      throw new Error(`Can only iterate over a collection: ${id}`)
+    }
+
+    const items = (coll.items) ? coll.items : coll.orderedItems
+
+    if (items) {
+      for (const item of items) {
+        this.#logger.debug(`Yielding ${item.id}`)
+        yield item
+      }
+    } else if (coll.first) {
+      for (let page = coll.first; page; page = page.next) {
+        this.#logger.debug(`Getting page ${page.id}`)
+        page = await this.get(page.id)
+        const items = (page.items) ? page.items : page.orderedItems
+        if (items) {
+          for (const item of items) {
+            this.#logger.debug(`Yielding ${item.id}`)
+            yield item
+          }
+        }
+      }
+    }
+  }
 }

package/lib/activitystreams.js

@@ -38,4 +38,35 @@ as2.registerContext('https://w3id.org/fep/5711', {
   }
 })
 
+as2.registerContext('https://w3id.org/security/v1', {
+  '@context': {
+    sec: 'https://w3id.org/security#',
+    id: '@id',
+    type: '@type',
+    owner: {
+      '@id': 'sec:owner',
+      '@type': '@id'
+    },
+    publicKey: {
+      '@id': 'sec:publicKey',
+      '@type': '@id'
+    },
+    publicKeyPem: 'sec:publicKeyPem'
+  }
+})
+
+as2.registerContext('https://purl.archive.org/socialweb/thread/1.0', {
+  '@context': {
+    thr: 'https://purl.archive.org/socialweb/thread#',
+    thread: {
+      '@id': 'thr:thread',
+      '@type': '@id'
+    },
+    root: {
+      '@id': 'thr:root',
+      '@type': '@id'
+    }
+  }
+})
+
 export default as2

package/lib/actorstorage.js

@@ -10,34 +10,6 @@ export class ActorStorage {
     this.#formatter = formatter
   }
 
-  async initialize () {
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS actorcollection (
-        username varchar(512) NOT NULL,
-        property varchar(512) NOT NULL,
-        first INTEGER NOT NULL,
-        totalItems INTEGER NOT NULL DEFAULT 0,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, property)
-      );`
-    )
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS actorcollectionpage (
-        username varchar(512) NOT NULL,
-        property varchar(512) NOT NULL,
-        item varchar(512) NOT NULL,
-        page INTEGER NOT NULL,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, property, item)
-      );`
-    )
-    await this.#connection.query(
-      `CREATE INDEX IF NOT EXISTS actorcollectionpage_username_property_page
-      ON actorcollectionpage (username, property, page);`
-    )
-  }
-
   async getActor (username, props = {}) {
     assert.ok(username)
     assert.equal(typeof username, 'string')
@@ -266,6 +238,24 @@ export class ActorStorage {
     return page <= first
   }
 
+  async getUsernamesWith (property, object) {
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+    const [result] = await this.#connection.query(
+      `SELECT username
+      FROM actorcollectionpage
+      WHERE property = ? AND item = ?`,
+      { replacements: [property, object.id] }
+    )
+    const usernames = []
+    for (const row of result) {
+      usernames.push(row.username)
+    }
+    return usernames
+  }
+
   async #getCollectionInfo (username, property) {
     const [result] = await this.#connection.query(
       `SELECT first, totalItems, createdAt, updatedAt

package/lib/app.js

@@ -22,32 +22,33 @@ import collectionRouter from './routes/collection.js'
 import inboxRouter from './routes/inbox.js'
 import healthRouter from './routes/health.js'
 import webfingerRouter from './routes/webfinger.js'
+import sharedInboxRouter from './routes/sharedinbox.js'
 import { BotContext } from './botcontext.js'
 import { Transformer } from './microsyntax.js'
 import { HTTPSignatureAuthenticator } from './httpsignatureauthenticator.js'
 import { Digester } from './digester.js'
+import { runMigrations } from './migrations/index.js'
+import { ActivityDeliverer } from './activitydeliverer.js'
 
 export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
   const logger = Logger({
     level: logLevel
   })
   logger.debug('Logger initialized')
-  const connection = new Sequelize(databaseUrl, { logging: false })
+  const connection = databaseUrl === 'sqlite::memory:' || databaseUrl === 'sqlite::memory'
+    ? new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
+    : new Sequelize(databaseUrl, { logging: false })
+  await runMigrations(connection)
   const formatter = new UrlFormatter(origin)
   const signer = new HTTPSignature(logger)
   const digester = new Digester(logger)
   const actorStorage = new ActorStorage(connection, formatter)
-  await actorStorage.initialize()
   const botDataStorage = new BotDataStorage(connection)
-  await botDataStorage.initialize()
   const keyStorage = new KeyStorage(connection, logger)
-  await keyStorage.initialize()
   const objectStorage = new ObjectStorage(connection)
-  await objectStorage.initialize()
   const client =
     new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
-  await remoteKeyStorage.initialize()
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const distributor = new ActivityDistributor(
     client,
@@ -71,6 +72,14 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     logger,
     client
   )
+  const deliverer = new ActivityDeliverer(
+    actorStorage,
+    activityHandler,
+    formatter,
+    logger,
+    client
+  )
+
   // TODO: Make an endpoint for tagged objects
   const transformer = new Transformer(origin + '/tag/', client)
   await Promise.all(
@@ -106,7 +115,8 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     authorizer,
     bots,
     activityHandler,
-    origin
+    origin,
+    deliverer
   }
 
   app.use(HTTPLogger({
@@ -134,6 +144,7 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
   app.use('/', inboxRouter)
   app.use('/', healthRouter)
   app.use('/', webfingerRouter)
+  app.use('/', sharedInboxRouter)
 
   app.use((err, req, res, next) => {
     const { logger } = req.app.locals

package/lib/bot.js

@@ -7,6 +7,9 @@ export default class Bot {
   }
 
   async initialize (context) {
+    if (context.botId !== this.#username) {
+      throw new Error(`Mismatched context: ${context.botId} !== ${this.#username}`)
+    }
     this.#context = context
   }
 
@@ -41,4 +44,8 @@ export default class Bot {
   async onAnnounce (object, activity) {
     ; // no-op
   }
+
+  async onPublic (activity) {
+    ; // no-op
+  }
 }

package/lib/botcontext.js

@@ -53,6 +53,22 @@ export class BotContext {
     this.#logger = logger.child({ class: 'BotContext', botId })
   }
 
+  // copy constructor
+
+  async duplicate (username) {
+    return new this.constructor(
+      username,
+      this.#botDataStorage,
+      this.#objectStorage,
+      this.#actorStorage,
+      this.#client,
+      this.#distributor,
+      this.#formatter,
+      this.#transformer,
+      this.#logger
+    )
+  }
+
   async setData (key, value) {
     await this.#botDataStorage.set(this.#botId, key, value)
   }
@@ -488,6 +504,40 @@ export class BotContext {
     return `${username}@${actorUrl.hostname}`
   }
 
+  async announceObject (obj) {
+    assert.ok(obj)
+    assert.equal(typeof obj, 'object')
+    const owners = obj.attributedTo
+      ? Array.from(obj.attributedTo).map((owner) => owner.id)
+      : Array.from(obj.actor).map((owner) => owner.id)
+    const activity = await as2.import({
+      type: 'Announce',
+      id: this.#formatter.format({
+        username: this.#botId,
+        type: 'Announce',
+        nanoid: nanoid()
+      }),
+      actor: this.#formatter.format({ username: this.#botId }),
+      summary: {
+        en: `${this.#botId} shared "${await this.#nameOf(obj)}"`
+      },
+      object: obj.id,
+      to: [
+        this.#formatter.format({
+          username: this.#botId,
+          collection: 'followers'
+        }),
+        'https://www.w3.org/ns/activitystreams#Public'
+      ],
+      cc: owners
+    })
+    await this.#objectStorage.create(activity)
+    await this.#actorStorage.addToCollection(this.#botId, 'outbox', activity)
+    await this.#actorStorage.addToCollection(this.#botId, 'inbox', activity)
+    await this.#distributor.distribute(activity, this.#botId)
+    return activity
+  }
+
   async #findInOutbox (type, obj) {
     const full = `https://www.w3.org/ns/activitystreams#${type}`
     let found = null
@@ -514,6 +564,18 @@ export class BotContext {
     return { to, cc, bto, bcc, audience }
   }
 
+  #nameOf (obj) {
+    if (obj.name) {
+      return obj.name.valueOf()
+    } else if (obj.summary) {
+      return obj.summary.valueOf()
+    } else if (obj.type) {
+      return `a(n) ${obj.type.first}`
+    } else {
+      return 'an object'
+    }
+  }
+
   async onIdle () {
     await this.#distributor.onIdle()
   }

package/lib/botdatastorage.js

@@ -15,19 +15,6 @@ export class BotDataStorage {
     this.#connection = connection
   }
 
-  async initialize () {
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS botdata (
-        username VARCHAR(512) not null,
-        key VARCHAR(512) not null,
-        value TEXT not null,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, key)
-      )
-    `)
-  }
-
   async terminate () {
 
   }

package/lib/botfactory.js

@@ -0,0 +1,24 @@
+export default class BotFactory {
+  #context
+
+  async initialize (context) {
+    this.#context = context
+  }
+
+  get _context () {
+    return this.#context
+  }
+
+  async canCreate (username) {
+    return false
+  }
+
+  async create (username) {
+    const name = this.constructor.name
+    throw new Error(`${name} class can't create bot named "${username}"`)
+  }
+
+  async onPublic (activity) {
+    ; // no-op
+  }
+}