@evanp/activitypub-bot 0.9.0 → 0.11.0

package/lib/activitypubclient.js

@@ -13,6 +13,13 @@ const { version } = JSON.parse(
   fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
 )
 
+const NS = 'https://www.w3.org/ns/activitystreams#'
+
+const COLLECTION_TYPES = [
+  `${NS}Collection`,
+  `${NS}OrderedCollection`
+]
+
 export class ActivityPubClient {
   static #githubUrl = 'https://github.com/evanp/activitypub-bot'
   static #userAgent = `activitypub.bot/${version} (${ActivityPubClient.#githubUrl})`
@@ -63,16 +70,20 @@ export class ActivityPubClient {
       'user-agent': ActivityPubClient.#userAgent
     }
     const method = 'GET'
+    this.#logger.debug(`Signing GET request for ${url}`)
     if (sign) {
       headers.signature =
         await this.#sign({ username, url, method, headers })
     }
-    return await fetch(url,
+    this.#logger.debug(`Fetching ${url} with GET`)
+    const result = await fetch(url,
       {
         method,
         headers
       }
     )
+    this.#logger.debug(`Finished getting ${url}`)
+    return result
   }
 
   async #handleRes (res, url) {
@@ -100,7 +111,9 @@ export class ActivityPubClient {
     }
     const method = 'POST'
     assert.ok(headers)
+    this.#logger.debug(`Signing POST for ${url}`)
     headers.signature = await this.#sign({ username, url, method, headers })
+    this.#logger.debug(`Fetching POST for ${url}`)
     const res = await fetch(url,
       {
         method,
@@ -108,6 +121,7 @@ export class ActivityPubClient {
         body
       }
     )
+    this.#logger.debug(`Done fetching POST for ${url}`)
     if (res.status < 200 || res.status > 299) {
       throw createHttpError(res.status, await res.text())
     }
@@ -123,4 +137,41 @@ export class ActivityPubClient {
       : this.#urlFormatter.format({ server: true, type: 'publickey' })
     return this.#signer.sign({ privateKey, keyId, url, method, headers })
   }
+
+  #isCollection (obj) {
+    return (Array.isArray(obj.type))
+      ? obj.type.some(item => COLLECTION_TYPES.includes(item))
+      : COLLECTION_TYPES.includes(obj.type)
+  }
+
+  async * items (id, username = null) {
+    const coll = await this.get(id, username)
+
+    this.#logger.debug(`Got object ${id}`)
+
+    if (!this.#isCollection(coll)) {
+      throw new Error(`Can only iterate over a collection: ${id}`)
+    }
+
+    const items = (coll.items) ? coll.items : coll.orderedItems
+
+    if (items) {
+      for (const item of items) {
+        this.#logger.debug(`Yielding ${item.id}`)
+        yield item
+      }
+    } else if (coll.first) {
+      for (let page = coll.first; page; page = page.next) {
+        this.#logger.debug(`Getting page ${page.id}`)
+        page = await this.get(page.id)
+        const items = (page.items) ? page.items : page.orderedItems
+        if (items) {
+          for (const item of items) {
+            this.#logger.debug(`Yielding ${item.id}`)
+            yield item
+          }
+        }
+      }
+    }
+  }
 }

package/lib/activitystreams.js

@@ -38,4 +38,35 @@ as2.registerContext('https://w3id.org/fep/5711', {
   }
 })
 
+as2.registerContext('https://w3id.org/security/v1', {
+  '@context': {
+    sec: 'https://w3id.org/security#',
+    id: '@id',
+    type: '@type',
+    owner: {
+      '@id': 'sec:owner',
+      '@type': '@id'
+    },
+    publicKey: {
+      '@id': 'sec:publicKey',
+      '@type': '@id'
+    },
+    publicKeyPem: 'sec:publicKeyPem'
+  }
+})
+
+as2.registerContext('https://purl.archive.org/socialweb/thread/1.0', {
+  '@context': {
+    thr: 'https://purl.archive.org/socialweb/thread#',
+    thread: {
+      '@id': 'thr:thread',
+      '@type': '@id'
+    },
+    root: {
+      '@id': 'thr:root',
+      '@type': '@id'
+    }
+  }
+})
+
 export default as2

package/lib/actorstorage.js

@@ -10,34 +10,6 @@ export class ActorStorage {
     this.#formatter = formatter
   }
 
-  async initialize () {
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS actorcollection (
-        username varchar(512) NOT NULL,
-        property varchar(512) NOT NULL,
-        first INTEGER NOT NULL,
-        totalItems INTEGER NOT NULL DEFAULT 0,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, property)
-      );`
-    )
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS actorcollectionpage (
-        username varchar(512) NOT NULL,
-        property varchar(512) NOT NULL,
-        item varchar(512) NOT NULL,
-        page INTEGER NOT NULL,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, property, item)
-      );`
-    )
-    await this.#connection.query(
-      `CREATE INDEX IF NOT EXISTS actorcollectionpage_username_property_page
-      ON actorcollectionpage (username, property, page);`
-    )
-  }
-
   async getActor (username, props = {}) {
     assert.ok(username)
     assert.equal(typeof username, 'string')
@@ -266,6 +238,24 @@ export class ActorStorage {
     return page <= first
   }
 
+  async getUsernamesWith (property, object) {
+    assert.ok(property, 'property is required')
+    assert.equal(typeof property, 'string', 'property must be a string')
+    assert.ok(object, 'object is required')
+    assert.equal(typeof object, 'object', 'object must be an object')
+    const [result] = await this.#connection.query(
+      `SELECT username
+      FROM actorcollectionpage
+      WHERE property = ? AND item = ?`,
+      { replacements: [property, object.id] }
+    )
+    const usernames = []
+    for (const row of result) {
+      usernames.push(row.username)
+    }
+    return usernames
+  }
+
   async #getCollectionInfo (username, property) {
     const [result] = await this.#connection.query(
       `SELECT first, totalItems, createdAt, updatedAt

package/lib/app.js

@@ -22,32 +22,33 @@ import collectionRouter from './routes/collection.js'
 import inboxRouter from './routes/inbox.js'
 import healthRouter from './routes/health.js'
 import webfingerRouter from './routes/webfinger.js'
+import sharedInboxRouter from './routes/sharedinbox.js'
 import { BotContext } from './botcontext.js'
 import { Transformer } from './microsyntax.js'
 import { HTTPSignatureAuthenticator } from './httpsignatureauthenticator.js'
 import { Digester } from './digester.js'
+import { runMigrations } from './migrations/index.js'
+import { ActivityDeliverer } from './activitydeliverer.js'
 
 export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
   const logger = Logger({
     level: logLevel
   })
   logger.debug('Logger initialized')
-  const connection = new Sequelize(databaseUrl, { logging: false })
+  const connection = databaseUrl === 'sqlite::memory:' || databaseUrl === 'sqlite::memory'
+    ? new Sequelize({ dialect: 'sqlite', storage: ':memory:', logging: false })
+    : new Sequelize(databaseUrl, { logging: false })
+  await runMigrations(connection)
   const formatter = new UrlFormatter(origin)
   const signer = new HTTPSignature(logger)
   const digester = new Digester(logger)
   const actorStorage = new ActorStorage(connection, formatter)
-  await actorStorage.initialize()
   const botDataStorage = new BotDataStorage(connection)
-  await botDataStorage.initialize()
   const keyStorage = new KeyStorage(connection, logger)
-  await keyStorage.initialize()
   const objectStorage = new ObjectStorage(connection)
-  await objectStorage.initialize()
   const client =
     new ActivityPubClient(keyStorage, formatter, signer, digester, logger)
   const remoteKeyStorage = new RemoteKeyStorage(client, connection, logger)
-  await remoteKeyStorage.initialize()
   const signature = new HTTPSignatureAuthenticator(remoteKeyStorage, signer, digester, logger)
   const distributor = new ActivityDistributor(
     client,
@@ -71,6 +72,14 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     logger,
     client
   )
+  const deliverer = new ActivityDeliverer(
+    actorStorage,
+    activityHandler,
+    formatter,
+    logger,
+    client
+  )
+
   // TODO: Make an endpoint for tagged objects
   const transformer = new Transformer(origin + '/tag/', client)
   await Promise.all(
@@ -106,7 +115,8 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
     authorizer,
     bots,
     activityHandler,
-    origin
+    origin,
+    deliverer
   }
 
   app.use(HTTPLogger({
@@ -134,6 +144,7 @@ export async function makeApp (databaseUrl, origin, bots, logLevel = 'silent') {
   app.use('/', inboxRouter)
   app.use('/', healthRouter)
   app.use('/', webfingerRouter)
+  app.use('/', sharedInboxRouter)
 
   app.use((err, req, res, next) => {
     const { logger } = req.app.locals

package/lib/bot.js

@@ -7,6 +7,9 @@ export default class Bot {
   }
 
   async initialize (context) {
+    if (context.botId !== this.#username) {
+      throw new Error(`Mismatched context: ${context.botId} !== ${this.#username}`)
+    }
     this.#context = context
   }
 
@@ -41,4 +44,8 @@ export default class Bot {
   async onAnnounce (object, activity) {
     ; // no-op
   }
+
+  async onPublic (activity) {
+    ; // no-op
+  }
 }

package/lib/botcontext.js

@@ -53,6 +53,22 @@ export class BotContext {
     this.#logger = logger.child({ class: 'BotContext', botId })
   }
 
+  // copy constructor
+
+  async duplicate (username) {
+    return new this.constructor(
+      username,
+      this.#botDataStorage,
+      this.#objectStorage,
+      this.#actorStorage,
+      this.#client,
+      this.#distributor,
+      this.#formatter,
+      this.#transformer,
+      this.#logger
+    )
+  }
+
   async setData (key, value) {
     await this.#botDataStorage.set(this.#botId, key, value)
   }
@@ -488,6 +504,40 @@ export class BotContext {
     return `${username}@${actorUrl.hostname}`
   }
 
+  async announceObject (obj) {
+    assert.ok(obj)
+    assert.equal(typeof obj, 'object')
+    const owners = obj.attributedTo
+      ? Array.from(obj.attributedTo).map((owner) => owner.id)
+      : Array.from(obj.actor).map((owner) => owner.id)
+    const activity = await as2.import({
+      type: 'Announce',
+      id: this.#formatter.format({
+        username: this.#botId,
+        type: 'Announce',
+        nanoid: nanoid()
+      }),
+      actor: this.#formatter.format({ username: this.#botId }),
+      summary: {
+        en: `${this.#botId} shared "${await this.#nameOf(obj)}"`
+      },
+      object: obj.id,
+      to: [
+        this.#formatter.format({
+          username: this.#botId,
+          collection: 'followers'
+        }),
+        'https://www.w3.org/ns/activitystreams#Public'
+      ],
+      cc: owners
+    })
+    await this.#objectStorage.create(activity)
+    await this.#actorStorage.addToCollection(this.#botId, 'outbox', activity)
+    await this.#actorStorage.addToCollection(this.#botId, 'inbox', activity)
+    await this.#distributor.distribute(activity, this.#botId)
+    return activity
+  }
+
   async #findInOutbox (type, obj) {
     const full = `https://www.w3.org/ns/activitystreams#${type}`
     let found = null
@@ -514,6 +564,18 @@ export class BotContext {
     return { to, cc, bto, bcc, audience }
   }
 
+  #nameOf (obj) {
+    if (obj.name) {
+      return obj.name.valueOf()
+    } else if (obj.summary) {
+      return obj.summary.valueOf()
+    } else if (obj.type) {
+      return `a(n) ${obj.type.first}`
+    } else {
+      return 'an object'
+    }
+  }
+
   async onIdle () {
     await this.#distributor.onIdle()
   }

package/lib/botdatastorage.js

@@ -15,19 +15,6 @@ export class BotDataStorage {
     this.#connection = connection
   }
 
-  async initialize () {
-    await this.#connection.query(`
-      CREATE TABLE IF NOT EXISTS botdata (
-        username VARCHAR(512) not null,
-        key VARCHAR(512) not null,
-        value TEXT not null,
-        createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-        PRIMARY KEY (username, key)
-      )
-    `)
-  }
-
   async terminate () {
 
   }

package/lib/botfactory.js

@@ -0,0 +1,24 @@
+export default class BotFactory {
+  #context
+
+  async initialize (context) {
+    this.#context = context
+  }
+
+  get _context () {
+    return this.#context
+  }
+
+  async canCreate (username) {
+    return false
+  }
+
+  async create (username) {
+    const name = this.constructor.name
+    throw new Error(`${name} class can't create bot named "${username}"`)
+  }
+
+  async onPublic (activity) {
+    ; // no-op
+  }
+}

package/lib/botmaker.js

@@ -0,0 +1,23 @@
+import assert from 'node:assert'
+
+export default class BotMaker {
+  static async makeBot (bots, username) {
+    assert.ok(bots)
+    assert.ok(typeof bots === 'object')
+    assert.ok(username)
+    assert.ok(typeof username === 'string')
+
+    let bot
+
+    if (username in bots) {
+      bot = bots[username]
+    } else if ('*' in bots) {
+      const factory = bots['*']
+      if (await factory.canCreate(username)) {
+        bot = await factory.create(username)
+      }
+    }
+
+    return bot
+  }
+}

package/lib/keystorage.js

@@ -17,27 +17,6 @@ export class KeyStorage {
     this.#hasher = new HumanHasher()
   }
 
-  async initialize () {
-    await this.#connection.query(`
-            CREATE TABLE IF NOT EXISTS new_keys (
-                username varchar(512) PRIMARY KEY,
-                public_key TEXT,
-                private_key TEXT
-            )
-        `)
-    try {
-      await this.#connection.query(`
-              INSERT OR IGNORE INTO new_keys (username, public_key, private_key)
-              SELECT bot_id, public_key, private_key
-              FROM keys
-          `)
-    } catch (error) {
-      this.#logger.debug(
-        { error, method: 'KeyStorage.initialize' },
-        'failed to copy keys to new_keys table')
-    }
-  }
-
   async getPublicKey (username) {
     this.#logger.debug(
       { username, method: 'KeyStorage.getPublicKey' },
@@ -57,9 +36,13 @@ export class KeyStorage {
   async #getKeys (username) {
     let privateKey
     let publicKey
-    const [result] = await this.#connection.query(`
-          SELECT public_key, private_key FROM new_keys WHERE username = ?
-      `, { replacements: [username] })
+    const qry = (username)
+      ? 'SELECT public_key, private_key FROM new_keys WHERE username = ?'
+      : 'SELECT public_key, private_key FROM new_keys WHERE username IS NULL'
+    const [result] = await this.#connection.query(
+      qry,
+      { replacements: [username] }
+    )
     if (result.length > 0) {
       this.#logger.debug(
         { username, method: 'KeyStorage.#getKeys' },

package/lib/migrations/001-initial.js

@@ -0,0 +1,107 @@
+export const id = '001-initial'
+
+export async function up (connection) {
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS actorcollection (
+      username varchar(512) NOT NULL,
+      property varchar(512) NOT NULL,
+      first INTEGER NOT NULL,
+      totalItems INTEGER NOT NULL DEFAULT 0,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (username, property)
+    );
+  `)
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS actorcollectionpage (
+      username varchar(512) NOT NULL,
+      property varchar(512) NOT NULL,
+      item varchar(512) NOT NULL,
+      page INTEGER NOT NULL,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (username, property, item)
+    );
+  `)
+  await connection.query(
+    `CREATE INDEX IF NOT EXISTS actorcollectionpage_username_property_page
+    ON actorcollectionpage (username, property, page);`
+  )
+
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS objects (
+      id VARCHAR(512) PRIMARY KEY,
+      data TEXT NOT NULL,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+    )
+  `)
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS collections (
+      id VARCHAR(512) NOT NULL,
+      property VARCHAR(512) NOT NULL,
+      first INTEGER NOT NULL,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (id, property)
+    )
+  `)
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS pages (
+      id VARCHAR(512) NOT NULL,
+      property VARCHAR(64) NOT NULL,
+      item VARCHAR(512) NOT NULL,
+      page INTEGER NOT NULL,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (id, property, item)
+    )
+  `)
+  await connection.query(
+    `CREATE INDEX IF NOT EXISTS pages_username_property_page
+    ON pages (id, property, page);`
+  )
+
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS botdata (
+      username VARCHAR(512) not null,
+      key VARCHAR(512) not null,
+      value TEXT not null,
+      createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (username, key)
+    )
+  `)
+
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS new_keys (
+      username varchar(512) PRIMARY KEY,
+      public_key TEXT,
+      private_key TEXT
+    )
+  `)
+  try {
+    await connection.query(`
+      INSERT OR IGNORE INTO new_keys (username, public_key, private_key)
+      SELECT bot_id, public_key, private_key
+      FROM keys
+    `)
+  } catch {
+  }
+
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS new_remotekeys (
+      id VARCHAR(512) PRIMARY KEY,
+      owner VARCHAR(512) NOT NULL,
+      publicKeyPem TEXT NOT NULL,
+      createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+      updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    )
+  `)
+  try {
+    await connection.query(`
+      INSERT OR IGNORE INTO new_remotekeys (id, owner, publicKeyPem)
+      SELECT id, owner, publicKeyPem
+      FROM remotekeys
+    `)
+  } catch {
+  }
+}

package/lib/migrations/index.js

@@ -0,0 +1,28 @@
+import { id as initialId, up as initialUp } from './001-initial.js'
+
+const migrations = [
+  { id: initialId, up: initialUp }
+]
+
+export async function runMigrations (connection) {
+  await connection.query(`
+    CREATE TABLE IF NOT EXISTS migrations (
+      id VARCHAR(255) PRIMARY KEY,
+      ranAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+    )
+  `)
+
+  const [rows] = await connection.query('SELECT id FROM migrations')
+  const applied = new Set(rows.map((row) => row.id))
+
+  for (const migration of migrations) {
+    if (applied.has(migration.id)) {
+      continue
+    }
+    await migration.up(connection)
+    await connection.query(
+      'INSERT INTO migrations (id) VALUES (?)',
+      { replacements: [migration.id] }
+    )
+  }
+}

package/lib/objectcache.js

@@ -1,4 +1,7 @@
-import TTLCache from '@isaacs/ttlcache'
+import * as ttlcachePkg from '@isaacs/ttlcache'
+
+const TTLCache =
+  ttlcachePkg.TTLCache ?? ttlcachePkg.default ?? ttlcachePkg
 
 export class ObjectCache {
   #objects = null

package/lib/objectstorage.js

@@ -17,42 +17,6 @@ export class ObjectStorage {
     this.#connection = connection
   }
 
-  async initialize () {
-    await this.#connection.query(`
-            CREATE TABLE IF NOT EXISTS objects (
-                id VARCHAR(512) PRIMARY KEY,
-                data TEXT NOT NULL,
-                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
-            )
-        `)
-    await this.#connection.query(`
-            CREATE TABLE IF NOT EXISTS collections (
-                id VARCHAR(512) NOT NULL,
-                property VARCHAR(512) NOT NULL,
-                first INTEGER NOT NULL,
-                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                updatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                PRIMARY KEY (id, property)
-            )
-        `)
-    await this.#connection.query(`
-            CREATE TABLE IF NOT EXISTS pages (
-                id VARCHAR(512) NOT NULL,
-                property VARCHAR(64) NOT NULL,
-                item VARCHAR(512) NOT NULL,
-                page INTEGER NOT NULL,
-                createdAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                PRIMARY KEY (id, property, item)
-            )
-        `)
-
-    await this.#connection.query(
-      `CREATE INDEX IF NOT EXISTS pages_username_property_page
-      ON pages (id, property, page);`
-    )
-  }
-
   async create (object) {
     assert.ok(this.#connection, 'ObjectStorage not initialized')
     assert.ok(object, 'object is required')