@atproto/bsky 0.0.16 → 0.0.18

package/tests/daemon.test.ts

@@ -0,0 +1,191 @@
+import assert from 'assert'
+import { AtUri } from '@atproto/api'
+import { TestNetwork } from '@atproto/dev-env'
+import { BskyDaemon, DaemonConfig, PrimaryDatabase } from '../src'
+import usersSeed from './seeds/users'
+import { countAll, excluded } from '../src/db/util'
+import { NotificationsDaemon } from '../src/daemon/notifications'
+import {
+  BEFORE_LAST_SEEN_DAYS,
+  BEFORE_LATEST_UNREAD_DAYS,
+  UNREAD_KEPT_COUNT,
+} from '../src/services/util/notification'
+
+describe('daemon', () => {
+  let network: TestNetwork
+  let daemon: BskyDaemon
+  let db: PrimaryDatabase
+  let actors: { did: string }[] = []
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'bsky_daemon',
+    })
+    db = network.bsky.ctx.db.getPrimary()
+    daemon = BskyDaemon.create({
+      db,
+      cfg: new DaemonConfig({
+        version: network.bsky.ctx.cfg.version,
+        dbPostgresUrl: network.bsky.ctx.cfg.dbPrimaryPostgresUrl,
+        dbPostgresSchema: network.bsky.ctx.cfg.dbPostgresSchema,
+      }),
+    })
+    const sc = network.getSeedClient()
+    await usersSeed(sc)
+    await network.processAll()
+    actors = await db.db.selectFrom('actor').selectAll().execute()
+  })
+
+  afterAll(async () => {
+    await network.close()
+  })
+
+  describe('notifications daemon', () => {
+    it('processes all dids', async () => {
+      for (const { did } of actors) {
+        await Promise.all([
+          setLastSeen(daemon.ctx.db, { did }),
+          createNotifications(daemon.ctx.db, {
+            did,
+            daysAgo: 2 * BEFORE_LAST_SEEN_DAYS,
+            count: 1,
+          }),
+        ])
+      }
+      await expect(countNotifications(db)).resolves.toBe(actors.length)
+      await runNotifsOnce(daemon.notifications)
+      await expect(countNotifications(db)).resolves.toBe(0)
+    })
+
+    it('removes read notifications older than threshold.', async () => {
+      const { did } = actors[0]
+      const lastSeenDaysAgo = 10
+      await Promise.all([
+        setLastSeen(daemon.ctx.db, { did, daysAgo: lastSeenDaysAgo }),
+        // read, delete
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS + 1,
+          count: 2,
+        }),
+        // read, keep
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS - 1,
+          count: 3,
+        }),
+        // unread, keep
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: lastSeenDaysAgo - 1,
+          count: 4,
+        }),
+      ])
+      await expect(countNotifications(db)).resolves.toBe(9)
+      await runNotifsOnce(daemon.notifications)
+      await expect(countNotifications(db)).resolves.toBe(7)
+      await clearNotifications(db)
+    })
+
+    it('removes unread notifications older than threshold.', async () => {
+      const { did } = actors[0]
+      await Promise.all([
+        setLastSeen(daemon.ctx.db, {
+          did,
+          daysAgo: 2 * BEFORE_LATEST_UNREAD_DAYS, // all are unread
+        }),
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: 0,
+          count: 1,
+        }),
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: BEFORE_LATEST_UNREAD_DAYS - 1,
+          count: 99,
+        }),
+        createNotifications(daemon.ctx.db, {
+          did,
+          daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1,
+          count: 400,
+        }),
+      ])
+      await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT)
+      await runNotifsOnce(daemon.notifications)
+      // none removed when within UNREAD_KEPT_COUNT
+      await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT)
+      // add one more, tip over UNREAD_KEPT_COUNT
+      await createNotifications(daemon.ctx.db, {
+        did,
+        daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1,
+        count: 1,
+      })
+      await runNotifsOnce(daemon.notifications)
+      // removed all older than BEFORE_LATEST_UNREAD_DAYS
+      await expect(countNotifications(db)).resolves.toBe(100)
+      await clearNotifications(db)
+    })
+  })
+
+  const runNotifsOnce = async (notifsDaemon: NotificationsDaemon) => {
+    assert(!notifsDaemon.running, 'notifications daemon is already running')
+    notifsDaemon.run({ forever: false, batchSize: 2 })
+    await notifsDaemon.running
+  }
+
+  const setLastSeen = async (
+    db: PrimaryDatabase,
+    opts: { did: string; daysAgo?: number },
+  ) => {
+    const { did, daysAgo = 0 } = opts
+    const lastSeenAt = new Date()
+    lastSeenAt.setDate(lastSeenAt.getDate() - daysAgo)
+    await db.db
+      .insertInto('actor_state')
+      .values({ did, lastSeenNotifs: lastSeenAt.toISOString() })
+      .onConflict((oc) =>
+        oc.column('did').doUpdateSet({
+          lastSeenNotifs: excluded(db.db, 'lastSeenNotifs'),
+        }),
+      )
+      .execute()
+  }
+
+  const createNotifications = async (
+    db: PrimaryDatabase,
+    opts: {
+      did: string
+      count: number
+      daysAgo: number
+    },
+  ) => {
+    const { did, count, daysAgo } = opts
+    const sortAt = new Date()
+    sortAt.setDate(sortAt.getDate() - daysAgo)
+    await db.db
+      .insertInto('notification')
+      .values(
+        [...Array(count)].map(() => ({
+          did,
+          author: did,
+          reason: 'none',
+          recordCid: 'bafycid',
+          recordUri: AtUri.make(did, 'invalid.collection', 'self').toString(),
+          sortAt: sortAt.toISOString(),
+        })),
+      )
+      .execute()
+  }
+
+  const clearNotifications = async (db: PrimaryDatabase) => {
+    await db.db.deleteFrom('notification').execute()
+  }
+
+  const countNotifications = async (db: PrimaryDatabase) => {
+    const { count } = await db.db
+      .selectFrom('notification')
+      .select(countAll.as('count'))
+      .executeTakeFirstOrThrow()
+    return count
+  }
+})

package/tests/did-cache.test.ts

@@ -1,14 +1,16 @@
 import { TestNetwork, SeedClient } from '@atproto/dev-env'
 import userSeed from './seeds/users'
 import { IdResolver } from '@atproto/identity'
-import DidSqlCache from '../src/did-cache'
+import DidRedisCache from '../src/did-cache'
 import { wait } from '@atproto/common'
+import { Redis } from '../src'
 
 describe('did cache', () => {
   let network: TestNetwork
   let sc: SeedClient
   let idResolver: IdResolver
-  let didCache: DidSqlCache
+  let redis: Redis
+  let didCache: DidRedisCache
 
   let alice: string
   let bob: string
@@ -20,6 +22,7 @@ describe('did cache', () => {
       dbPostgresSchema: 'bsky_did_cache',
     })
     idResolver = network.bsky.indexer.ctx.idResolver
+    redis = network.bsky.indexer.ctx.redis
     didCache = network.bsky.indexer.ctx.didCache
     sc = network.getSeedClient()
     await userSeed(sc)
@@ -50,7 +53,12 @@ describe('did cache', () => {
   })
 
   it('clears cache and repopulates', async () => {
-    await idResolver.did.cache?.clear()
+    await Promise.all([
+      idResolver.did.cache?.clearEntry(alice),
+      idResolver.did.cache?.clearEntry(bob),
+      idResolver.did.cache?.clearEntry(carol),
+      idResolver.did.cache?.clearEntry(dan),
+    ])
     const docsCleared = await Promise.all([
       idResolver.did.cache?.checkCache(alice),
       idResolver.did.cache?.checkCache(bob),
@@ -81,7 +89,10 @@ describe('did cache', () => {
   })
 
   it('accurately reports expired dids & refreshes the cache', async () => {
-    const didCache = new DidSqlCache(network.bsky.ctx.db.getPrimary(), 1, 60000)
+    const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
+      staleTTL: 1,
+      maxTTL: 60000,
+    })
     const shortCacheResolver = new IdResolver({
       plcUrl: network.bsky.ctx.cfg.didPlcUrl,
       didCache,
@@ -110,7 +121,10 @@ describe('did cache', () => {
   })
 
   it('does not return expired dids & refreshes the cache', async () => {
-    const didCache = new DidSqlCache(network.bsky.ctx.db.getPrimary(), 0, 1)
+    const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
+      staleTTL: 0,
+      maxTTL: 1,
+    })
     const shortExpireResolver = new IdResolver({
       plcUrl: network.bsky.ctx.cfg.didPlcUrl,
       didCache,
@@ -125,5 +139,6 @@ describe('did cache', () => {
     // see that the resolver does not return expired value & instead force refreshes
     const staleGet = await shortExpireResolver.did.resolve(alice)
     expect(staleGet?.id).toEqual(alice)
+    await didCache.destroy()
   })
 })

package/tests/handle-invalidation.test.ts

@@ -102,11 +102,7 @@ describe('handle invalidation', () => {
   it('deals with handle contention', async () => {
     await backdateIndexedAt(bob)
     // update alices handle so that the pds will let bob take her old handle
-    await network.pds.ctx.db.db
-      .updateTable('did_handle')
-      .where('did', '=', alice)
-      .set({ handle: 'not-alice.test' })
-      .execute()
+    await network.pds.ctx.accountManager.updateHandle(alice, 'not-alice.test')
 
     await pdsAgent.api.com.atproto.identity.updateHandle(
       {

package/tests/indexing.test.ts

@@ -498,7 +498,8 @@ describe('indexing', () => {
 
     it('skips invalid records.', async () => {
       const { db, services } = network.bsky.indexer.ctx
-      const { db: pdsDb, services: pdsServices } = network.pds.ctx
+      const { accountManager } = network.pds.ctx
+      // const { db: pdsDb, services: pdsServices } = network.pds.ctx
       // Create a good and a bad post record
       const writes = await Promise.all([
         pdsRepo.prepareCreate({
@@ -513,9 +514,20 @@ describe('indexing', () => {
           validate: false,
         }),
       ])
-      await pdsServices
-        .repo(pdsDb)
-        .processWrites({ did: sc.dids.alice, writes }, 1)
+      const writeCommit = await network.pds.ctx.actorStore.transact(
+        sc.dids.alice,
+        (store) => store.repo.processWrites(writes),
+      )
+      await accountManager.updateRepoRoot(
+        sc.dids.alice,
+        writeCommit.cid,
+        writeCommit.rev,
+      )
+      await network.pds.ctx.sequencer.sequenceCommit(
+        sc.dids.alice,
+        writeCommit,
+        writes,
+      )
       // Index
       const { data: commit } =
         await pdsAgent.api.com.atproto.sync.getLatestCommit({
@@ -643,15 +655,10 @@ describe('indexing', () => {
       )
       await expect(getProfileBefore).resolves.toBeDefined()
       // Delete account on pds
-      await pdsAgent.api.com.atproto.server.requestAccountDelete(undefined, {
-        headers: sc.getHeaders(alice),
-      })
-      const { token } = await network.pds.ctx.db.db
-        .selectFrom('email_token')
-        .selectAll()
-        .where('purpose', '=', 'delete_account')
-        .where('did', '=', alice)
-        .executeTakeFirstOrThrow()
+      const token = await network.pds.ctx.accountManager.createEmailToken(
+        alice,
+        'delete_account',
+      )
       await pdsAgent.api.com.atproto.server.deleteAccount({
         token,
         did: alice,

package/tests/redis-cache.test.ts

@@ -0,0 +1,231 @@
+import { wait } from '@atproto/common'
+import { Redis } from '../src/'
+import { ReadThroughCache } from '../src/cache/read-through'
+
+describe('redis cache', () => {
+  let redis: Redis
+
+  beforeAll(async () => {
+    redis = new Redis({ host: process.env.REDIS_HOST || '' })
+  })
+
+  afterAll(async () => {
+    await redis.destroy()
+  })
+
+  it('caches according to namespace', async () => {
+    const ns1 = redis.withNamespace('ns1')
+    const ns2 = redis.withNamespace('ns2')
+    await Promise.all([
+      ns1.set('key', 'a'),
+      ns2.set('key', 'b'),
+      redis.set('key', 'c'),
+    ])
+    const got = await Promise.all([
+      ns1.get('key'),
+      ns2.get('key'),
+      redis.get('key'),
+    ])
+    expect(got[0]).toEqual('a')
+    expect(got[1]).toEqual('b')
+    expect(got[2]).toEqual('c')
+
+    await Promise.all([
+      ns1.setMulti({ key1: 'a', key2: 'b' }),
+      ns2.setMulti({ key1: 'c', key2: 'd' }),
+      redis.setMulti({ key1: 'e', key2: 'f' }),
+    ])
+    const gotMany = await Promise.all([
+      ns1.getMulti(['key1', 'key2']),
+      ns2.getMulti(['key1', 'key2']),
+      redis.getMulti(['key1', 'key2']),
+    ])
+    expect(gotMany[0]['key1']).toEqual('a')
+    expect(gotMany[0]['key2']).toEqual('b')
+    expect(gotMany[1]['key1']).toEqual('c')
+    expect(gotMany[1]['key2']).toEqual('d')
+    expect(gotMany[2]['key1']).toEqual('e')
+    expect(gotMany[2]['key2']).toEqual('f')
+  })
+
+  it('caches values when empty', async () => {
+    const vals = {
+      '1': 'a',
+      '2': 'b',
+      '3': 'c',
+    }
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test1'), {
+      staleTTL: 60000,
+      maxTTL: 60000,
+      fetchMethod: async (key) => {
+        hits++
+        return vals[key]
+      },
+    })
+    const got = await Promise.all([
+      cache.get('1'),
+      cache.get('2'),
+      cache.get('3'),
+    ])
+    expect(got[0]).toEqual('a')
+    expect(got[1]).toEqual('b')
+    expect(got[2]).toEqual('c')
+    expect(hits).toBe(3)
+
+    const refetched = await Promise.all([
+      cache.get('1'),
+      cache.get('2'),
+      cache.get('3'),
+    ])
+    expect(refetched[0]).toEqual('a')
+    expect(refetched[1]).toEqual('b')
+    expect(refetched[2]).toEqual('c')
+    expect(hits).toBe(3)
+  })
+
+  it('skips and refreshes cache when requested', async () => {
+    let val = 'a'
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test2'), {
+      staleTTL: 60000,
+      maxTTL: 60000,
+      fetchMethod: async () => {
+        hits++
+        return val
+      },
+    })
+
+    const try1 = await cache.get('1')
+    expect(try1).toEqual('a')
+    expect(hits).toBe(1)
+
+    val = 'b'
+
+    const try2 = await cache.get('1')
+    expect(try2).toEqual('a')
+    expect(hits).toBe(1)
+
+    const try3 = await cache.get('1', { revalidate: true })
+    expect(try3).toEqual('b')
+    expect(hits).toBe(2)
+
+    const try4 = await cache.get('1')
+    expect(try4).toEqual('b')
+    expect(hits).toBe(2)
+  })
+
+  it('accurately reports stale entries & refreshes the cache', async () => {
+    let val = 'a'
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test3'), {
+      staleTTL: 1,
+      maxTTL: 60000,
+      fetchMethod: async () => {
+        hits++
+        return val
+      },
+    })
+
+    const try1 = await cache.get('1')
+    expect(try1).toEqual('a')
+
+    await wait(5)
+
+    val = 'b'
+
+    const try2 = await cache.get('1')
+    // cache gives us stale value while it revalidates
+    expect(try2).toEqual('a')
+
+    await wait(5)
+
+    const try3 = await cache.get('1')
+    expect(try3).toEqual('b')
+    expect(hits).toEqual(3)
+  })
+
+  it('does not return expired dids & refreshes the cache', async () => {
+    let val = 'a'
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test4'), {
+      staleTTL: 0,
+      maxTTL: 1,
+      fetchMethod: async () => {
+        hits++
+        return val
+      },
+    })
+
+    const try1 = await cache.get('1')
+    expect(try1).toEqual('a')
+
+    await wait(5)
+
+    val = 'b'
+
+    const try2 = await cache.get('1')
+    expect(try2).toEqual('b')
+    expect(hits).toBe(2)
+  })
+
+  it('caches negative values', async () => {
+    let val: string | null = null
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test5'), {
+      staleTTL: 60000,
+      maxTTL: 60000,
+      fetchMethod: async () => {
+        hits++
+        return val
+      },
+    })
+
+    const try1 = await cache.get('1')
+    expect(try1).toEqual(null)
+    expect(hits).toBe(1)
+
+    val = 'b'
+
+    const try2 = await cache.get('1')
+    // returns cached negative value
+    expect(try2).toEqual(null)
+    expect(hits).toBe(1)
+
+    const try3 = await cache.get('1', { revalidate: true })
+    expect(try3).toEqual('b')
+    expect(hits).toEqual(2)
+
+    const try4 = await cache.get('1')
+    expect(try4).toEqual('b')
+    expect(hits).toEqual(2)
+  })
+
+  it('times out and fails open', async () => {
+    let val = 'a'
+    let hits = 0
+    const cache = new ReadThroughCache<string>(redis.withNamespace('test6'), {
+      staleTTL: 60000,
+      maxTTL: 60000,
+      fetchMethod: async () => {
+        hits++
+        return val
+      },
+    })
+
+    const try1 = await cache.get('1')
+    expect(try1).toEqual('a')
+
+    const orig = cache.redis.driver.get
+    cache.redis.driver.get = async (key) => {
+      await wait(600)
+      return orig(key)
+    }
+
+    val = 'b'
+
+    const try2 = await cache.get('1')
+    expect(try2).toEqual('b')
+    expect(hits).toBe(2)
+  })
+})

package/tests/seeds/basic.ts

@@ -103,6 +103,8 @@ export default async (sc: SeedClient, users = true) => {
     'tests/sample-img/key-landscape-small.jpg',
     'image/jpeg',
   )
+  // must ensure ordering of replies in indexing
+  await sc.network.processAll()
   await sc.reply(
     bob,
     sc.posts[alice][1].ref,
@@ -117,6 +119,7 @@ export default async (sc: SeedClient, users = true) => {
     sc.posts[alice][1].ref,
     replies.carol[0],
   )
+  await sc.network.processAll()
   const alicesReplyToBob = await sc.reply(
     alice,
     sc.posts[alice][1].ref,

package/tests/subscription/repo.test.ts

@@ -1,7 +1,6 @@
 import AtpAgent from '@atproto/api'
 import { TestNetwork, SeedClient } from '@atproto/dev-env'
 import { CommitData } from '@atproto/repo'
-import { RepoService } from '@atproto/pds/src/services/repo'
 import { PreparedWrite } from '@atproto/pds/src/repo'
 import * as sequencer from '@atproto/pds/src/sequencer'
 import { cborDecode, cborEncode } from '@atproto/common'
@@ -84,9 +83,8 @@ describe('sync', () => {
 
   it('indexes actor when commit is unprocessable.', async () => {
     // mock sequencing to create an unprocessable commit event
-    const afterWriteProcessingOriginal =
-      RepoService.prototype.afterWriteProcessing
-    RepoService.prototype.afterWriteProcessing = async function (
+    const sequenceCommitOrig = network.pds.ctx.sequencer.sequenceCommit
+    network.pds.ctx.sequencer.sequenceCommit = async function (
       did: string,
       commitData: CommitData,
       writes: PreparedWrite[],
@@ -95,7 +93,7 @@ describe('sync', () => {
       const evt = cborDecode(seqEvt.event) as sequencer.CommitEvt
       evt.blocks = new Uint8Array() // bad blocks
       seqEvt.event = cborEncode(evt)
-      await sequencer.sequenceEvt(this.db, seqEvt)
+      await network.pds.ctx.sequencer.sequenceEvt(seqEvt)
     }
     // create account and index the initial commit event
     await sc.createAccount('jack', {
@@ -103,12 +101,11 @@ describe('sync', () => {
       email: 'jack@test.com',
       password: 'password',
     })
-    await network.pds.ctx.sequencerLeader?.isCaughtUp()
     await network.processAll()
     // confirm jack was indexed as an actor despite the bad event
     const actors = await dumpTable(ctx.db.getPrimary(), 'actor', ['did'])
     expect(actors.map((a) => a.handle)).toContain('jack.test')
-    RepoService.prototype.afterWriteProcessing = afterWriteProcessingOriginal
+    network.pds.ctx.sequencer.sequenceCommit = sequenceCommitOrig
   })
 
   async function updateProfile(

package/tests/views/profile.test.ts

@@ -25,7 +25,6 @@ describe('pds profile views', () => {
     sc = network.getSeedClient()
     await basicSeed(sc)
     await network.processAll()
-    await network.bsky.processAll()
     alice = sc.dids.alice
     bob = sc.dids.bob
     dan = sc.dids.dan