@atproto/bsky 0.0.124 → 0.0.126

package/tests/etcd.test.ts

@@ -0,0 +1,301 @@
+import EventEmitter from 'node:events'
+import { Etcd3, IKeyValue } from 'etcd3'
+import { EtcdHostList } from '../src'
+import { EtcdMap } from '../src/etcd'
+
+describe('etcd', () => {
+  describe('EtcdMap', () => {
+    it('initializes values based on current keys', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      etcd.watcher.set('service/b', { value: '2' })
+      etcd.watcher.set('service/c', { value: '3' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      expect(map.get('service/a')).toBe('1')
+      expect(map.get('service/b')).toBe('2')
+      expect(map.get('service/c')).toBe('3')
+      expect([...map.values()]).toEqual(['1', '2', '3'])
+    })
+
+    it('maintains key updates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      etcd.watcher.set('service/b', { value: '2' })
+      etcd.watcher.set('service/c', { value: '3' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      etcd.watcher.set('service/b', { value: '4' })
+      expect(map.get('service/a')).toBe('1')
+      expect(map.get('service/b')).toBe('4')
+      expect(map.get('service/c')).toBe('3')
+      expect([...map.values()]).toEqual(['1', '4', '3'])
+    })
+
+    it('maintains key creates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      etcd.watcher.set('service/b', { value: '2' })
+      expect(map.get('service/a')).toBe('1')
+      expect(map.get('service/b')).toBe('2')
+      expect([...map.values()]).toEqual(['1', '2'])
+    })
+
+    it('maintains key deletions', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      etcd.watcher.set('service/b', { value: '2' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      etcd.watcher.del('service/b')
+      expect(map.get('service/a')).toBe('1')
+      expect(map.get('service/b')).toBe(null)
+      expect([...map.values()]).toEqual(['1'])
+    })
+
+    it('notifies of updates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      etcd.watcher.set('service/b', { value: '2' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      const states: string[][] = [[...map.values()]]
+      map.onUpdate((update) => {
+        states.push([...update.values()])
+      })
+      etcd.watcher.set('service/c', { value: '3' })
+      etcd.watcher.del('service/b')
+      etcd.watcher.set('service/a', { value: '4' })
+      expect(states).toEqual([
+        ['1', '2'],
+        ['1', '2', '3'],
+        ['1', '3'],
+        ['4', '3'],
+      ])
+    })
+
+    it('ignores out-of-order updates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: '1' })
+      const map = new EtcdMap(etcd as unknown as Etcd3)
+      await map.connect()
+      const states: string[][] = [[...map.values()]]
+      map.onUpdate((update) => {
+        states.push([...update.values()])
+      })
+      etcd.watcher.set('service/a', { value: '2' })
+      etcd.watcher.set('service/a', { value: '3', overrideRev: 1 }) // old rev
+      etcd.watcher.set('service/a', { value: '4' })
+      expect(map.get('service/a')).toBe('4')
+      expect(states).toEqual([['1'], ['2'], ['4']]) // never witnessed 3
+    })
+  })
+
+  describe('EtcdHostList', () => {
+    it('initializes values based on current keys', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      etcd.watcher.set('service/c', { value: 'http://192.168.1.3' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      expect([...hostList.get()]).toEqual([
+        'http://192.168.1.1',
+        'http://192.168.1.2',
+        'http://192.168.1.3',
+      ])
+    })
+
+    it('maintains key updates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      etcd.watcher.set('service/c', { value: 'http://192.168.1.3' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.4' })
+      expect([...hostList.get()]).toEqual([
+        'http://192.168.1.1',
+        'http://192.168.1.4',
+        'http://192.168.1.3',
+      ])
+    })
+
+    it('maintains key creates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      expect([...hostList.get()]).toEqual([
+        'http://192.168.1.1',
+        'http://192.168.1.2',
+      ])
+    })
+
+    it('maintains key deletions', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      etcd.watcher.del('service/b')
+      expect([...hostList.get()]).toEqual(['http://192.168.1.1'])
+    })
+
+    it('notifies of updates', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      const states: string[][] = [[...hostList.get()]]
+      hostList.onUpdate((updated) => {
+        expect([...updated]).toEqual([...hostList.get()])
+        states.push([...updated])
+      })
+      etcd.watcher.set('service/c', { value: 'http://192.168.1.3' })
+      etcd.watcher.del('service/b')
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.4' })
+      expect(states).toEqual([
+        ['http://192.168.1.1', 'http://192.168.1.2'],
+        ['http://192.168.1.1', 'http://192.168.1.2', 'http://192.168.1.3'],
+        ['http://192.168.1.1', 'http://192.168.1.3'],
+        ['http://192.168.1.4', 'http://192.168.1.3'],
+      ])
+    })
+
+    it('ignores bad host values', async () => {
+      const etcd = new MockEtcd()
+      etcd.watcher.set('service/a', { value: 'not-a-host' })
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '')
+      await hostList.connect()
+      expect([...hostList.get()]).toEqual(['http://192.168.1.2'])
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      etcd.watcher.set('service/c', { value: 'not-a-host' })
+      expect([...hostList.get()]).toEqual([
+        'http://192.168.1.1',
+        'http://192.168.1.2',
+      ])
+      etcd.watcher.set('service/c', { value: 'http://192.168.1.3' })
+      expect([...hostList.get()]).toEqual([
+        'http://192.168.1.1',
+        'http://192.168.1.2',
+        'http://192.168.1.3',
+      ])
+    })
+
+    it('falls back to static host list when uninitialized or no keys available', async () => {
+      const etcd = new MockEtcd()
+      const hostList = new EtcdHostList(etcd as unknown as Etcd3, '', [
+        'http://10.0.0.1',
+        'http://10.0.0.2',
+      ])
+      etcd.watcher.set('service/a', { value: 'http://192.168.1.1' })
+      expect([...hostList.get()]).toEqual([
+        'http://10.0.0.1',
+        'http://10.0.0.2',
+      ])
+      await hostList.connect()
+      const states: string[][] = [[...hostList.get()]]
+      hostList.onUpdate((updated) => {
+        states.push([...updated])
+      })
+      etcd.watcher.del('service/a')
+      etcd.watcher.set('service/b', { value: 'http://192.168.1.2' })
+      expect(states).toEqual([
+        ['http://192.168.1.1'],
+        ['http://10.0.0.1', 'http://10.0.0.2'],
+        ['http://192.168.1.2'],
+      ])
+    })
+  })
+})
+
+class MockEtcd {
+  public watcher = new MockWatcher()
+  watch() {
+    const watcher = this.watcher
+    return {
+      prefix() {
+        return {
+          watcher() {
+            return watcher
+          },
+        }
+      },
+    }
+  }
+  getAll() {
+    const watcher = this.watcher
+    return {
+      prefix() {
+        return {
+          async exec(): Promise<{ kvs: IKeyValue[] }> {
+            return { kvs: watcher.getAll() }
+          },
+        }
+      },
+    }
+  }
+}
+
+class MockWatcher extends EventEmitter {
+  rev = 1
+  kvs: IKeyValue[] = []
+  constructor() {
+    super()
+    process.nextTick(() => this.emit('connected', {}))
+  }
+  get(key: string): IKeyValue | null {
+    const found = this.kvs.find((kv) => kv.key.toString() === key)
+    return found ?? null
+  }
+  getAll(): IKeyValue[] {
+    return [...this.kvs]
+  }
+  set(
+    key: string,
+    { value, overrideRev }: { value: string; overrideRev?: number },
+  ) {
+    const found = this.kvs.find((kv) => kv.key.toString() === key)
+    const rev = overrideRev ?? ++this.rev
+    if (found) {
+      found.value = Buffer.from(value)
+      found.mod_revision = rev.toString()
+      found.version = (parseInt(found.version, 10) + 1).toString()
+      this.emit('put', found)
+    } else {
+      const created = {
+        key: Buffer.from(key),
+        value: Buffer.from(value),
+        create_revision: rev.toString(),
+        mod_revision: rev.toString(),
+        version: '1',
+        lease: '0',
+      }
+      this.kvs.push(created)
+      this.emit('put', created)
+    }
+  }
+  del(key: string) {
+    const foundIdx = this.kvs.findIndex((kv) => kv.key.toString() === key)
+    if (foundIdx === -1) return
+    const [deleted] = this.kvs.splice(foundIdx, 1)
+    const rev = ++this.rev
+    deleted.value = Buffer.from('')
+    deleted.mod_revision = rev.toString()
+    deleted.create_revision = '0'
+    deleted.version = '0'
+    this.emit('delete', deleted)
+  }
+  on(evt: 'connected', listener: (res: unknown) => void): any
+  on(evt: 'put', listener: (kv: IKeyValue) => void): any
+  on(evt: 'delete', listener: (kv: IKeyValue) => void): any
+  on(evt: string, listener: (...args: any[]) => void) {
+    super.on(evt, listener)
+  }
+}

package/tests/views/__snapshots__/notifications.test.ts.snap

@@ -384,7 +384,7 @@ Array [
 
 exports[`notification views fetches notifications with default priority 1`] = `
 Object {
-  "cursor": "0000000000000__bafycid",
+  "cursor": "1970-01-01T00:00:00.000Z",
   "notifications": Array [
     Object {
       "author": Object {
@@ -486,7 +486,7 @@ Object {
 
 exports[`notification views fetches notifications with explicit priority 1`] = `
 Object {
-  "cursor": "0000000000000__bafycid",
+  "cursor": "1970-01-01T00:00:00.000Z",
   "notifications": Array [
     Object {
       "author": Object {
@@ -588,7 +588,7 @@ Object {
 
 exports[`notification views fetches notifications with explicit priority 2`] = `
 Object {
-  "cursor": "0000000000000__bafycid",
+  "cursor": "1970-01-01T00:00:00.000Z",
   "notifications": Array [
     Object {
       "author": Object {

package/tests/views/notifications.test.ts

@@ -1,5 +1,6 @@
 import { AtpAgent } from '@atproto/api'
 import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env'
+import { delayCursor } from '../../src/api/app/bsky/notification/listNotifications'
 import { ids } from '../../src/lexicon/lexicons'
 import { Notification } from '../../src/lexicon/types/app/bsky/notification/listNotifications'
 import { forSnapshot, paginateAll } from '../_util'
@@ -497,17 +498,196 @@ describe('notification views', () => {
     expect(results(paginatedAll)).toEqual(results([full.data]))
   })
 
-  it('fails open on clearly bad cursor.', async () => {
-    const { data: notifs } =
-      await agent.api.app.bsky.notification.listNotifications(
-        { cursor: '90210::bafycid' },
-        {
-          headers: await network.serviceHeaders(
-            alice,
-            ids.AppBskyNotificationListNotifications,
-          ),
+  describe('notifications delay', () => {
+    const notificationsDelayMs = 5_000
+
+    let delayNetwork: TestNetwork
+    let delayAgent: AtpAgent
+    let delaySc: SeedClient
+    let delayAlice: string
+
+    beforeAll(async () => {
+      delayNetwork = await TestNetwork.create({
+        bsky: {
+          notificationsDelayMs,
         },
+        dbPostgresSchema: 'bsky_views_notifications_delay',
+      })
+      delayAgent = delayNetwork.bsky.getClient()
+      delaySc = delayNetwork.getSeedClient()
+      await basicSeed(delaySc)
+      await delayNetwork.processAll()
+      delayAlice = delaySc.dids.alice
+
+      // Add to reply chain, post ancestors: alice -> bob -> alice -> carol.
+      // Should have added one notification for each of alice and bob.
+      await delaySc.reply(
+        delaySc.dids.carol,
+        delaySc.posts[delayAlice][1].ref,
+        delaySc.replies[delayAlice][0].ref,
+        'indeed',
+      )
+      await delayNetwork.processAll()
+
+      // @NOTE: Use fake timers after inserting seed data,
+      // to avoid inserting all notifications with the same timestamp.
+      jest.useFakeTimers({
+        doNotFake: [
+          'nextTick',
+          'performance',
+          'setImmediate',
+          'setInterval',
+          'setTimeout',
+        ],
+      })
+    })
+
+    afterAll(async () => {
+      jest.useRealTimers()
+      await delayNetwork.close()
+    })
+
+    it('paginates', async () => {
+      const firstNotification = await delayNetwork.bsky.db.db
+        .selectFrom('notification')
+        .selectAll()
+        .limit(1)
+        .orderBy('sortAt', 'asc')
+        .executeTakeFirstOrThrow()
+      // Sets the system time to when the first notification happened.
+      // At this point we won't have any notifications that already crossed the delay threshold.
+      jest.setSystemTime(new Date(firstNotification.sortAt))
+
+      const results = (results) =>
+        sort(results.flatMap((res) => res.notifications))
+      const paginator = async (cursor?: string) => {
+        const res =
+          await delayAgent.api.app.bsky.notification.listNotifications(
+            { cursor, limit: 6 },
+            {
+              headers: await delayNetwork.serviceHeaders(
+                delayAlice,
+                ids.AppBskyNotificationListNotifications,
+              ),
+            },
+          )
+        return res.data
+      }
+
+      const paginatedAllBeforeDelay = await paginateAll(paginator)
+      paginatedAllBeforeDelay.forEach((res) =>
+        expect(res.notifications.length).toBe(0),
+      )
+      const fullBeforeDelay =
+        await delayAgent.api.app.bsky.notification.listNotifications(
+          {},
+          {
+            headers: await delayNetwork.serviceHeaders(
+              delayAlice,
+              ids.AppBskyNotificationListNotifications,
+            ),
+          },
+        )
+
+      expect(fullBeforeDelay.data.notifications.length).toEqual(0)
+      expect(results(paginatedAllBeforeDelay)).toEqual(
+        results([fullBeforeDelay.data]),
+      )
+
+      const lastNotification = await delayNetwork.bsky.db.db
+        .selectFrom('notification')
+        .selectAll()
+        .limit(1)
+        .orderBy('sortAt', 'desc')
+        .executeTakeFirstOrThrow()
+      // Sets the system time to when the last notification happened and the delay has elapsed.
+      // At this point we all notifications already crossed the delay threshold.
+      jest.setSystemTime(
+        new Date(
+          new Date(lastNotification.sortAt).getTime() +
+            notificationsDelayMs +
+            1,
+        ),
       )
-    expect(notifs).toMatchObject({ notifications: [] })
+
+      const paginatedAllAfterDelay = await paginateAll(paginator)
+      paginatedAllAfterDelay.forEach((res) =>
+        expect(res.notifications.length).toBeLessThanOrEqual(6),
+      )
+      const fullAfterDelay =
+        await delayAgent.api.app.bsky.notification.listNotifications(
+          {},
+          {
+            headers: await delayNetwork.serviceHeaders(
+              delayAlice,
+              ids.AppBskyNotificationListNotifications,
+            ),
+          },
+        )
+
+      expect(fullAfterDelay.data.notifications.length).toEqual(13)
+      expect(results(paginatedAllAfterDelay)).toEqual(
+        results([fullAfterDelay.data]),
+      )
+    })
+
+    describe('cursor delay', () => {
+      const delay0s = 0
+      const delay5s = 5_000
+
+      const now = '2021-01-01T01:00:00.000Z'
+      const nowMinus2s = '2021-01-01T00:59:58.000Z'
+      const nowMinus5s = '2021-01-01T00:59:55.000Z'
+      const nowMinus8s = '2021-01-01T00:59:52.000Z'
+
+      beforeAll(async () => {
+        jest.useFakeTimers({ doNotFake: ['performance'] })
+        jest.setSystemTime(new Date(now))
+      })
+
+      afterAll(async () => {
+        jest.useRealTimers()
+      })
+
+      describe('for undefined cursor', () => {
+        it('returns now minus delay', async () => {
+          const delayedCursor = delayCursor(undefined, delay5s)
+          expect(delayedCursor).toBe(nowMinus5s)
+        })
+
+        it('returns now if delay is 0', async () => {
+          const delayedCursor = delayCursor(undefined, delay0s)
+          expect(delayedCursor).toBe(now)
+        })
+      })
+
+      describe('for defined cursor', () => {
+        it('returns original cursor if delay is 0', async () => {
+          const originalCursor = nowMinus2s
+          const delayedCursor = delayCursor(originalCursor, delay0s)
+          expect(delayedCursor).toBe(originalCursor)
+        })
+
+        it('returns "now minus delay" for cursor that is after that', async () => {
+          // Cursor is "now - 2s", should become "now - 5s"
+          const originalCursor = nowMinus2s
+          const cursor = delayCursor(originalCursor, delay5s)
+          expect(cursor).toBe(nowMinus5s)
+        })
+
+        it('returns original cursor for cursor that is before "now minus delay"', async () => {
+          // Cursor is "now - 8s", should stay like that.
+          const originalCursor = nowMinus8s
+          const cursor = delayCursor(originalCursor, delay5s)
+          expect(cursor).toBe(originalCursor)
+        })
+
+        it('passes through a non-date cursor', async () => {
+          const originalCursor = '123_abc'
+          const cursor = delayCursor(originalCursor, delay5s)
+          expect(cursor).toBe(originalCursor)
+        })
+      })
+    })
   })
 })