@atproto/sync 0.1.0

package/tests/firehose.test.ts

@@ -0,0 +1,180 @@
+import {
+  mockResolvers,
+  SeedClient,
+  TestNetworkNoAppView,
+} from '@atproto/dev-env'
+import { Firehose, FirehoseOptions, MemoryRunner } from '../src'
+import { IdResolver } from '@atproto/identity'
+import { Create, Event } from '../src/events'
+import { createDeferrable, wait } from '@atproto/common'
+
+describe('firehose', () => {
+  let network: TestNetworkNoAppView
+  let sc: SeedClient
+  let idResolver: IdResolver
+
+  beforeAll(async () => {
+    network = await TestNetworkNoAppView.create({
+      dbPostgresSchema: 'sync_firehose',
+    })
+    idResolver = new IdResolver({ plcUrl: network.plc.url })
+    mockResolvers(idResolver, network.pds)
+    sc = network.getSeedClient()
+  })
+
+  afterAll(async () => {
+    await network.close()
+  })
+
+  const createAndReadFirehose = async (
+    count: number,
+    opts: Partial<FirehoseOptions> = {},
+    addRandomWait = false,
+  ): Promise<Event[]> => {
+    const defer = createDeferrable()
+    const evts: Event[] = []
+    const firehose = new Firehose({
+      idResolver,
+      service: network.pds.url.replace('http', 'ws'),
+      handleEvent: async (evt) => {
+        if (addRandomWait) {
+          const time = Math.floor(Math.random()) * 20
+          await wait(time)
+        }
+        evts.push(evt)
+        if (evts.length >= count) {
+          defer.resolve()
+        }
+      },
+      onError: (err) => {
+        throw err
+      },
+      ...opts,
+    })
+    firehose.start()
+    await defer.complete
+    await firehose.destroy()
+    return evts
+  }
+
+  let alice: string
+
+  it('reads events from firehose', async () => {
+    const evtsPromise = createAndReadFirehose(5)
+    await wait(10) // give the websocket just a second to spin up
+    const aliceRes = await sc.createAccount('alice', {
+      handle: 'alice.test',
+      email: 'alice@test.com',
+      password: 'alice-pass',
+    })
+    alice = aliceRes.did
+    await sc.post(alice, 'one')
+    await sc.post(alice, 'two')
+    await sc.post(alice, 'three')
+
+    const evts = await evtsPromise
+    expect(evts.length).toBe(5)
+    expect(evts.at(0)).toMatchObject({
+      event: 'identity',
+      did: alice,
+      handle: aliceRes.handle,
+      didDocument: {
+        id: alice,
+      },
+    })
+    expect(evts.at(1)).toMatchObject({
+      event: 'account',
+      did: alice,
+      active: true,
+      status: undefined,
+    })
+    expect(evts.at(2)).toMatchObject({
+      event: 'create',
+      did: alice,
+      collection: 'app.bsky.feed.post',
+      record: {
+        text: 'one',
+      },
+    })
+    expect(evts.at(3)).toMatchObject({
+      event: 'create',
+      did: alice,
+      collection: 'app.bsky.feed.post',
+      record: {
+        text: 'two',
+      },
+    })
+    expect(evts.at(4)).toMatchObject({
+      event: 'create',
+      did: alice,
+      collection: 'app.bsky.feed.post',
+      record: {
+        text: 'three',
+      },
+    })
+  })
+
+  it('does not naively pass through invalid handle evts', async () => {
+    const evtsPromise = createAndReadFirehose(1)
+    await wait(10) // give the websocket just a second to spin up
+    await network.pds.ctx.sequencer.sequenceIdentityEvt(
+      alice,
+      'bad-handle.test',
+    )
+    const evts = await evtsPromise
+    expect(evts.at(0)).toMatchObject({ handle: 'alice.test' })
+  })
+
+  it('processes events through the sync queue', async () => {
+    const currCursor = await network.pds.ctx.sequencer.curr()
+    const runner = new MemoryRunner({
+      startCursor: currCursor ?? undefined,
+    })
+    const evtsPromise = createAndReadFirehose(20, { runner }, true)
+    const createAndPost = async (name: string) => {
+      const user = await sc.createAccount('name', {
+        handle: `${name}.test`,
+        email: `${name}@example.com`,
+        password: `${name}-pass`,
+      })
+      const did = user.did
+      const post1 = await sc.post(did, 'one')
+      const post2 = await sc.post(did, 'two')
+      const post3 = await sc.post(did, 'three')
+      return {
+        did,
+        post1: post1.ref.uriStr,
+        post2: post2.ref.uriStr,
+        post3: post3.ref.uriStr,
+      }
+    }
+    const res = await Promise.all([
+      createAndPost('user1'),
+      createAndPost('user2'),
+      createAndPost('user3'),
+      createAndPost('user4'),
+    ])
+    const evts = await evtsPromise
+    const user1Evts = evts.filter((e) => e.did === res[0].did)
+    const user2Evts = evts.filter((e) => e.did === res[1].did)
+    const user3Evts = evts.filter((e) => e.did === res[2].did)
+    const user4Evts = evts.filter((e) => e.did === res[3].did)
+    const EVT_ORDER = ['identity', 'account', 'create', 'create', 'create']
+    expect(user1Evts.map((e) => e.event)).toEqual(EVT_ORDER)
+    expect(user2Evts.map((e) => e.event)).toEqual(EVT_ORDER)
+    expect(user3Evts.map((e) => e.event)).toEqual(EVT_ORDER)
+    expect(user4Evts.map((e) => e.event)).toEqual(EVT_ORDER)
+    expect(
+      user1Evts.slice(2, 5).map((e) => (e as Create).uri.toString()),
+    ).toEqual([res[0].post1, res[0].post2, res[0].post3])
+    expect(
+      user2Evts.slice(2, 5).map((e) => (e as Create).uri.toString()),
+    ).toEqual([res[1].post1, res[1].post2, res[1].post3])
+    expect(
+      user3Evts.slice(2, 5).map((e) => (e as Create).uri.toString()),
+    ).toEqual([res[2].post1, res[2].post2, res[2].post3])
+    expect(
+      user4Evts.slice(2, 5).map((e) => (e as Create).uri.toString()),
+    ).toEqual([res[3].post1, res[3].post2, res[3].post3])
+  })
+})

package/tests/runner.test.ts

@@ -0,0 +1,122 @@
+import { wait } from '@atproto/common'
+import { ConsecutiveList, MemoryRunner } from '../src/runner'
+
+describe('EventRunner utils', () => {
+  describe('ConsecutiveList', () => {
+    it('tracks consecutive complete items.', () => {
+      const consecutive = new ConsecutiveList<number>()
+      // add items
+      const item1 = consecutive.push(1)
+      const item2 = consecutive.push(2)
+      const item3 = consecutive.push(3)
+      expect(item1.isComplete).toEqual(false)
+      expect(item2.isComplete).toEqual(false)
+      expect(item3.isComplete).toEqual(false)
+      // complete items out of order
+      expect(consecutive.list.length).toBe(3)
+      expect(item2.complete()).toEqual([])
+      expect(item2.isComplete).toEqual(true)
+      expect(consecutive.list.length).toBe(3)
+      expect(item1.complete()).toEqual([1, 2])
+      expect(item1.isComplete).toEqual(true)
+      expect(consecutive.list.length).toBe(1)
+      expect(item3.complete()).toEqual([3])
+      expect(consecutive.list.length).toBe(0)
+      expect(item3.isComplete).toEqual(true)
+    })
+  })
+
+  describe('MemoryRunner', () => {
+    it('performs work in parallel across partitions, serial within a partition.', async () => {
+      const runner = new MemoryRunner({ concurrency: Infinity })
+      const complete: number[] = []
+      // partition 1 items start slow but get faster: slow should still complete first.
+      runner.addTask('1', async () => {
+        await wait(30)
+        complete.push(11)
+      })
+      runner.addTask('1', async () => {
+        await wait(20)
+        complete.push(12)
+      })
+      runner.addTask('1', async () => {
+        await wait(1)
+        complete.push(13)
+      })
+      expect(runner.partitions.size).toEqual(1)
+      // partition 2 items complete quickly except the last, which is slowest of all events.
+      runner.addTask('2', async () => {
+        await wait(1)
+        complete.push(21)
+      })
+      runner.addTask('2', async () => {
+        await wait(1)
+        complete.push(22)
+      })
+      runner.addTask('2', async () => {
+        await wait(1)
+        complete.push(23)
+      })
+      runner.addTask('2', async () => {
+        await wait(60)
+        complete.push(24)
+      })
+      expect(runner.partitions.size).toEqual(2)
+      await runner.mainQueue.onIdle()
+      expect(complete).toEqual([21, 22, 23, 11, 12, 13, 24])
+      expect(runner.partitions.size).toEqual(0)
+    })
+
+    it('limits overall concurrency.', async () => {
+      const runner = new MemoryRunner({ concurrency: 1 })
+      const complete: number[] = []
+      // if concurrency were not constrained, partition 1 would complete all items
+      // before any items from partition 2. since it is constrained, the work is complete in the order added.
+      runner.addTask('1', async () => {
+        await wait(1)
+        complete.push(11)
+      })
+      runner.addTask('2', async () => {
+        await wait(10)
+        complete.push(21)
+      })
+      runner.addTask('1', async () => {
+        await wait(1)
+        complete.push(12)
+      })
+      runner.addTask('2', async () => {
+        await wait(10)
+        complete.push(22)
+      })
+      // only partition 1 exists so far due to the concurrency
+      expect(runner.partitions.size).toEqual(1)
+      await runner.mainQueue.onIdle()
+      expect(complete).toEqual([11, 21, 12, 22])
+      expect(runner.partitions.size).toEqual(0)
+    })
+
+    it('settles with many items.', async () => {
+      const runner = new MemoryRunner({ concurrency: 100 })
+      const complete: { partition: string; id: number }[] = []
+      const partitions = new Set<string>()
+      for (let i = 0; i < 500; ++i) {
+        const partition = Math.floor(Math.random() * 16).toString(10)
+        partitions.add(partition)
+        runner.addTask(partition, async () => {
+          await wait((i % 2) * 2)
+          complete.push({ partition, id: i })
+        })
+      }
+      expect(runner.partitions.size).toEqual(partitions.size)
+      await runner.mainQueue.onIdle()
+      expect(complete.length).toEqual(500)
+      for (const partition of partitions) {
+        const ids = complete
+          .filter((item) => item.partition === partition)
+          .map((item) => item.id)
+        expect(ids).toEqual([...ids].sort((a, b) => a - b))
+      }
+      expect(runner.partitions.size).toEqual(0)
+    })
+  })
+})

package/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig/node.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
+}

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "include": [],
+  "references": [{ "path": "./tsconfig.build.json" }]
+}