@atproto/bsky 0.0.81 → 0.0.83

package/src/data-plane/server/subscription/util.ts

@@ -1,156 +0,0 @@
-import assert from 'node:assert'
-import PQueue from 'p-queue'
-import { OutputSchema as RepoMessage } from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-import * as message from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-
-// A queue with arbitrarily many partitions, each processing work sequentially.
-// Partitions are created lazily and taken out of memory when they go idle.
-export class PartitionedQueue {
-  main: PQueue
-  partitions = new Map<string, PQueue>()
-
-  constructor(opts: { concurrency: number }) {
-    this.main = new PQueue({ concurrency: opts.concurrency })
-  }
-
-  async add(partitionId: string, task: () => Promise<void>) {
-    if (this.main.isPaused) return
-    return this.main.add(() => {
-      return this.getPartition(partitionId).add(task)
-    })
-  }
-
-  async destroy() {
-    this.main.pause()
-    this.main.clear()
-    this.partitions.forEach((p) => p.clear())
-    await this.main.onIdle() // All in-flight work completes
-  }
-
-  private getPartition(partitionId: string) {
-    let partition = this.partitions.get(partitionId)
-    if (!partition) {
-      partition = new PQueue({ concurrency: 1 })
-      partition.once('idle', () => this.partitions.delete(partitionId))
-      this.partitions.set(partitionId, partition)
-    }
-    return partition
-  }
-}
-
-export class LatestQueue {
-  queue = new PQueue({ concurrency: 1 })
-
-  async add(task: () => Promise<void>) {
-    if (this.queue.isPaused) return
-    this.queue.clear() // Only queue the latest task, invalidate any previous ones
-    return this.queue.add(task)
-  }
-
-  async destroy() {
-    this.queue.pause()
-    this.queue.clear()
-    await this.queue.onIdle() // All in-flight work completes
-  }
-}
-
-/**
- * Add items to a list, and mark those items as
- * completed. Upon item completion, get list of consecutive
- * items completed at the head of the list. Example:
- *
- * const consecutive = new ConsecutiveList<number>()
- * const item1 = consecutive.push(1)
- * const item2 = consecutive.push(2)
- * const item3 = consecutive.push(3)
- * item2.complete() // []
- * item1.complete() // [1, 2]
- * item3.complete() // [3]
- *
- */
-export class ConsecutiveList<T> {
-  list: ConsecutiveItem<T>[] = []
-
-  push(value: T) {
-    const item = new ConsecutiveItem<T>(this, value)
-    this.list.push(item)
-    return item
-  }
-
-  complete(): T[] {
-    let i = 0
-    while (this.list[i]?.isComplete) {
-      i += 1
-    }
-    return this.list.splice(0, i).map((item) => item.value)
-  }
-}
-
-export class ConsecutiveItem<T> {
-  isComplete = false
-  constructor(
-    private consecutive: ConsecutiveList<T>,
-    public value: T,
-  ) {}
-
-  complete() {
-    this.isComplete = true
-    return this.consecutive.complete()
-  }
-}
-
-export class PerfectMap<K, V> extends Map<K, V> {
-  get(key: K): V {
-    const val = super.get(key)
-    assert(val !== undefined, `Key not found in PerfectMap: ${key}`)
-    return val
-  }
-}
-
-// These are the message types that have a sequence number and a repo
-export type ProcessableMessage =
-  | message.Commit
-  | message.Handle
-  | message.Identity
-  | message.Migrate
-  | message.Tombstone
-
-export function loggableMessage(msg: RepoMessage) {
-  if (message.isCommit(msg)) {
-    const { seq, rebase, prev, repo, commit, time, tooBig, blobs } = msg
-    return {
-      $type: msg.$type,
-      seq,
-      rebase,
-      prev: prev?.toString(),
-      repo,
-      commit: commit.toString(),
-      time,
-      tooBig,
-      hasBlobs: blobs.length > 0,
-    }
-  } else if (message.isHandle(msg)) {
-    return msg
-  } else if (message.isIdentity(msg)) {
-    return msg
-  } else if (message.isAccount(msg)) {
-    return msg
-  } else if (message.isMigrate(msg)) {
-    return msg
-  } else if (message.isTombstone(msg)) {
-    return msg
-  } else if (message.isInfo(msg)) {
-    return msg
-  }
-  return msg
-}
-
-export function jitter(maxMs) {
-  return Math.round((Math.random() - 0.5) * maxMs * 2)
-}
-
-export function strToInt(str: string) {
-  const int = parseInt(str, 10)
-  assert(!isNaN(int), 'string could not be parsed to an integer')
-  return int
-}

package/tests/data-plane/subscription/util.test.ts

@@ -1,185 +0,0 @@
-import { wait } from '@atproto/common'
-import { randomStr } from '@atproto/crypto'
-import {
-  ConsecutiveList,
-  LatestQueue,
-  PartitionedQueue,
-} from '../../../src/data-plane/server/subscription/util'
-
-describe('subscription utils', () => {
-  describe('ConsecutiveList', () => {
-    it('tracks consecutive complete items.', () => {
-      const consecutive = new ConsecutiveList<number>()
-      // add items
-      const item1 = consecutive.push(1)
-      const item2 = consecutive.push(2)
-      const item3 = consecutive.push(3)
-      expect(item1.isComplete).toEqual(false)
-      expect(item2.isComplete).toEqual(false)
-      expect(item3.isComplete).toEqual(false)
-      // complete items out of order
-      expect(consecutive.list.length).toBe(3)
-      expect(item2.complete()).toEqual([])
-      expect(item2.isComplete).toEqual(true)
-      expect(consecutive.list.length).toBe(3)
-      expect(item1.complete()).toEqual([1, 2])
-      expect(item1.isComplete).toEqual(true)
-      expect(consecutive.list.length).toBe(1)
-      expect(item3.complete()).toEqual([3])
-      expect(consecutive.list.length).toBe(0)
-      expect(item3.isComplete).toEqual(true)
-    })
-  })
-
-  describe('LatestQueue', () => {
-    it('only performs most recently queued item.', async () => {
-      const latest = new LatestQueue()
-      const complete: number[] = []
-      latest.add(async () => {
-        await wait(1)
-        complete.push(1)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(2)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(3)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(4)
-      })
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1, 4]) // skip 2, 3
-      latest.add(async () => {
-        await wait(1)
-        complete.push(5)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(6)
-      })
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1, 4, 5, 6])
-    })
-
-    it('stops processing queued messages on destroy.', async () => {
-      const latest = new LatestQueue()
-      const complete: number[] = []
-      latest.add(async () => {
-        await wait(1)
-        complete.push(1)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(2)
-      })
-      const destroyed = latest.destroy()
-      latest.add(async () => {
-        await wait(1)
-        complete.push(3)
-      })
-      await destroyed
-      expect(complete).toEqual([1]) // 2 was cleared, 3 was after destroy
-      // show that waiting on destroyed above was already enough to reflect all complete items
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1])
-    })
-  })
-
-  describe('PartitionedQueue', () => {
-    it('performs work in parallel across partitions, serial within a partition.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: Infinity })
-      const complete: number[] = []
-      // partition 1 items start slow but get faster: slow should still complete first.
-      partitioned.add('1', async () => {
-        await wait(30)
-        complete.push(11)
-      })
-      partitioned.add('1', async () => {
-        await wait(20)
-        complete.push(12)
-      })
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(13)
-      })
-      expect(partitioned.partitions.size).toEqual(1)
-      // partition 2 items complete quickly except the last, which is slowest of all events.
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(21)
-      })
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(22)
-      })
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(23)
-      })
-      partitioned.add('2', async () => {
-        await wait(60)
-        complete.push(24)
-      })
-      expect(partitioned.partitions.size).toEqual(2)
-      await partitioned.main.onIdle()
-      expect(complete).toEqual([21, 22, 23, 11, 12, 13, 24])
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-
-    it('limits overall concurrency.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: 1 })
-      const complete: number[] = []
-      // if concurrency were not constrained, partition 1 would complete all items
-      // before any items from partition 2. since it is constrained, the work is complete in the order added.
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(11)
-      })
-      partitioned.add('2', async () => {
-        await wait(10)
-        complete.push(21)
-      })
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(12)
-      })
-      partitioned.add('2', async () => {
-        await wait(10)
-        complete.push(22)
-      })
-      // only partition 1 exists so far due to the concurrency
-      expect(partitioned.partitions.size).toEqual(1)
-      await partitioned.main.onIdle()
-      expect(complete).toEqual([11, 21, 12, 22])
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-
-    it('settles with many items.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: 100 })
-      const complete: { partition: string; id: number }[] = []
-      const partitions = new Set<string>()
-      for (let i = 0; i < 500; ++i) {
-        const partition = randomStr(1, 'base16').slice(0, 1)
-        partitions.add(partition)
-        partitioned.add(partition, async () => {
-          await wait((i % 2) * 2)
-          complete.push({ partition, id: i })
-        })
-      }
-      expect(partitioned.partitions.size).toEqual(partitions.size)
-      await partitioned.main.onIdle()
-      expect(complete.length).toEqual(500)
-      for (const partition of partitions) {
-        const ids = complete
-          .filter((item) => item.partition === partition)
-          .map((item) => item.id)
-        expect(ids).toEqual([...ids].sort((a, b) => a - b))
-      }
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-  })
-})