@atproto/bsky 0.0.80 → 0.0.82

package/src/data-plane/server/subscription/index.ts

@@ -1,352 +0,0 @@
-import assert from 'node:assert'
-import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
-import { Subscription } from '@atproto/xrpc-server'
-import { cborDecode, handleAllSettledErrors } from '@atproto/common'
-import { ValidationError } from '@atproto/lexicon'
-import { IdResolver } from '@atproto/identity'
-import {
-  WriteOpAction,
-  readCarWithRoot,
-  cborToLexRecord,
-  def,
-  Commit,
-} from '@atproto/repo'
-import { ids, lexicons } from '../../../lexicon/lexicons'
-import { OutputSchema as Message } from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-import * as message from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-import { subLogger as log } from '../../../logger'
-import { IndexingService } from '../indexing'
-import { Database } from '../db'
-import {
-  ConsecutiveItem,
-  ConsecutiveList,
-  PartitionedQueue,
-  ProcessableMessage,
-  loggableMessage,
-} from './util'
-import { BackgroundQueue } from '../background'
-
-export class RepoSubscription {
-  ac = new AbortController()
-  running: Promise<void> | undefined
-  cursor = 0
-  seenSeq: number | null = null
-  repoQueue = new PartitionedQueue({ concurrency: Infinity })
-  consecutive = new ConsecutiveList<number>()
-  background: BackgroundQueue
-  indexingSvc: IndexingService
-
-  constructor(
-    private opts: {
-      service: string
-      db: Database
-      idResolver: IdResolver
-      background: BackgroundQueue
-    },
-  ) {
-    this.background = new BackgroundQueue(this.opts.db)
-    this.indexingSvc = new IndexingService(
-      this.opts.db,
-      this.opts.idResolver,
-      this.background,
-    )
-  }
-
-  run() {
-    if (this.running) return
-    this.ac = new AbortController()
-    this.repoQueue = new PartitionedQueue({ concurrency: Infinity })
-    this.consecutive = new ConsecutiveList<number>()
-    this.running = this.process()
-      .catch((err) => {
-        if (err.name !== 'AbortError') {
-          // allow this to cause an unhandled rejection, let deployment handle the crash.
-          log.error({ err }, 'subscription crashed')
-          throw err
-        }
-      })
-      .finally(() => (this.running = undefined))
-  }
-
-  private async process() {
-    const sub = this.getSubscription()
-    for await (const msg of sub) {
-      const details = getMessageDetails(msg)
-      if ('info' in details) {
-        // These messages are not sequenced, we just log them and carry on
-        log.warn(
-          { provider: this.opts.service, message: loggableMessage(msg) },
-          `sub ${details.info ? 'info' : 'unknown'} message`,
-        )
-        continue
-      }
-      const item = this.consecutive.push(details.seq)
-      this.repoQueue.add(details.repo, async () => {
-        await this.handleMessage(item, details)
-      })
-      this.seenSeq = details.seq
-      await this.repoQueue.main.onEmpty() // backpressure
-    }
-  }
-
-  private async handleMessage(
-    item: ConsecutiveItem<number>,
-    envelope: Envelope,
-  ) {
-    const msg = envelope.message
-    try {
-      if (message.isCommit(msg)) {
-        await this.handleCommit(msg)
-      } else if (message.isHandle(msg)) {
-        await this.handleUpdateHandle(msg)
-      } else if (message.isIdentity(msg)) {
-        await this.handleIdentityEvt(msg)
-      } else if (message.isAccount(msg)) {
-        await this.handleAccountEvt(msg)
-      } else if (message.isTombstone(msg)) {
-        // Ignore tombstones
-      } else if (message.isMigrate(msg)) {
-        // Ignore migrations
-      } else {
-        const exhaustiveCheck: never = msg
-        throw new Error(`Unhandled message type: ${exhaustiveCheck['$type']}`)
-      }
-    } catch (err) {
-      // We log messages we can't process and move on:
-      // otherwise the cursor would get stuck on a poison message.
-      log.error(
-        { err, message: loggableMessage(msg) },
-        'indexer message processing error',
-      )
-    } finally {
-      const latest = item.complete().at(-1)
-      if (latest !== undefined) {
-        this.cursor = latest
-      }
-    }
-  }
-
-  private async handleCommit(msg: message.Commit) {
-    const indexRecords = async () => {
-      const { root, rootCid, ops } = await getOps(msg)
-      if (msg.tooBig) {
-        await this.indexingSvc.indexRepo(msg.repo, rootCid.toString())
-        await this.indexingSvc.setCommitLastSeen(root, msg)
-        return
-      }
-      if (msg.rebase) {
-        const needsReindex =
-          await this.indexingSvc.checkCommitNeedsIndexing(root)
-        if (needsReindex) {
-          await this.indexingSvc.indexRepo(msg.repo, rootCid.toString())
-        }
-        await this.indexingSvc.setCommitLastSeen(root, msg)
-        return
-      }
-      for (const op of ops) {
-        if (op.action === WriteOpAction.Delete) {
-          await this.indexingSvc.deleteRecord(op.uri)
-        } else {
-          try {
-            await this.indexingSvc.indexRecord(
-              op.uri,
-              op.cid,
-              op.record,
-              op.action, // create or update
-              msg.time,
-            )
-          } catch (err) {
-            if (err instanceof ValidationError) {
-              log.warn(
-                {
-                  did: msg.repo,
-                  commit: msg.commit.toString(),
-                  uri: op.uri.toString(),
-                  cid: op.cid.toString(),
-                },
-                'skipping indexing of invalid record',
-              )
-            } else {
-              log.error(
-                {
-                  err,
-                  did: msg.repo,
-                  commit: msg.commit.toString(),
-                  uri: op.uri.toString(),
-                  cid: op.cid.toString(),
-                },
-                'skipping indexing due to error processing record',
-              )
-            }
-          }
-        }
-      }
-      await this.indexingSvc.setCommitLastSeen(root, msg)
-    }
-    const results = await Promise.allSettled([
-      indexRecords(),
-      this.indexingSvc.indexHandle(msg.repo, msg.time),
-    ])
-    handleAllSettledErrors(results)
-  }
-
-  private async handleUpdateHandle(msg: message.Handle) {
-    await this.indexingSvc.indexHandle(msg.did, msg.time, true)
-  }
-
-  private async handleIdentityEvt(msg: message.Identity) {
-    await this.indexingSvc.indexHandle(msg.did, msg.time, true)
-  }
-
-  private async handleAccountEvt(msg: message.Account) {
-    if (msg.active === false && msg.status === 'deleted') {
-      await this.indexingSvc.deleteActor(msg.did)
-    } else {
-      await this.indexingSvc.updateActorStatus(msg.did, msg.active, msg.status)
-    }
-  }
-
-  private getSubscription() {
-    return new Subscription({
-      service: this.opts.service,
-      method: ids.ComAtprotoSyncSubscribeRepos,
-      signal: this.ac.signal,
-      getParams: async () => {
-        return { cursor: this.cursor }
-      },
-      onReconnectError: (err, reconnects, initial) => {
-        log.warn({ err, reconnects, initial }, 'sub reconnect')
-      },
-      validate: (value) => {
-        try {
-          return lexicons.assertValidXrpcMessage<Message>(
-            ids.ComAtprotoSyncSubscribeRepos,
-            value,
-          )
-        } catch (err) {
-          log.warn(
-            {
-              err,
-              seq: ifNumber(value?.['seq']),
-              repo: ifString(value?.['repo']),
-              commit: ifString(value?.['commit']?.toString()),
-              time: ifString(value?.['time']),
-              provider: this.opts.service,
-            },
-            'ingester sub skipped invalid message',
-          )
-        }
-      },
-    })
-  }
-
-  async destroy() {
-    this.ac.abort()
-    await this.running
-    await this.repoQueue.destroy()
-    await this.background.processAll()
-  }
-}
-
-type Envelope = {
-  repo: string
-  message: ProcessableMessage
-}
-
-function ifString(val: unknown): string | undefined {
-  return typeof val === 'string' ? val : undefined
-}
-
-function ifNumber(val: unknown): number | undefined {
-  return typeof val === 'number' ? val : undefined
-}
-
-function getMessageDetails(msg: Message):
-  | { info: message.Info | null }
-  | {
-      seq: number
-      repo: string
-      message: ProcessableMessage
-    } {
-  if (message.isCommit(msg)) {
-    return { seq: msg.seq, repo: msg.repo, message: msg }
-  } else if (message.isHandle(msg)) {
-    return { seq: msg.seq, repo: msg.did, message: msg }
-  } else if (message.isIdentity(msg)) {
-    return { seq: msg.seq, repo: msg.did, message: msg }
-  } else if (message.isAccount(msg)) {
-    return { seq: msg.seq, repo: msg.did, message: msg }
-  } else if (message.isMigrate(msg)) {
-    return { seq: msg.seq, repo: msg.did, message: msg }
-  } else if (message.isTombstone(msg)) {
-    return { seq: msg.seq, repo: msg.did, message: msg }
-  } else if (message.isInfo(msg)) {
-    return { info: msg }
-  }
-  return { info: null }
-}
-
-async function getOps(
-  msg: message.Commit,
-): Promise<{ root: Commit; rootCid: CID; ops: PreparedWrite[] }> {
-  const car = await readCarWithRoot(msg.blocks as Uint8Array)
-  const rootBytes = car.blocks.get(car.root)
-  assert(rootBytes, 'Missing commit block in car slice')
-
-  const root = def.commit.schema.parse(cborDecode(rootBytes))
-  const ops: PreparedWrite[] = msg.ops.map((op) => {
-    const [collection, rkey] = op.path.split('/')
-    assert(collection && rkey)
-    if (
-      op.action === WriteOpAction.Create ||
-      op.action === WriteOpAction.Update
-    ) {
-      assert(op.cid)
-      const record = car.blocks.get(op.cid)
-      assert(record)
-      return {
-        action:
-          op.action === WriteOpAction.Create
-            ? WriteOpAction.Create
-            : WriteOpAction.Update,
-        cid: op.cid,
-        record: cborToLexRecord(record),
-        blobs: [],
-        uri: AtUri.make(msg.repo, collection, rkey),
-      }
-    } else if (op.action === WriteOpAction.Delete) {
-      return {
-        action: WriteOpAction.Delete,
-        uri: AtUri.make(msg.repo, collection, rkey),
-      }
-    } else {
-      throw new Error(`Unknown repo op action: ${op.action}`)
-    }
-  })
-
-  return { root, rootCid: car.root, ops }
-}
-
-type PreparedCreate = {
-  action: WriteOpAction.Create
-  uri: AtUri
-  cid: CID
-  record: Record<string, unknown>
-  blobs: CID[] // differs from similar type in pds
-}
-
-type PreparedUpdate = {
-  action: WriteOpAction.Update
-  uri: AtUri
-  cid: CID
-  record: Record<string, unknown>
-  blobs: CID[] // differs from similar type in pds
-}
-
-type PreparedDelete = {
-  action: WriteOpAction.Delete
-  uri: AtUri
-}
-
-type PreparedWrite = PreparedCreate | PreparedUpdate | PreparedDelete

package/src/data-plane/server/subscription/util.ts

@@ -1,156 +0,0 @@
-import assert from 'node:assert'
-import PQueue from 'p-queue'
-import { OutputSchema as RepoMessage } from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-import * as message from '../../../lexicon/types/com/atproto/sync/subscribeRepos'
-
-// A queue with arbitrarily many partitions, each processing work sequentially.
-// Partitions are created lazily and taken out of memory when they go idle.
-export class PartitionedQueue {
-  main: PQueue
-  partitions = new Map<string, PQueue>()
-
-  constructor(opts: { concurrency: number }) {
-    this.main = new PQueue({ concurrency: opts.concurrency })
-  }
-
-  async add(partitionId: string, task: () => Promise<void>) {
-    if (this.main.isPaused) return
-    return this.main.add(() => {
-      return this.getPartition(partitionId).add(task)
-    })
-  }
-
-  async destroy() {
-    this.main.pause()
-    this.main.clear()
-    this.partitions.forEach((p) => p.clear())
-    await this.main.onIdle() // All in-flight work completes
-  }
-
-  private getPartition(partitionId: string) {
-    let partition = this.partitions.get(partitionId)
-    if (!partition) {
-      partition = new PQueue({ concurrency: 1 })
-      partition.once('idle', () => this.partitions.delete(partitionId))
-      this.partitions.set(partitionId, partition)
-    }
-    return partition
-  }
-}
-
-export class LatestQueue {
-  queue = new PQueue({ concurrency: 1 })
-
-  async add(task: () => Promise<void>) {
-    if (this.queue.isPaused) return
-    this.queue.clear() // Only queue the latest task, invalidate any previous ones
-    return this.queue.add(task)
-  }
-
-  async destroy() {
-    this.queue.pause()
-    this.queue.clear()
-    await this.queue.onIdle() // All in-flight work completes
-  }
-}
-
-/**
- * Add items to a list, and mark those items as
- * completed. Upon item completion, get list of consecutive
- * items completed at the head of the list. Example:
- *
- * const consecutive = new ConsecutiveList<number>()
- * const item1 = consecutive.push(1)
- * const item2 = consecutive.push(2)
- * const item3 = consecutive.push(3)
- * item2.complete() // []
- * item1.complete() // [1, 2]
- * item3.complete() // [3]
- *
- */
-export class ConsecutiveList<T> {
-  list: ConsecutiveItem<T>[] = []
-
-  push(value: T) {
-    const item = new ConsecutiveItem<T>(this, value)
-    this.list.push(item)
-    return item
-  }
-
-  complete(): T[] {
-    let i = 0
-    while (this.list[i]?.isComplete) {
-      i += 1
-    }
-    return this.list.splice(0, i).map((item) => item.value)
-  }
-}
-
-export class ConsecutiveItem<T> {
-  isComplete = false
-  constructor(
-    private consecutive: ConsecutiveList<T>,
-    public value: T,
-  ) {}
-
-  complete() {
-    this.isComplete = true
-    return this.consecutive.complete()
-  }
-}
-
-export class PerfectMap<K, V> extends Map<K, V> {
-  get(key: K): V {
-    const val = super.get(key)
-    assert(val !== undefined, `Key not found in PerfectMap: ${key}`)
-    return val
-  }
-}
-
-// These are the message types that have a sequence number and a repo
-export type ProcessableMessage =
-  | message.Commit
-  | message.Handle
-  | message.Identity
-  | message.Migrate
-  | message.Tombstone
-
-export function loggableMessage(msg: RepoMessage) {
-  if (message.isCommit(msg)) {
-    const { seq, rebase, prev, repo, commit, time, tooBig, blobs } = msg
-    return {
-      $type: msg.$type,
-      seq,
-      rebase,
-      prev: prev?.toString(),
-      repo,
-      commit: commit.toString(),
-      time,
-      tooBig,
-      hasBlobs: blobs.length > 0,
-    }
-  } else if (message.isHandle(msg)) {
-    return msg
-  } else if (message.isIdentity(msg)) {
-    return msg
-  } else if (message.isAccount(msg)) {
-    return msg
-  } else if (message.isMigrate(msg)) {
-    return msg
-  } else if (message.isTombstone(msg)) {
-    return msg
-  } else if (message.isInfo(msg)) {
-    return msg
-  }
-  return msg
-}
-
-export function jitter(maxMs) {
-  return Math.round((Math.random() - 0.5) * maxMs * 2)
-}
-
-export function strToInt(str: string) {
-  const int = parseInt(str, 10)
-  assert(!isNaN(int), 'string could not be parsed to an integer')
-  return int
-}

package/tests/data-plane/subscription/util.test.ts

@@ -1,185 +0,0 @@
-import { wait } from '@atproto/common'
-import { randomStr } from '@atproto/crypto'
-import {
-  ConsecutiveList,
-  LatestQueue,
-  PartitionedQueue,
-} from '../../../src/data-plane/server/subscription/util'
-
-describe('subscription utils', () => {
-  describe('ConsecutiveList', () => {
-    it('tracks consecutive complete items.', () => {
-      const consecutive = new ConsecutiveList<number>()
-      // add items
-      const item1 = consecutive.push(1)
-      const item2 = consecutive.push(2)
-      const item3 = consecutive.push(3)
-      expect(item1.isComplete).toEqual(false)
-      expect(item2.isComplete).toEqual(false)
-      expect(item3.isComplete).toEqual(false)
-      // complete items out of order
-      expect(consecutive.list.length).toBe(3)
-      expect(item2.complete()).toEqual([])
-      expect(item2.isComplete).toEqual(true)
-      expect(consecutive.list.length).toBe(3)
-      expect(item1.complete()).toEqual([1, 2])
-      expect(item1.isComplete).toEqual(true)
-      expect(consecutive.list.length).toBe(1)
-      expect(item3.complete()).toEqual([3])
-      expect(consecutive.list.length).toBe(0)
-      expect(item3.isComplete).toEqual(true)
-    })
-  })
-
-  describe('LatestQueue', () => {
-    it('only performs most recently queued item.', async () => {
-      const latest = new LatestQueue()
-      const complete: number[] = []
-      latest.add(async () => {
-        await wait(1)
-        complete.push(1)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(2)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(3)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(4)
-      })
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1, 4]) // skip 2, 3
-      latest.add(async () => {
-        await wait(1)
-        complete.push(5)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(6)
-      })
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1, 4, 5, 6])
-    })
-
-    it('stops processing queued messages on destroy.', async () => {
-      const latest = new LatestQueue()
-      const complete: number[] = []
-      latest.add(async () => {
-        await wait(1)
-        complete.push(1)
-      })
-      latest.add(async () => {
-        await wait(1)
-        complete.push(2)
-      })
-      const destroyed = latest.destroy()
-      latest.add(async () => {
-        await wait(1)
-        complete.push(3)
-      })
-      await destroyed
-      expect(complete).toEqual([1]) // 2 was cleared, 3 was after destroy
-      // show that waiting on destroyed above was already enough to reflect all complete items
-      await latest.queue.onIdle()
-      expect(complete).toEqual([1])
-    })
-  })
-
-  describe('PartitionedQueue', () => {
-    it('performs work in parallel across partitions, serial within a partition.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: Infinity })
-      const complete: number[] = []
-      // partition 1 items start slow but get faster: slow should still complete first.
-      partitioned.add('1', async () => {
-        await wait(30)
-        complete.push(11)
-      })
-      partitioned.add('1', async () => {
-        await wait(20)
-        complete.push(12)
-      })
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(13)
-      })
-      expect(partitioned.partitions.size).toEqual(1)
-      // partition 2 items complete quickly except the last, which is slowest of all events.
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(21)
-      })
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(22)
-      })
-      partitioned.add('2', async () => {
-        await wait(1)
-        complete.push(23)
-      })
-      partitioned.add('2', async () => {
-        await wait(60)
-        complete.push(24)
-      })
-      expect(partitioned.partitions.size).toEqual(2)
-      await partitioned.main.onIdle()
-      expect(complete).toEqual([21, 22, 23, 11, 12, 13, 24])
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-
-    it('limits overall concurrency.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: 1 })
-      const complete: number[] = []
-      // if concurrency were not constrained, partition 1 would complete all items
-      // before any items from partition 2. since it is constrained, the work is complete in the order added.
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(11)
-      })
-      partitioned.add('2', async () => {
-        await wait(10)
-        complete.push(21)
-      })
-      partitioned.add('1', async () => {
-        await wait(1)
-        complete.push(12)
-      })
-      partitioned.add('2', async () => {
-        await wait(10)
-        complete.push(22)
-      })
-      // only partition 1 exists so far due to the concurrency
-      expect(partitioned.partitions.size).toEqual(1)
-      await partitioned.main.onIdle()
-      expect(complete).toEqual([11, 21, 12, 22])
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-
-    it('settles with many items.', async () => {
-      const partitioned = new PartitionedQueue({ concurrency: 100 })
-      const complete: { partition: string; id: number }[] = []
-      const partitions = new Set<string>()
-      for (let i = 0; i < 500; ++i) {
-        const partition = randomStr(1, 'base16').slice(0, 1)
-        partitions.add(partition)
-        partitioned.add(partition, async () => {
-          await wait((i % 2) * 2)
-          complete.push({ partition, id: i })
-        })
-      }
-      expect(partitioned.partitions.size).toEqual(partitions.size)
-      await partitioned.main.onIdle()
-      expect(complete.length).toEqual(500)
-      for (const partition of partitions) {
-        const ids = complete
-          .filter((item) => item.partition === partition)
-          .map((item) => item.id)
-        expect(ids).toEqual([...ids].sort((a, b) => a - b))
-      }
-      expect(partitioned.partitions.size).toEqual(0)
-    })
-  })
-})