@atproto/pds 0.5.11 → 0.5.14

package/src/sequencer/db/schema.ts

@@ -1,19 +0,0 @@
-import { Generated, GeneratedAlways, Insertable, Selectable } from 'kysely'
-
-export type RepoSeqEventType = 'append' | 'sync' | 'identity' | 'account'
-
-export interface RepoSeq {
-  seq: GeneratedAlways<number>
-  did: string
-  eventType: RepoSeqEventType
-  event: Uint8Array
-  invalidated: Generated<0 | 1>
-  sequencedAt: string
-}
-
-export type RepoSeqInsert = Insertable<RepoSeq>
-export type RepoSeqEntry = Selectable<RepoSeq>
-
-export type SequencerDbSchema = {
-  repo_seq: RepoSeq
-}

package/src/sequencer/events.ts

@@ -1,199 +0,0 @@
-import assert from 'node:assert'
-import { z } from 'zod'
-import { schema } from '@atproto/common'
-import {
-  DatetimeString,
-  DidString,
-  HandleString,
-  isDidString,
-  isHandleString,
-} from '@atproto/lex'
-import { encode as cborEncode } from '@atproto/lex-cbor'
-import { BlockMap, blocksToCarFile } from '@atproto/repo'
-import { AccountStatus } from '../account-manager/account-manager.js'
-import { CommitDataWithOps, SyncEvtData } from '../repo/index.js'
-import { RepoSeqInsert } from './db/index.js'
-
-export const formatSeqCommit = async (
-  did: string,
-  commitData: CommitDataWithOps,
-): Promise<RepoSeqInsert> => {
-  const blocksToSend = new BlockMap()
-  blocksToSend.addMap(commitData.newBlocks)
-  blocksToSend.addMap(commitData.relevantBlocks)
-
-  const evt = {
-    repo: did,
-    commit: commitData.cid,
-    rev: commitData.rev,
-    since: commitData.since,
-    blocks: await blocksToCarFile(commitData.cid, blocksToSend),
-    ops: commitData.ops,
-    prevData: commitData.prevData ?? undefined,
-    // deprecated (but still required) fields
-    rebase: false,
-    tooBig: false,
-    blobs: [],
-  }
-
-  return {
-    did,
-    eventType: 'append' as const,
-    event: cborEncode(evt),
-    sequencedAt: new Date().toISOString(),
-  }
-}
-
-export const formatSeqSyncEvt = async (
-  did: DidString,
-  data: SyncEvtData,
-): Promise<RepoSeqInsert> => {
-  const blocks = await blocksToCarFile(data.cid, data.blocks)
-  const evt: SyncEvt = {
-    did,
-    rev: data.rev,
-    blocks,
-  }
-  return {
-    did,
-    eventType: 'sync',
-    event: cborEncode(evt),
-    sequencedAt: new Date().toISOString(),
-  }
-}
-
-export const syncEvtDataFromCommit = (
-  commitData: CommitDataWithOps,
-): SyncEvtData => {
-  const { blocks, missing } = commitData.relevantBlocks.getMany([
-    commitData.cid,
-  ])
-  assert(
-    !missing.length,
-    'commit block was not found, could not build sync event',
-  )
-  return {
-    rev: commitData.rev,
-    cid: commitData.cid,
-    blocks,
-  }
-}
-
-export const formatSeqIdentityEvt = async (
-  did: DidString,
-  handle?: HandleString,
-): Promise<RepoSeqInsert> => {
-  const evt: IdentityEvt = handle //
-    ? { did, handle }
-    : { did }
-
-  return {
-    did,
-    eventType: 'identity',
-    event: cborEncode(evt),
-    sequencedAt: new Date().toISOString(),
-  }
-}
-
-export const formatSeqAccountEvt = async (
-  did: DidString,
-  status: AccountStatus,
-): Promise<RepoSeqInsert> => {
-  const evt: AccountEvt =
-    status === AccountStatus.Active
-      ? { did, active: true }
-      : { did, active: false, status }
-
-  return {
-    did,
-    eventType: 'account',
-    event: cborEncode(evt),
-    sequencedAt: new Date().toISOString(),
-  }
-}
-
-export const commitEvtOp = z.object({
-  action: z.union([
-    z.literal('create'),
-    z.literal('update'),
-    z.literal('delete'),
-  ]),
-  // @TODO should we validate that the "path" is a valid "<nsid>/<rkey>" ?
-  path: z.string(),
-  cid: schema.cid.nullable(),
-  prev: schema.cid.optional(),
-})
-export type CommitEvtOp = z.infer<typeof commitEvtOp>
-
-// @TODO This runtime code is only used to generate "types". We should either
-// make use of it or only use types.
-export const commitEvt = z.object({
-  rebase: z.boolean(),
-  tooBig: z.boolean(),
-  repo: z.string().refine(isDidString),
-  commit: schema.cid,
-  rev: z.string(),
-  since: z.string().nullable(),
-  blocks: schema.bytes,
-  ops: z.array(commitEvtOp),
-  blobs: z.array(schema.cid),
-  prevData: schema.cid.optional(),
-})
-export type CommitEvt = z.infer<typeof commitEvt>
-
-export const syncEvt = z.object({
-  did: z.string().refine(isDidString),
-  blocks: schema.bytes,
-  rev: z.string(),
-})
-export type SyncEvt = z.infer<typeof syncEvt>
-
-export const identityEvt = z.object({
-  did: z.string().refine(isDidString),
-  handle: z.string().refine(isHandleString).optional(),
-})
-export type IdentityEvt = z.infer<typeof identityEvt>
-
-export const accountEvt = z.object({
-  did: z.string().refine(isDidString),
-  active: z.boolean(),
-  status: z
-    .enum([
-      AccountStatus.Takendown,
-      AccountStatus.Suspended,
-      AccountStatus.Deleted,
-      AccountStatus.Deactivated,
-    ])
-    .optional(),
-})
-export type AccountEvt = z.infer<typeof accountEvt>
-
-type TypedCommitEvt = {
-  type: 'commit'
-  seq: number
-  time: DatetimeString
-  evt: CommitEvt
-}
-type TypedSyncEvt = {
-  type: 'sync'
-  seq: number
-  time: DatetimeString
-  evt: SyncEvt
-}
-type TypedIdentityEvt = {
-  type: 'identity'
-  seq: number
-  time: DatetimeString
-  evt: IdentityEvt
-}
-type TypedAccountEvt = {
-  type: 'account'
-  seq: number
-  time: DatetimeString
-  evt: AccountEvt
-}
-export type SeqEvt =
-  | TypedCommitEvt
-  | TypedSyncEvt
-  | TypedIdentityEvt
-  | TypedAccountEvt

package/src/sequencer/index.ts

@@ -1,3 +0,0 @@
-export * from './sequencer.js'
-export * from './outbox.js'
-export * from './events.js'

package/src/sequencer/outbox.ts

@@ -1,123 +0,0 @@
-import { AsyncBuffer, AsyncBufferFullError } from '@atproto/common'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { SeqEvt, Sequencer } from './index.js'
-
-export type OutboxOpts = {
-  maxBufferSize: number
-}
-
-export class Outbox {
-  private caughtUp = false
-  lastSeen = -1
-
-  cutoverBuffer: SeqEvt[]
-  outBuffer: AsyncBuffer<SeqEvt>
-
-  constructor(
-    public sequencer: Sequencer,
-    opts: Partial<OutboxOpts> = {},
-  ) {
-    const { maxBufferSize = 500 } = opts
-    this.cutoverBuffer = []
-    this.outBuffer = new AsyncBuffer<SeqEvt>(maxBufferSize)
-  }
-
-  // event stream occurs in 3 phases
-  // 1. backfill events: events that have been added to the DB since the last time a connection was open.
-  // The outbox is not yet listening for new events from the sequencer
-  // 2. cutover: the outbox has caught up with where the sequencer purports to be,
-  // but the sequencer might already be halfway through sending out a round of updates.
-  // Therefore, we start accepting the sequencer's events in a buffer, while making our own request to the
-  // database to ensure we're caught up. We then dedupe the query & the buffer & stream the events in order
-  // 3. streaming: we're all caught up on historic state, so the sequencer outputs events and we
-  // immediately yield them
-  async *events(
-    backfillCursor?: number,
-    signal?: AbortSignal,
-  ): AsyncGenerator<SeqEvt> {
-    // catch up as much as we can
-    if (backfillCursor !== undefined) {
-      for await (const evt of this.getBackfill(backfillCursor)) {
-        if (signal?.aborted) return
-        this.lastSeen = evt.seq
-        yield evt
-      }
-    } else {
-      // if not backfill, we don't need to cutover, just start streaming
-      this.caughtUp = true
-    }
-
-    // streams updates from sequencer, but buffers them for cutover as it makes a last request
-
-    const addToBuffer = (evts) => {
-      if (this.caughtUp) {
-        this.outBuffer.pushMany(evts)
-      } else {
-        this.cutoverBuffer = [...this.cutoverBuffer, ...evts]
-      }
-    }
-
-    if (!signal?.aborted) {
-      this.sequencer.on('events', addToBuffer)
-    }
-    signal?.addEventListener('abort', () =>
-      this.sequencer.off('events', addToBuffer),
-    )
-
-    const cutover = async () => {
-      // only need to perform cutover if we've been backfilling
-      if (backfillCursor !== undefined) {
-        const cutoverEvts = await this.sequencer.requestSeqRange({
-          earliestSeq: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
-        })
-        this.outBuffer.pushMany(cutoverEvts)
-        // dont worry about dupes, we ensure order on yield
-        this.outBuffer.pushMany(this.cutoverBuffer)
-        this.caughtUp = true
-        this.cutoverBuffer = []
-      } else {
-        this.caughtUp = true
-      }
-    }
-    cutover()
-
-    while (true) {
-      try {
-        for await (const evt of this.outBuffer.events()) {
-          if (signal?.aborted) return
-          if (evt.seq > this.lastSeen) {
-            this.lastSeen = evt.seq
-            yield evt
-          }
-        }
-      } catch (err) {
-        if (err instanceof AsyncBufferFullError) {
-          throw new InvalidRequestError(
-            'Stream consumer too slow',
-            'ConsumerTooSlow',
-          )
-        } else {
-          throw err
-        }
-      }
-    }
-  }
-
-  // yields only historical events
-  async *getBackfill(backfillCursor: number) {
-    const PAGE_SIZE = 500
-    while (true) {
-      const evts = await this.sequencer.requestSeqRange({
-        earliestSeq: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
-        limit: PAGE_SIZE,
-      })
-      for (const evt of evts) {
-        yield evt
-      }
-      // if we're within half a pagesize of the sequencer, we call it good & switch to cutover
-      const seqCursor = this.sequencer.lastSeen ?? -1
-      if (seqCursor - this.lastSeen < PAGE_SIZE / 2) break
-      if (evts.length < 1) break
-    }
-  }
-}

package/src/sequencer/sequencer.ts

@@ -1,290 +0,0 @@
-import EventEmitter from 'node:events'
-import type TypedEmitter from 'typed-emitter'
-import { SECOND, wait } from '@atproto/common'
-import { decode as cborDecode } from '@atproto/lex-cbor'
-import { DatetimeString, DidString, HandleString } from '@atproto/syntax'
-import { AccountStatus } from '../account-manager/helpers/account.js'
-import { Crawlers } from '../crawlers.js'
-import { seqLogger as log } from '../logger.js'
-import { CommitDataWithOps, SyncEvtData } from '../repo/index.js'
-import {
-  RepoSeqEntry,
-  RepoSeqInsert,
-  SequencerDb,
-  getDb,
-  getMigrator,
-} from './db/index.js'
-import {
-  AccountEvt,
-  CommitEvt,
-  IdentityEvt,
-  SeqEvt,
-  SyncEvt,
-  formatSeqAccountEvt,
-  formatSeqCommit,
-  formatSeqIdentityEvt,
-  formatSeqSyncEvt,
-  syncEvtDataFromCommit,
-} from './events.js'
-
-export * from './events.js'
-
-export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
-  db: SequencerDb
-  destroyed = false
-  pollPromise: Promise<void> | null = null
-  triesWithNoResults = 0
-
-  constructor(
-    public dbLocation: string,
-    public crawlers: Crawlers,
-    public lastSeen = 0,
-    disableWalAutoCheckpoint = false,
-  ) {
-    super()
-    // note: this does not err when surpassed, just prints a warning to stderr
-    this.setMaxListeners(100)
-    this.db = getDb(dbLocation, disableWalAutoCheckpoint)
-  }
-
-  async start() {
-    await this.db.ensureWal()
-    const migrator = getMigrator(this.db)
-    await migrator.migrateToLatestOrThrow()
-    const curr = await this.curr()
-    this.lastSeen = curr ?? 0
-    if (this.pollPromise === null) {
-      this.pollPromise = this.pollDb()
-    }
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.pollPromise) {
-      await this.pollPromise
-    }
-    this.emit('close')
-  }
-
-  async curr(): Promise<number | null> {
-    const got = await this.db.db
-      .selectFrom('repo_seq')
-      .selectAll()
-      .orderBy('seq', 'desc')
-      .limit(1)
-      .executeTakeFirst()
-    return got?.seq ?? null
-  }
-
-  async next(cursor: number): Promise<SeqRow | null> {
-    const got = await this.db.db
-      .selectFrom('repo_seq')
-      .selectAll()
-      .where('seq', '>', cursor)
-      .limit(1)
-      .orderBy('seq', 'asc')
-      .executeTakeFirst()
-    return got || null
-  }
-
-  async earliestAfterTime(time: string): Promise<SeqRow | null> {
-    const got = await this.db.db
-      .selectFrom('repo_seq')
-      .selectAll()
-      .where('sequencedAt', '>=', time)
-      .orderBy('sequencedAt', 'asc')
-      .limit(1)
-      .executeTakeFirst()
-    return got || null
-  }
-
-  async requestSeqRange(opts: {
-    earliestSeq?: number
-    latestSeq?: number
-    earliestTime?: string
-    limit?: number
-  }): Promise<SeqEvt[]> {
-    const { earliestSeq, latestSeq, earliestTime, limit } = opts
-
-    let seqQb = this.db.db
-      .selectFrom('repo_seq')
-      .selectAll()
-      .orderBy('seq', 'asc')
-      .where('invalidated', '=', 0)
-    if (earliestSeq !== undefined) {
-      seqQb = seqQb.where('seq', '>', earliestSeq)
-    }
-    if (latestSeq !== undefined) {
-      seqQb = seqQb.where('seq', '<=', latestSeq)
-    }
-    if (earliestTime !== undefined) {
-      seqQb = seqQb.where('sequencedAt', '>=', earliestTime)
-    }
-    if (limit !== undefined) {
-      seqQb = seqQb.limit(limit)
-    }
-
-    const rows = await seqQb.execute()
-    if (rows.length < 1) {
-      return []
-    }
-
-    return parseRepoSeqRows(rows)
-  }
-
-  private async pollDb(): Promise<void> {
-    if (this.destroyed) return
-    // if already polling, do not start another poll
-    try {
-      const evts = await this.requestSeqRange({
-        earliestSeq: this.lastSeen,
-        limit: 1000,
-      })
-      if (evts.length > 0) {
-        this.triesWithNoResults = 0
-        this.emit('events', evts)
-        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen
-      } else {
-        await this.exponentialBackoff()
-      }
-      this.pollPromise = this.pollDb()
-    } catch (err) {
-      log.error({ err, lastSeen: this.lastSeen }, 'sequencer failed to poll db')
-      await this.exponentialBackoff()
-      this.pollPromise = this.pollDb()
-    }
-  }
-
-  // when no results, exponential backoff on pulling, with a max of a second wait
-  private async exponentialBackoff(): Promise<void> {
-    this.triesWithNoResults++
-    const waitTime = Math.min(Math.pow(2, this.triesWithNoResults), SECOND)
-    await wait(waitTime)
-  }
-
-  async sequenceEvts(events: readonly RepoSeqInsert[]): Promise<number[]> {
-    if (!events.length) return []
-    const rows = await this.db.executeWithRetry(
-      this.db.db.insertInto('repo_seq').values(events).returning('seq'),
-    )
-    this.crawlers.notifyOfUpdate()
-    return rows.map((row) => row.seq)
-  }
-
-  public async sequenceCommit(
-    did: DidString,
-    commitData: CommitDataWithOps,
-  ): Promise<void> {
-    await this.sequenceEvts([await formatSeqCommit(did, commitData)])
-  }
-
-  public async sequenceSync(did: DidString, data: SyncEvtData): Promise<void> {
-    await this.sequenceEvts([await formatSeqSyncEvt(did, data)])
-  }
-
-  public async sequenceIdentity(
-    did: DidString,
-    handle?: HandleString,
-  ): Promise<void> {
-    await this.sequenceEvts([await formatSeqIdentityEvt(did, handle)])
-  }
-
-  public async sequenceAccount(
-    did: DidString,
-    status: AccountStatus,
-  ): Promise<void> {
-    await this.sequenceEvts([await formatSeqAccountEvt(did, status)])
-  }
-
-  public async sequenceAccountCreation(
-    did: DidString,
-    handle: HandleString,
-    commit: CommitDataWithOps,
-  ): Promise<void> {
-    // Atomically sequence all events
-    await this.sequenceEvts([
-      await formatSeqIdentityEvt(did, handle),
-      await formatSeqAccountEvt(did, AccountStatus.Active),
-      await formatSeqCommit(did, commit),
-      await formatSeqSyncEvt(did, syncEvtDataFromCommit(commit)),
-    ])
-  }
-
-  public async sequenceAccountActivation(
-    did: DidString,
-    handle: HandleString,
-    status: AccountStatus,
-    syncData: SyncEvtData,
-  ): Promise<void> {
-    // Atomically sequence all events
-    await this.sequenceEvts([
-      await formatSeqAccountEvt(did, status),
-      await formatSeqIdentityEvt(did, handle),
-      await formatSeqSyncEvt(did, syncData),
-    ])
-  }
-
-  public async sequenceAccountDeletion(did: DidString) {
-    const [seq] = await this.sequenceEvts([
-      await formatSeqAccountEvt(did, AccountStatus.Deleted),
-    ])
-    await this.db.executeWithRetry(
-      this.db.db
-        .deleteFrom('repo_seq')
-        .where('did', '=', did)
-        .where('seq', '!=', seq),
-    )
-  }
-}
-
-export const parseRepoSeqRows = (rows: RepoSeqEntry[]): SeqEvt[] => {
-  const seqEvts: SeqEvt[] = []
-  for (const row of rows) {
-    // should never hit this because of WHERE clause
-    if (row.seq === null) {
-      continue
-    }
-    const evt = cborDecode(row.event)
-    if (row.eventType === 'append') {
-      seqEvts.push({
-        type: 'commit',
-        seq: row.seq,
-        time: row.sequencedAt as DatetimeString,
-        evt: evt as CommitEvt,
-      })
-    } else if (row.eventType === 'sync') {
-      seqEvts.push({
-        type: 'sync',
-        seq: row.seq,
-        time: row.sequencedAt as DatetimeString,
-        evt: evt as SyncEvt,
-      })
-    } else if (row.eventType === 'identity') {
-      seqEvts.push({
-        type: 'identity',
-        seq: row.seq,
-        time: row.sequencedAt as DatetimeString,
-        evt: evt as IdentityEvt,
-      })
-    } else if (row.eventType === 'account') {
-      seqEvts.push({
-        type: 'account',
-        seq: row.seq,
-        time: row.sequencedAt as DatetimeString,
-        evt: evt as AccountEvt,
-      })
-    }
-  }
-  return seqEvts
-}
-
-type SeqRow = RepoSeqEntry
-
-type SequencerEvents = {
-  events: (evts: SeqEvt[]) => void
-  close: () => void
-}
-
-export type SequencerEmitter = TypedEmitter.default<SequencerEvents>
-
-export default Sequencer

package/src/util/compression.ts

@@ -1,16 +0,0 @@
-import compression from 'compression'
-import express from 'express'
-
-export default function () {
-  return compression({
-    filter,
-  })
-}
-
-function filter(_req: express.Request, res: express.Response) {
-  const contentType = res.getHeader('Content-type')
-  if (contentType === 'application/vnd.ipld.car') {
-    return true
-  }
-  return compression.filter(_req, res)
-}

package/src/util/debug.ts

@@ -1,10 +0,0 @@
-export const debugCatch = <Func extends (...args: any[]) => any>(fn: Func) => {
-  return async (...args: Parameters<Func>) => {
-    try {
-      return (await fn(...args)) as Awaited<ReturnType<Func>>
-    } catch (err) {
-      console.error(err)
-      throw err
-    }
-  }
-}

package/src/util/http.ts

@@ -1,31 +0,0 @@
-import { ServerResponse } from 'node:http'
-
-/**
- * Set or appends a value to the `Vary` header in the response, only if the
- * value is not already present.
- */
-export function appendVary(res: ServerResponse, value: string) {
-  if (!varyContains(res, value.toLowerCase())) {
-    res.appendHeader('Vary', value)
-  }
-}
-
-function varyContains(res: ServerResponse, searchValue: string) {
-  const headerValue = res.getHeader('Vary')
-  switch (typeof headerValue) {
-    case 'string':
-      return varyStringContains(headerValue, searchValue)
-    case 'object':
-      // headerValue is a string[] here
-      return headerValue.some((h) => varyStringContains(h, searchValue))
-    default:
-      return false
-  }
-}
-
-function varyStringContains(headerValue: string, searchValue: string): boolean {
-  return headerValue
-    .split(',')
-    .map((v) => v.trim().toLowerCase())
-    .some((v) => v === searchValue || v === `*`)
-}

package/src/util/types.ts

@@ -1,7 +0,0 @@
-export type Simplify<T> = {
-  [K in keyof T]: T[K]
-} & NonNullable<unknown>
-
-export type WithRequired<T, K extends keyof T> = Simplify<
-  Omit<T, K> & Required<Pick<T, K>>
->