@atproto/ozone 0.2.8 → 0.2.11

package/src/scheduled-action/service.ts

@@ -1,281 +0,0 @@
-import { Selectable } from 'kysely'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { ScheduledActionStatus, ScheduledActionType } from '../api/util.js'
-import { Database } from '../db/index.js'
-import { ScheduledAction } from '../db/schema/scheduled-action.js'
-import { ScheduledActionView } from '../lexicon/types/tools/ozone/moderation/defs.js'
-import { dbLogger } from '../logger.js'
-import { SchedulingParams } from './types.js'
-
-export type ScheduledActionServiceCreator = (
-  db: Database,
-) => ScheduledActionService
-
-export class ScheduledActionService {
-  constructor(public db: Database) {}
-
-  static creator() {
-    return (db: Database) => new ScheduledActionService(db)
-  }
-
-  formatScheduledAction(
-    action: Selectable<ScheduledAction>,
-  ): ScheduledActionView {
-    return {
-      id: action.id,
-      action: action.action,
-      eventData: action.eventData as { [x: string]: unknown } | undefined,
-      did: action.did,
-      executeAt: action.executeAt
-        ? new Date(action.executeAt).toISOString()
-        : undefined,
-      executeAfter: action.executeAfter
-        ? new Date(action.executeAfter).toISOString()
-        : undefined,
-      executeUntil: action.executeUntil
-        ? new Date(action.executeUntil).toISOString()
-        : undefined,
-      randomizeExecution: action.randomizeExecution,
-      createdBy: action.createdBy,
-      createdAt: new Date(action.createdAt).toISOString(),
-      updatedAt: new Date(action.updatedAt).toISOString(),
-      status: action.status,
-      lastExecutedAt: action.lastExecutedAt
-        ? new Date(action.lastExecutedAt).toISOString()
-        : undefined,
-      lastFailureReason: action.lastFailureReason || undefined,
-      executionEventId: action.executionEventId || undefined,
-    }
-  }
-
-  async scheduleAction(
-    schedulingParams: SchedulingParams,
-  ): Promise<Selectable<ScheduledAction>> {
-    const { action, eventData, did, createdBy } = schedulingParams
-
-    // Only allow one pending action at a time for a given subject and action type
-    const existingAction = await this.getPendingActionForSubject(did, action)
-    if (existingAction) {
-      throw new InvalidRequestError(
-        'A pending scheduled action already exists for this subject',
-        'ActionAlreadyExists',
-      )
-    }
-
-    // When a time-range for action is specified, ensure that the range is valid
-    if (
-      'executeAfter' in schedulingParams &&
-      schedulingParams.executeAfter &&
-      schedulingParams.executeUntil &&
-      schedulingParams.executeAfter >= schedulingParams.executeUntil
-    ) {
-      throw new InvalidRequestError(
-        'executeAfter must be before executeUntil',
-        'InvalidScheduling',
-      )
-    }
-
-    const now = new Date().toISOString()
-    const randomizeExecution =
-      !('executeAt' in schedulingParams) && 'executeAfter' in schedulingParams
-
-    const scheduledAction = await this.db.db
-      .insertInto('scheduled_action')
-      .values({
-        action,
-        eventData: JSON.stringify(eventData),
-        did,
-        executeAt: randomizeExecution
-          ? null
-          : schedulingParams.executeAt?.toISOString(),
-        executeAfter: randomizeExecution
-          ? schedulingParams.executeAfter?.toISOString()
-          : null,
-        executeUntil: randomizeExecution
-          ? schedulingParams.executeUntil?.toISOString()
-          : null,
-        randomizeExecution,
-        createdBy,
-        createdAt: now,
-        updatedAt: now,
-        status: 'pending',
-      })
-      .returningAll()
-      .executeTakeFirstOrThrow()
-
-    return scheduledAction
-  }
-
-  async getPendingActionForSubject(
-    did: string,
-    action: ScheduledActionType,
-  ): Promise<Selectable<ScheduledAction> | null> {
-    const scheduledAction = await this.db.db
-      .selectFrom('scheduled_action')
-      .selectAll()
-      .where('did', '=', did)
-      .where('action', '=', action)
-      .where('status', '=', 'pending')
-      .executeTakeFirst()
-
-    return scheduledAction || null
-  }
-
-  async listScheduledActions({
-    cursor,
-    limit = 50,
-    startTime,
-    endTime,
-    subjects,
-    statuses = [],
-    direction = 'desc',
-  }: {
-    cursor?: string
-    limit?: number
-    startTime?: Date
-    endTime?: Date
-    subjects?: string[]
-    statuses: ScheduledActionStatus[]
-    direction?: 'asc' | 'desc'
-  }): Promise<{
-    actions: Selectable<ScheduledAction>[]
-    cursor?: string
-  }> {
-    let query = this.db.db
-      .selectFrom('scheduled_action')
-      .where('status', 'in', statuses)
-      .selectAll()
-
-    if (subjects && subjects.length > 0) {
-      query = query.where('did', 'in', subjects)
-    }
-
-    if (startTime) {
-      query = query.where((eb) =>
-        eb.or([
-          eb('executeAt', '>=', startTime.toISOString()),
-          eb('executeAfter', '>=', startTime.toISOString()),
-        ]),
-      )
-    }
-
-    if (endTime) {
-      query = query.where((eb) =>
-        eb.or([
-          eb('executeAt', '<=', endTime.toISOString()),
-          eb('executeUntil', '<=', endTime.toISOString()),
-          eb.and([
-            eb('executeUntil', 'is', null),
-            eb('executeAfter', '<=', endTime.toISOString()),
-          ]),
-        ]),
-      )
-    }
-
-    if (cursor) {
-      query = query.where(
-        'id',
-        direction === 'asc' ? '>' : '<',
-        parseInt(cursor, 10),
-      )
-    }
-
-    const actions = await query.orderBy('id', direction).limit(limit).execute()
-
-    return {
-      actions,
-      cursor: actions.at(-1)?.id?.toString(),
-    }
-  }
-
-  async cancelScheduledActions(subjects: string[]): Promise<{
-    succeeded: string[]
-    failed: { did: string; error: string; errorCode?: string }[]
-  }> {
-    const succeeded: string[] = []
-    const failed: { did: string; error: string; errorCode?: string }[] = []
-
-    for (const did of subjects) {
-      try {
-        const result = await this.db.db
-          .updateTable('scheduled_action')
-          .set({
-            status: 'cancelled',
-            updatedAt: new Date().toISOString(),
-          })
-          .where('did', '=', did)
-          .where('status', '=', 'pending')
-          .executeTakeFirst()
-
-        if (result.numUpdatedRows && result.numUpdatedRows > 0) {
-          succeeded.push(did)
-        } else {
-          failed.push({
-            did,
-            error: 'No pending scheduled actions found for subject',
-            errorCode: 'NoPendingActions',
-          })
-        }
-      } catch (err) {
-        dbLogger.error({ err, subjects }, 'Error cancelling scheduled action')
-        failed.push({
-          did,
-          error: 'Unknown error',
-          errorCode: 'DatabaseError',
-        })
-      }
-    }
-
-    return { succeeded, failed }
-  }
-
-  async getPendingActionsToExecute(
-    now: Date,
-  ): Promise<Selectable<ScheduledAction>[]> {
-    return await this.db.db
-      .selectFrom('scheduled_action')
-      .selectAll()
-      .where('status', '=', 'pending')
-      .where((eb) =>
-        eb.or([
-          eb('executeAfter', '<=', now.toISOString()),
-          eb('executeAt', '<=', now.toISOString()),
-        ]),
-      )
-      .execute()
-  }
-
-  async markActionAsExecuted(
-    actionId: number,
-    executionEventId: number,
-  ): Promise<void> {
-    const now = new Date().toISOString()
-    await this.db.db
-      .updateTable('scheduled_action')
-      .set({
-        status: 'executed',
-        lastExecutedAt: now,
-        executionEventId,
-        updatedAt: now,
-      })
-      .where('id', '=', actionId)
-      .execute()
-  }
-
-  async markActionAsFailed(
-    actionId: number,
-    failureReason: string,
-  ): Promise<void> {
-    const now = new Date().toISOString()
-    await this.db.db
-      .updateTable('scheduled_action')
-      .set({
-        status: 'failed',
-        lastExecutedAt: now,
-        lastFailureReason: failureReason,
-        updatedAt: now,
-      })
-      .where('id', '=', actionId)
-      .execute()
-  }
-}

package/src/scheduled-action/types.ts

@@ -1,17 +0,0 @@
-import { ScheduledActionType } from '../api/util.js'
-
-export type ExecutionSchedule =
-  | {
-      executeAt: Date
-    }
-  | {
-      executeAfter: Date
-      executeUntil?: Date
-    }
-
-export type SchedulingParams = {
-  action: ScheduledActionType
-  eventData: unknown
-  did: string
-  createdBy: string
-} & ExecutionSchedule

package/src/sequencer/index.ts

@@ -1,2 +0,0 @@
-export * from './sequencer.js'
-export * from './outbox.js'

package/src/sequencer/outbox.ts

@@ -1,123 +0,0 @@
-import { AsyncBuffer, AsyncBufferFullError } from '@atproto/common'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { LabelsEvt, Sequencer } from './sequencer.js'
-
-export type OutboxOpts = {
-  maxBufferSize: number
-}
-
-export class Outbox {
-  private caughtUp = false
-  lastSeen = -1
-
-  cutoverBuffer: LabelsEvt[]
-  outBuffer: AsyncBuffer<LabelsEvt>
-
-  constructor(
-    public sequencer: Sequencer,
-    opts: Partial<OutboxOpts> = {},
-  ) {
-    const { maxBufferSize = 500 } = opts
-    this.cutoverBuffer = []
-    this.outBuffer = new AsyncBuffer<LabelsEvt>(maxBufferSize)
-  }
-
-  // event stream occurs in 3 phases
-  // 1. backfill events: events that have been added to the DB since the last time a connection was open.
-  // The outbox is not yet listening for new events from the sequencer
-  // 2. cutover: the outbox has caught up with where the sequencer purports to be,
-  // but the sequencer might already be halfway through sending out a round of updates.
-  // Therefore, we start accepting the sequencer's events in a buffer, while making our own request to the
-  // database to ensure we're caught up. We then dedupe the query & the buffer & stream the events in order
-  // 3. streaming: we're all caught up on historic state, so the sequencer outputs events and we
-  // immediately yield them
-  async *events(
-    backfillCursor?: number,
-    signal?: AbortSignal,
-  ): AsyncGenerator<LabelsEvt> {
-    // catch up as much as we can
-    if (backfillCursor !== undefined) {
-      for await (const evt of this.getBackfill(backfillCursor)) {
-        if (signal?.aborted) return
-        this.lastSeen = evt.seq
-        yield evt
-      }
-    } else {
-      // if not backfill, we don't need to cutover, just start streaming
-      this.caughtUp = true
-    }
-
-    // streams updates from sequencer, but buffers them for cutover as it makes a last request
-
-    const addToBuffer = (evts) => {
-      if (this.caughtUp) {
-        this.outBuffer.pushMany(evts)
-      } else {
-        this.cutoverBuffer = [...this.cutoverBuffer, ...evts]
-      }
-    }
-
-    if (!signal?.aborted) {
-      this.sequencer.on('events', addToBuffer)
-    }
-    signal?.addEventListener('abort', () =>
-      this.sequencer.off('events', addToBuffer),
-    )
-
-    const cutover = async () => {
-      // only need to perform cutover if we've been backfilling
-      if (backfillCursor !== undefined) {
-        const cutoverEvts = await this.sequencer.requestLabelRange({
-          earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
-        })
-        this.outBuffer.pushMany(cutoverEvts)
-        // dont worry about dupes, we ensure order on yield
-        this.outBuffer.pushMany(this.cutoverBuffer)
-        this.caughtUp = true
-        this.cutoverBuffer = []
-      } else {
-        this.caughtUp = true
-      }
-    }
-    cutover()
-
-    while (true) {
-      try {
-        for await (const evt of this.outBuffer.events()) {
-          if (signal?.aborted) return
-          if (evt.seq > this.lastSeen) {
-            this.lastSeen = evt.seq
-            yield evt
-          }
-        }
-      } catch (err) {
-        if (err instanceof AsyncBufferFullError) {
-          throw new InvalidRequestError(
-            'Stream consumer too slow',
-            'ConsumerTooSlow',
-          )
-        } else {
-          throw err
-        }
-      }
-    }
-  }
-
-  // yields only historical events
-  async *getBackfill(backfillCursor: number) {
-    const PAGE_SIZE = 500
-    while (true) {
-      const evts = await this.sequencer.requestLabelRange({
-        earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
-        limit: PAGE_SIZE,
-      })
-      for (const evt of evts) {
-        yield evt
-      }
-      // if we're within half a pagesize of the sequencer, we call it good & switch to cutover
-      const seqCursor = this.sequencer.lastSeen ?? -1
-      if (seqCursor - this.lastSeen < PAGE_SIZE / 2) break
-      if (evts.length < 1) break
-    }
-  }
-}

package/src/sequencer/sequencer.ts

@@ -1,147 +0,0 @@
-import EventEmitter from 'node:events'
-import { Selectable } from 'kysely'
-// eslint-disable-next-line import/default
-import pg from 'pg'
-type PoolClient = pg.PoolClient
-import type TypedEmitter from 'typed-emitter'
-import { Database } from '../db/index.js'
-import { Label as LabelTable, LabelChannel } from '../db/schema/label.js'
-import { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels.js'
-import { seqLogger as log } from '../logger.js'
-import { ModerationService } from '../mod-service/index.js'
-
-export type { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels.js'
-type LabelRow = Selectable<LabelTable>
-
-export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
-  db: Database
-  destroyed = false
-  pollPromise: Promise<void> | undefined
-  queued = false
-  conn: PoolClient | undefined
-
-  constructor(
-    public modSrvc: ModerationService,
-    public lastSeen = 0,
-  ) {
-    super()
-    this.db = modSrvc.db
-    // note: this does not err when surpassed, just prints a warning to stderr
-    this.setMaxListeners(100)
-  }
-
-  async start() {
-    const curr = await this.curr()
-    this.lastSeen = curr ?? 0
-    this.poll()
-    this.conn = await this.db.pool.connect()
-    this.conn.query(`listen ${LabelChannel}`) // if this errors, unhandled rejection should cause process to exit
-    this.conn.on('notification', (notif) => {
-      if (notif.channel === LabelChannel) {
-        this.poll()
-      }
-    })
-  }
-
-  async destroy() {
-    if (this.destroyed) return
-    this.destroyed = true
-    if (this.conn) {
-      this.conn.release()
-      this.conn = undefined
-    }
-    if (this.pollPromise) {
-      await this.pollPromise
-    }
-    this.emit('close')
-  }
-
-  async curr(): Promise<number | null> {
-    const got = await this.db.db
-      .selectFrom('label')
-      .selectAll()
-      .orderBy('id', 'desc')
-      .limit(1)
-      .executeTakeFirst()
-    return got?.id ?? null
-  }
-
-  async next(cursor: number): Promise<LabelRow | null> {
-    const got = await this.db.db
-      .selectFrom('label')
-      .selectAll()
-      .where('id', '>', cursor)
-      .limit(1)
-      .orderBy('id', 'asc')
-      .executeTakeFirst()
-    return got || null
-  }
-
-  async requestLabelRange(opts: {
-    earliestId?: number
-    limit?: number
-  }): Promise<LabelsEvt[]> {
-    const { earliestId, limit } = opts
-
-    let seqQb = this.db.db.selectFrom('label').selectAll().orderBy('id', 'asc')
-    if (earliestId !== undefined) {
-      seqQb = seqQb.where('id', '>', earliestId)
-    }
-    if (limit !== undefined) {
-      seqQb = seqQb.limit(limit)
-    }
-
-    const rows = await seqQb.execute()
-    if (rows.length < 1) {
-      return []
-    }
-
-    const evts: LabelsEvt[] = await Promise.all(
-      rows.map(async (row) => {
-        const formatted = await this.modSrvc.views.formatLabelAndEnsureSig(row)
-        return { seq: row.id, labels: [formatted] }
-      }),
-    )
-
-    return evts
-  }
-
-  private poll() {
-    if (this.destroyed) return
-    if (this.pollPromise) {
-      this.queued = true
-      return
-    }
-    this.queued = false
-    this.pollPromise = this.requestLabelRange({
-      earliestId: this.lastSeen,
-      limit: 500,
-    })
-      .then((evts) => {
-        this.emit('events', evts)
-        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen
-        if (evts.length > 0) {
-          this.queued = true
-        }
-      })
-      .catch((err) => {
-        log.error(
-          { err, lastSeen: this.lastSeen },
-          'sequencer failed to poll db',
-        )
-      })
-      .finally(() => {
-        this.pollPromise = undefined
-        if (this.queued) {
-          this.poll()
-        }
-      })
-  }
-}
-
-type SequencerEvents = {
-  events: (evts: LabelsEvt[]) => void
-  close: () => void
-}
-
-export type SequencerEmitter = TypedEmitter.default<SequencerEvents>