@atproto/ozone 0.2.8 → 0.2.11

package/src/daemon/context.ts

@@ -1,247 +0,0 @@
-import { AtpAgent } from '@atproto/api'
-import { allFulfilled } from '@atproto/common'
-import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
-import { IdResolver } from '@atproto/identity'
-import { createServiceAuthHeaders } from '@atproto/xrpc-server'
-import { BackgroundQueue } from '../background.js'
-import { OzoneConfig, OzoneSecrets } from '../config/index.js'
-import { Database } from '../db/index.js'
-import { ModerationService } from '../mod-service/index.js'
-import { StrikeService } from '../mod-service/strike.js'
-import { QueueService } from '../queue/service.js'
-import { ReportStatsService } from '../report/stats.js'
-import { ScheduledActionService } from '../scheduled-action/service.js'
-import { SettingService } from '../setting/service.js'
-import { TeamService } from '../team/index.js'
-import { getSigningKeyId } from '../util.js'
-import { EventPusher } from './event-pusher.js'
-import { EventReverser } from './event-reverser.js'
-import { MaterializedViewRefresher } from './materialized-view-refresher.js'
-import { QueueRouter } from './queue-router.js'
-import { ScheduledActionProcessor } from './scheduled-action-processor.js'
-import { StatsComputer } from './stats-computer.js'
-import { StrikeExpiryProcessor } from './strike-expiry-processor.js'
-import { TeamProfileSynchronizer } from './team-profile-synchronizer.js'
-import { VerificationListener } from './verification-listener.js'
-
-export type DaemonContextOptions = {
-  db: Database
-  cfg: OzoneConfig
-  backgroundQueue: BackgroundQueue
-  signingKey: Keypair
-  eventPusher: EventPusher
-  eventReverser: EventReverser
-  materializedViewRefresher: MaterializedViewRefresher
-  teamProfileSynchronizer: TeamProfileSynchronizer
-  scheduledActionProcessor: ScheduledActionProcessor
-  strikeExpiryProcessor: StrikeExpiryProcessor
-  queueRouter: QueueRouter
-  verificationListener?: VerificationListener
-  statsComputer?: StatsComputer
-}
-
-export class DaemonContext {
-  constructor(private opts: DaemonContextOptions) {}
-
-  static async fromConfig(
-    cfg: OzoneConfig,
-    secrets: OzoneSecrets,
-    overrides?: Partial<DaemonContextOptions>,
-  ): Promise<DaemonContext> {
-    const db = new Database({
-      url: cfg.db.postgresUrl,
-      schema: cfg.db.postgresSchema,
-    })
-    const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
-    const signingKeyId = await getSigningKeyId(db, signingKey.did())
-
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-    })
-
-    const appviewAgent = new AtpAgent({ service: cfg.appview.url })
-    const createAuthHeaders = (aud: string, lxm: string) =>
-      createServiceAuthHeaders({
-        iss: `${cfg.service.did}#atproto_labeler`,
-        aud,
-        lxm,
-        keypair: signingKey,
-      })
-
-    const eventPusher = new EventPusher(db, createAuthHeaders, {
-      appview: cfg.appview.pushEvents ? cfg.appview : undefined,
-      pds: cfg.pds ?? undefined,
-    })
-
-    const backgroundQueue = new BackgroundQueue(db, { concurrency: 20 })
-
-    const settingService = SettingService.creator()
-    const strikeService = StrikeService.creator()
-    const modService = ModerationService.creator(
-      signingKey,
-      signingKeyId,
-      cfg,
-      backgroundQueue,
-      idResolver,
-      eventPusher,
-      appviewAgent,
-      createAuthHeaders,
-      strikeService,
-    )
-    const scheduledActionService = ScheduledActionService.creator()
-    const teamService = TeamService.creator(
-      appviewAgent,
-      cfg.appview.did,
-      createAuthHeaders,
-    )
-    const teamProfileSynchronizer = new TeamProfileSynchronizer(
-      backgroundQueue,
-      teamService(db),
-      cfg.db.teamProfileRefreshIntervalMs,
-    )
-
-    const eventReverser = new EventReverser(db, modService)
-
-    const materializedViewRefresher = new MaterializedViewRefresher(
-      backgroundQueue,
-      cfg.db.materializedViewRefreshIntervalMs,
-    )
-
-    const scheduledActionProcessor = new ScheduledActionProcessor(
-      db,
-      cfg.service.did,
-      settingService,
-      modService,
-      scheduledActionService,
-    )
-
-    const strikeExpiryProcessor = new StrikeExpiryProcessor(db, strikeService)
-
-    const queueService = QueueService.creator()
-    const queueRouter = new QueueRouter(db, queueService)
-
-    const reportStatsService = ReportStatsService.creator()
-    const statsComputer = new StatsComputer(
-      db,
-      reportStatsService,
-      cfg.stats.computerIntervalMinutes,
-    )
-
-    // Only spawn the listener if verifier config exists and a jetstream URL is provided
-    const verificationListener =
-      cfg.verifier && cfg.jetstreamUrl
-        ? new VerificationListener(
-            db,
-            cfg.jetstreamUrl,
-            cfg.verifier?.issuersToIndex,
-          )
-        : undefined
-
-    return new DaemonContext({
-      db,
-      cfg,
-      backgroundQueue,
-      signingKey,
-      eventPusher,
-      eventReverser,
-      materializedViewRefresher,
-      teamProfileSynchronizer,
-      scheduledActionProcessor,
-      strikeExpiryProcessor,
-      queueRouter,
-      verificationListener,
-      statsComputer,
-      ...(overrides ?? {}),
-    })
-  }
-
-  get db(): Database {
-    return this.opts.db
-  }
-
-  get cfg(): OzoneConfig {
-    return this.opts.cfg
-  }
-
-  get backgroundQueue(): BackgroundQueue {
-    return this.opts.backgroundQueue
-  }
-
-  get eventPusher(): EventPusher {
-    return this.opts.eventPusher
-  }
-
-  get eventReverser(): EventReverser {
-    return this.opts.eventReverser
-  }
-
-  get materializedViewRefresher(): MaterializedViewRefresher {
-    return this.opts.materializedViewRefresher
-  }
-
-  get teamProfileSynchronizer(): TeamProfileSynchronizer {
-    return this.opts.teamProfileSynchronizer
-  }
-
-  get scheduledActionProcessor(): ScheduledActionProcessor {
-    return this.opts.scheduledActionProcessor
-  }
-
-  get strikeExpiryProcessor(): StrikeExpiryProcessor {
-    return this.opts.strikeExpiryProcessor
-  }
-
-  get queueRouter(): QueueRouter {
-    return this.opts.queueRouter
-  }
-
-  get verificationListener(): VerificationListener | undefined {
-    return this.opts.verificationListener
-  }
-
-  get statsComputer(): StatsComputer | undefined {
-    return this.opts.statsComputer
-  }
-
-  async start() {
-    this.eventPusher.start()
-    this.eventReverser.start()
-    this.materializedViewRefresher.start()
-    this.teamProfileSynchronizer.start()
-    this.scheduledActionProcessor.start()
-    this.strikeExpiryProcessor.start()
-    this.queueRouter.start()
-    this.verificationListener?.start()
-    this.statsComputer?.start()
-  }
-
-  async processAll() {
-    // Sequential because the materialized view values depend on the events.
-    await this.eventPusher.processAll()
-    // Drain pending modEventReport rows into the report table so test code
-    // that calls processAll() after creating reports sees the report rows
-    // immediately (in production this happens via the 1-min poll).
-    await this.queueRouter.routeReports()
-    await this.materializedViewRefresher.run()
-    await this.teamProfileSynchronizer.run()
-  }
-
-  async destroy() {
-    try {
-      await allFulfilled([
-        this.eventReverser.destroy(),
-        this.eventPusher.destroy(),
-        this.materializedViewRefresher.destroy(),
-        this.teamProfileSynchronizer.destroy(),
-        this.scheduledActionProcessor.destroy(),
-        this.strikeExpiryProcessor.destroy(),
-        this.queueRouter.destroy(),
-        this.verificationListener?.stop(),
-        this.statsComputer?.destroy(),
-      ])
-    } finally {
-      await this.backgroundQueue.destroy()
-      await this.db.close()
-    }
-  }
-}

package/src/daemon/event-pusher.ts

@@ -1,363 +0,0 @@
-import assert from 'node:assert'
-import { Insertable, Selectable } from 'kysely'
-import { AtpAgent } from '@atproto/api'
-import { SECOND } from '@atproto/common'
-import { Database } from '../db/index.js'
-import { BlobPushEvent } from '../db/schema/blob_push_event.js'
-import { RepoPushEventType } from '../db/schema/repo_push_event.js'
-import { ids } from '../lexicon/lexicons.js'
-import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus.js'
-import { dbLogger } from '../logger.js'
-import { retryHttp } from '../util.js'
-
-type EventSubject = InputSchema['subject']
-
-type PollState = {
-  timer?: NodeJS.Timeout
-  promise: Promise<void>
-}
-
-type AuthHeaders = {
-  headers: {
-    authorization: string
-  }
-}
-
-type Service = {
-  agent: AtpAgent
-  did: string
-}
-
-export class EventPusher {
-  destroyed = false
-
-  repoPollState: PollState = {
-    promise: Promise.resolve(),
-  }
-  recordPollState: PollState = {
-    promise: Promise.resolve(),
-  }
-  blobPollState: PollState = {
-    promise: Promise.resolve(),
-  }
-
-  appview: Service | undefined
-  pds: Service | undefined
-
-  constructor(
-    public db: Database,
-    public createAuthHeaders: (
-      aud: string,
-      method: string,
-    ) => Promise<AuthHeaders>,
-    services: {
-      appview?: {
-        url: string
-        did: string
-      }
-      pds?: {
-        url: string
-        did: string
-      }
-    },
-  ) {
-    if (services.appview) {
-      this.appview = {
-        agent: new AtpAgent({ service: services.appview.url }),
-        did: services.appview.did,
-      }
-    }
-    if (services.pds) {
-      this.pds = {
-        agent: new AtpAgent({ service: services.pds.url }),
-        did: services.pds.did,
-      }
-    }
-  }
-
-  start() {
-    this.poll(this.repoPollState, () => this.pushRepoEvents())
-    this.poll(this.recordPollState, () => this.pushRecordEvents())
-    this.poll(this.blobPollState, () => this.pushBlobEvents())
-  }
-
-  // event pusher may be configured with both appview and pds
-  // but the takedown may particularly want only one of them
-  // unless the target services are specified, we will push to all configured services
-  getTakedownServices(targetServices: Set<string>): RepoPushEventType[] {
-    let configured: RepoPushEventType[] = []
-    if (this.pds) configured.push('pds_takedown')
-    if (this.appview) configured.push('appview_takedown')
-
-    if (!targetServices.size) {
-      return configured
-    }
-
-    if (!targetServices.has('appview')) {
-      configured = configured.filter(
-        (service) => service !== 'appview_takedown',
-      )
-    }
-    if (!targetServices.has('pds')) {
-      configured = configured.filter((service) => service !== 'pds_takedown')
-    }
-
-    return configured
-  }
-
-  poll(state: PollState, fn: () => Promise<void>) {
-    if (this.destroyed) return
-    state.promise = fn()
-      .catch((err) => {
-        dbLogger.error({ err }, 'event push failed')
-      })
-      .finally(() => {
-        state.timer = setTimeout(() => this.poll(state, fn), 30 * SECOND)
-      })
-  }
-
-  async processAll() {
-    await Promise.all([
-      this.pushRepoEvents(),
-      this.pushRecordEvents(),
-      this.pushBlobEvents(),
-      this.repoPollState.promise,
-      this.recordPollState.promise,
-      this.blobPollState.promise,
-    ])
-  }
-
-  async destroy() {
-    this.destroyed = true
-    const destroyState = (state: PollState) => {
-      if (state.timer) {
-        clearTimeout(state.timer)
-      }
-      return state.promise
-    }
-    await Promise.all([
-      destroyState(this.repoPollState),
-      destroyState(this.recordPollState),
-      destroyState(this.blobPollState),
-    ])
-  }
-
-  async pushRepoEvents() {
-    const toPush = await this.db.db
-      .selectFrom('repo_push_event')
-      .select('id')
-      .forUpdate()
-      .skipLocked()
-      .where('confirmedAt', 'is', null)
-      .where('attempts', '<', 10)
-      .execute()
-    await Promise.all(toPush.map((evt) => this.attemptRepoEvent(evt.id)))
-  }
-
-  async pushRecordEvents() {
-    const toPush = await this.db.db
-      .selectFrom('record_push_event')
-      .select('id')
-      .forUpdate()
-      .skipLocked()
-      .where('confirmedAt', 'is', null)
-      .where('attempts', '<', 10)
-      .execute()
-    await Promise.all(toPush.map((evt) => this.attemptRecordEvent(evt.id)))
-  }
-
-  async pushBlobEvents() {
-    const toPush = await this.db.db
-      .selectFrom('blob_push_event')
-      .select('id')
-      .forUpdate()
-      .skipLocked()
-      .where('confirmedAt', 'is', null)
-      .where('attempts', '<', 10)
-      .execute()
-    await Promise.all(toPush.map((evt) => this.attemptBlobEvent(evt.id)))
-  }
-
-  private async updateSubjectOnService(
-    service: Service,
-    subject: EventSubject,
-    takedownRef: string | null,
-  ): Promise<boolean> {
-    const auth = await this.createAuthHeaders(
-      service.did,
-      ids.ComAtprotoAdminUpdateSubjectStatus,
-    )
-    try {
-      await retryHttp(() =>
-        service.agent.com.atproto.admin.updateSubjectStatus(
-          {
-            subject,
-            takedown: {
-              applied: !!takedownRef,
-              ref: takedownRef ?? undefined,
-            },
-          },
-          {
-            ...auth,
-            encoding: 'application/json',
-          },
-        ),
-      )
-      return true
-    } catch (err) {
-      dbLogger.error({ err, subject, takedownRef }, 'failed to push out event')
-      return false
-    }
-  }
-
-  async attemptRepoEvent(id: number) {
-    await this.db.transaction(async (dbTxn) => {
-      const evt = await dbTxn.db
-        .selectFrom('repo_push_event')
-        .selectAll()
-        .forUpdate()
-        .skipLocked()
-        .where('id', '=', id)
-        .where('confirmedAt', 'is', null)
-        .executeTakeFirst()
-      if (!evt) return
-      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
-      assert(service)
-      const subject = {
-        $type: 'com.atproto.admin.defs#repoRef',
-        did: evt.subjectDid,
-      }
-      const succeeded = await this.updateSubjectOnService(
-        service,
-        subject,
-        evt.takedownRef,
-      )
-      await dbTxn.db
-        .updateTable('repo_push_event')
-        .set(
-          succeeded
-            ? { confirmedAt: new Date() }
-            : {
-                lastAttempted: new Date(),
-                attempts: (evt.attempts ?? 0) + 1,
-              },
-        )
-        .where('subjectDid', '=', evt.subjectDid)
-        .where('eventType', '=', evt.eventType)
-        .execute()
-    })
-  }
-
-  async attemptRecordEvent(id: number) {
-    await this.db.transaction(async (dbTxn) => {
-      const evt = await dbTxn.db
-        .selectFrom('record_push_event')
-        .selectAll()
-        .forUpdate()
-        .skipLocked()
-        .where('id', '=', id)
-        .where('confirmedAt', 'is', null)
-        .executeTakeFirst()
-      if (!evt) return
-      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
-      assert(service)
-      const subject = {
-        $type: 'com.atproto.repo.strongRef',
-        uri: evt.subjectUri,
-        cid: evt.subjectCid,
-      }
-      const succeeded = await this.updateSubjectOnService(
-        service,
-        subject,
-        evt.takedownRef,
-      )
-      await dbTxn.db
-        .updateTable('record_push_event')
-        .set(
-          succeeded
-            ? { confirmedAt: new Date() }
-            : {
-                lastAttempted: new Date(),
-                attempts: (evt.attempts ?? 0) + 1,
-              },
-        )
-        .where('subjectUri', '=', evt.subjectUri)
-        .where('eventType', '=', evt.eventType)
-        .execute()
-    })
-  }
-
-  async attemptBlobEvent(id: number) {
-    await this.db.transaction(async (dbTxn) => {
-      const evt = await dbTxn.db
-        .selectFrom('blob_push_event')
-        .selectAll()
-        .forUpdate()
-        .skipLocked()
-        .where('id', '=', id)
-        .where('confirmedAt', 'is', null)
-        .executeTakeFirst()
-      if (!evt) return
-
-      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
-      assert(service)
-      const subject = {
-        $type: 'com.atproto.admin.defs#repoBlobRef',
-        did: evt.subjectDid,
-        cid: evt.subjectBlobCid,
-      }
-      const succeeded = await this.updateSubjectOnService(
-        service,
-        subject,
-        evt.takedownRef,
-      )
-      await this.markBlobEventAttempt(dbTxn, evt, succeeded)
-    })
-  }
-
-  async markBlobEventAttempt(
-    dbTxn: Database,
-    event: Selectable<BlobPushEvent>,
-    succeeded: boolean,
-  ) {
-    await dbTxn.db
-      .updateTable('blob_push_event')
-      .set(
-        succeeded
-          ? { confirmedAt: new Date() }
-          : {
-              lastAttempted: new Date(),
-              attempts: (event.attempts ?? 0) + 1,
-            },
-      )
-      .where('subjectDid', '=', event.subjectDid)
-      .where('subjectBlobCid', '=', event.subjectBlobCid)
-      .where('eventType', '=', event.eventType)
-      .execute()
-  }
-
-  async logBlobPushEvent(
-    blobValues: Insertable<BlobPushEvent>[],
-    takedownRef?: string | null,
-  ) {
-    return this.db.db
-      .insertInto('blob_push_event')
-      .values(blobValues)
-      .onConflict((oc) =>
-        oc.columns(['subjectDid', 'subjectBlobCid', 'eventType']).doUpdateSet({
-          takedownRef,
-          confirmedAt: null,
-          attempts: 0,
-          lastAttempted: null,
-        }),
-      )
-      .returning([
-        'id',
-        'subjectDid',
-        'subjectUri',
-        'subjectBlobCid',
-        'eventType',
-      ])
-      .execute()
-  }
-}