@atproto/ozone 0.0.2

package/src/config/secrets.ts

@@ -0,0 +1,23 @@
+import assert from 'assert'
+import { OzoneEnvironment } from './env'
+
+export const envToSecrets = (env: OzoneEnvironment): OzoneSecrets => {
+  assert(env.adminPassword)
+  assert(env.moderatorPassword)
+  assert(env.triagePassword)
+  assert(env.signingKeyHex)
+
+  return {
+    adminPassword: env.adminPassword,
+    moderatorPassword: env.moderatorPassword,
+    triagePassword: env.triagePassword,
+    signingKeyHex: env.signingKeyHex,
+  }
+}
+
+export type OzoneSecrets = {
+  adminPassword: string
+  moderatorPassword: string
+  triagePassword: string
+  signingKeyHex: string
+}

package/src/context.ts

@@ -0,0 +1,180 @@
+import * as plc from '@did-plc/lib'
+import { IdResolver } from '@atproto/identity'
+import { AtpAgent } from '@atproto/api'
+import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
+import { createServiceAuthHeaders } from '@atproto/xrpc-server'
+import { Database } from './db'
+import { OzoneConfig, OzoneSecrets } from './config'
+import { ModerationService, ModerationServiceCreator } from './mod-service'
+import * as auth from './auth'
+import { BackgroundQueue } from './background'
+import assert from 'assert'
+import { EventPusher } from './daemon'
+
+export type AppContextOptions = {
+  db: Database
+  cfg: OzoneConfig
+  modService: ModerationServiceCreator
+  appviewAgent: AtpAgent
+  pdsAgent: AtpAgent | undefined
+  signingKey: Keypair
+  idResolver: IdResolver
+  backgroundQueue: BackgroundQueue
+}
+
+export class AppContext {
+  constructor(private opts: AppContextOptions, private secrets: OzoneSecrets) {}
+
+  static async fromConfig(
+    cfg: OzoneConfig,
+    secrets: OzoneSecrets,
+    overrides?: Partial<AppContextOptions>,
+  ): Promise<AppContext> {
+    const db = new Database({
+      url: cfg.db.postgresUrl,
+      schema: cfg.db.postgresSchema,
+    })
+    const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+    const appviewAgent = new AtpAgent({ service: cfg.appview.url })
+    const pdsAgent = cfg.pds
+      ? new AtpAgent({ service: cfg.pds.url })
+      : undefined
+
+    const createAuthHeaders = (aud: string) =>
+      createServiceAuthHeaders({
+        iss: cfg.service.did,
+        aud,
+        keypair: signingKey,
+      })
+    const appviewAuth = async () =>
+      cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined
+
+    const backgroundQueue = new BackgroundQueue(db)
+    const eventPusher = new EventPusher(db, createAuthHeaders, {
+      appview: cfg.appview,
+      pds: cfg.pds ?? undefined,
+    })
+
+    const modService = ModerationService.creator(
+      backgroundQueue,
+      eventPusher,
+      appviewAgent,
+      appviewAuth,
+    )
+
+    const idResolver = new IdResolver({
+      plcUrl: cfg.identity.plcUrl,
+    })
+
+    return new AppContext(
+      {
+        db,
+        cfg,
+        modService,
+        appviewAgent,
+        pdsAgent,
+        signingKey,
+        idResolver,
+        backgroundQueue,
+        ...(overrides ?? {}),
+      },
+      secrets,
+    )
+  }
+
+  assignPort(port: number) {
+    assert(
+      !this.cfg.service.port || this.cfg.service.port === port,
+      'Conflicting port in config',
+    )
+    this.opts.cfg.service.port = port
+  }
+
+  get db(): Database {
+    return this.opts.db
+  }
+
+  get cfg(): OzoneConfig {
+    return this.opts.cfg
+  }
+
+  get modService(): ModerationServiceCreator {
+    return this.opts.modService
+  }
+
+  get appviewAgent(): AtpAgent {
+    return this.opts.appviewAgent
+  }
+
+  get pdsAgent(): AtpAgent | undefined {
+    return this.opts.pdsAgent
+  }
+
+  get signingKey(): Keypair {
+    return this.opts.signingKey
+  }
+
+  get plcClient(): plc.Client {
+    return new plc.Client(this.cfg.identity.plcUrl)
+  }
+
+  get idResolver(): IdResolver {
+    return this.opts.idResolver
+  }
+
+  get backgroundQueue(): BackgroundQueue {
+    return this.opts.backgroundQueue
+  }
+
+  get authVerifier() {
+    return auth.authVerifier(this.idResolver, { aud: this.cfg.service.did })
+  }
+
+  get authVerifierAnyAudience() {
+    return auth.authVerifier(this.idResolver, { aud: null })
+  }
+
+  get authOptionalVerifierAnyAudience() {
+    return auth.authOptionalVerifier(this.idResolver, { aud: null })
+  }
+
+  get authOptionalVerifier() {
+    return auth.authOptionalVerifier(this.idResolver, {
+      aud: this.cfg.service.did,
+    })
+  }
+
+  get authOptionalAccessOrRoleVerifier() {
+    return auth.authOptionalAccessOrRoleVerifier(
+      this.idResolver,
+      this.secrets,
+      this.cfg.service.did,
+    )
+  }
+
+  get roleVerifier() {
+    return auth.roleVerifier(this.secrets)
+  }
+
+  async serviceAuthHeaders(aud: string) {
+    const iss = this.cfg.service.did
+    return createServiceAuthHeaders({
+      iss,
+      aud,
+      keypair: this.signingKey,
+    })
+  }
+
+  async pdsAuth() {
+    if (!this.cfg.pds) {
+      return undefined
+    }
+    return this.serviceAuthHeaders(this.cfg.pds.did)
+  }
+
+  async appviewAuth() {
+    return this.serviceAuthHeaders(this.cfg.appview.did)
+  }
+}
+
+export default AppContext

package/src/daemon/context.ts

@@ -0,0 +1,90 @@
+import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
+import { createServiceAuthHeaders } from '@atproto/xrpc-server'
+import AtpAgent from '@atproto/api'
+import { OzoneConfig, OzoneSecrets } from '../config'
+import { Database } from '../db'
+import { EventPusher } from './event-pusher'
+import { EventReverser } from './event-reverser'
+import { ModerationService, ModerationServiceCreator } from '../mod-service'
+import { BackgroundQueue } from '../background'
+
+export type DaemonContextOptions = {
+  db: Database
+  cfg: OzoneConfig
+  modService: ModerationServiceCreator
+  signingKey: Keypair
+  eventPusher: EventPusher
+  eventReverser: EventReverser
+}
+
+export class DaemonContext {
+  constructor(private opts: DaemonContextOptions) {}
+
+  static async fromConfig(
+    cfg: OzoneConfig,
+    secrets: OzoneSecrets,
+    overrides?: Partial<DaemonContextOptions>,
+  ): Promise<DaemonContext> {
+    const db = new Database({
+      url: cfg.db.postgresUrl,
+      schema: cfg.db.postgresSchema,
+    })
+    const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+
+    const appviewAgent = new AtpAgent({ service: cfg.appview.url })
+    const createAuthHeaders = (aud: string) =>
+      createServiceAuthHeaders({
+        iss: cfg.service.did,
+        aud,
+        keypair: signingKey,
+      })
+
+    const appviewAuth = async () =>
+      cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined
+
+    const eventPusher = new EventPusher(db, createAuthHeaders, {
+      appview: cfg.appview,
+      pds: cfg.pds ?? undefined,
+    })
+    const backgroundQueue = new BackgroundQueue(db)
+    const modService = ModerationService.creator(
+      backgroundQueue,
+      eventPusher,
+      appviewAgent,
+      appviewAuth,
+    )
+    const eventReverser = new EventReverser(db, modService)
+
+    return new DaemonContext({
+      db,
+      cfg,
+      modService,
+      signingKey,
+      eventPusher,
+      eventReverser,
+      ...(overrides ?? {}),
+    })
+  }
+
+  get db(): Database {
+    return this.opts.db
+  }
+
+  get cfg(): OzoneConfig {
+    return this.opts.cfg
+  }
+
+  get modService(): ModerationServiceCreator {
+    return this.opts.modService
+  }
+
+  get eventPusher(): EventPusher {
+    return this.opts.eventPusher
+  }
+
+  get eventReverser(): EventReverser {
+    return this.opts.eventReverser
+  }
+}
+
+export default DaemonContext

package/src/daemon/event-pusher.ts

@@ -0,0 +1,296 @@
+import AtpAgent from '@atproto/api'
+import { SECOND } from '@atproto/common'
+import Database from '../db'
+import { retryHttp } from '../util'
+import { dbLogger } from '../logger'
+import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus'
+import assert from 'assert'
+
+type EventSubject = InputSchema['subject']
+
+type PollState = {
+  timer?: NodeJS.Timer
+  promise: Promise<void>
+}
+
+type AuthHeaders = {
+  headers: {
+    authorization: string
+  }
+}
+
+type Service = {
+  agent: AtpAgent
+  did: string
+}
+
+export class EventPusher {
+  destroyed = false
+
+  repoPollState: PollState = {
+    promise: Promise.resolve(),
+  }
+  recordPollState: PollState = {
+    promise: Promise.resolve(),
+  }
+  blobPollState: PollState = {
+    promise: Promise.resolve(),
+  }
+
+  appview: Service | undefined
+  pds: Service | undefined
+
+  constructor(
+    public db: Database,
+    public createAuthHeaders: (aud: string) => Promise<AuthHeaders>,
+    services: {
+      appview?: {
+        url: string
+        did: string
+      }
+      pds?: {
+        url: string
+        did: string
+      }
+    },
+  ) {
+    if (services.appview) {
+      this.appview = {
+        agent: new AtpAgent({ service: services.appview.url }),
+        did: services.appview.did,
+      }
+    }
+    if (services.pds) {
+      this.pds = {
+        agent: new AtpAgent({ service: services.pds.url }),
+        did: services.pds.did,
+      }
+    }
+  }
+
+  start() {
+    this.poll(this.repoPollState, () => this.pushRepoEvents())
+    this.poll(this.recordPollState, () => this.pushRecordEvents())
+    this.poll(this.blobPollState, () => this.pushBlobEvents())
+  }
+
+  poll(state: PollState, fn: () => Promise<void>) {
+    if (this.destroyed) return
+    state.promise = fn()
+      .catch((err) => {
+        dbLogger.error({ err }, 'event push failed')
+      })
+      .finally(() => {
+        state.timer = setTimeout(() => this.poll(state, fn), 30 * SECOND)
+      })
+  }
+
+  async processAll() {
+    await Promise.all([
+      this.pushRepoEvents(),
+      this.pushRecordEvents(),
+      this.pushBlobEvents(),
+      this.repoPollState.promise,
+      this.recordPollState.promise,
+      this.blobPollState.promise,
+    ])
+  }
+
+  async destroy() {
+    this.destroyed = true
+    const destroyState = (state: PollState) => {
+      if (state.timer) {
+        clearTimeout(state.timer)
+      }
+      return state.promise
+    }
+    await Promise.all([
+      destroyState(this.repoPollState),
+      destroyState(this.recordPollState),
+      destroyState(this.blobPollState),
+    ])
+  }
+
+  async pushRepoEvents() {
+    const toPush = await this.db.db
+      .selectFrom('repo_push_event')
+      .select('id')
+      .forUpdate()
+      .skipLocked()
+      .where('confirmedAt', 'is', null)
+      .where('attempts', '<', 10)
+      .execute()
+    await Promise.all(toPush.map((evt) => this.attemptRepoEvent(evt.id)))
+  }
+
+  async pushRecordEvents() {
+    const toPush = await this.db.db
+      .selectFrom('record_push_event')
+      .select('id')
+      .forUpdate()
+      .skipLocked()
+      .where('confirmedAt', 'is', null)
+      .where('attempts', '<', 10)
+      .execute()
+    await Promise.all(toPush.map((evt) => this.attemptRecordEvent(evt.id)))
+  }
+
+  async pushBlobEvents() {
+    const toPush = await this.db.db
+      .selectFrom('blob_push_event')
+      .select('id')
+      .forUpdate()
+      .skipLocked()
+      .where('confirmedAt', 'is', null)
+      .where('attempts', '<', 10)
+      .execute()
+    await Promise.all(toPush.map((evt) => this.attemptBlobEvent(evt.id)))
+  }
+
+  private async updateSubjectOnService(
+    service: Service,
+    subject: EventSubject,
+    takedownRef: string | null,
+  ): Promise<boolean> {
+    const auth = await this.createAuthHeaders(service.did)
+    try {
+      await retryHttp(() =>
+        service.agent.com.atproto.admin.updateSubjectStatus(
+          {
+            subject,
+            takedown: {
+              applied: !!takedownRef,
+              ref: takedownRef ?? undefined,
+            },
+          },
+          {
+            ...auth,
+            encoding: 'application/json',
+          },
+        ),
+      )
+      return true
+    } catch (err) {
+      dbLogger.error({ err, subject, takedownRef }, 'failed to push out event')
+      return false
+    }
+  }
+
+  async attemptRepoEvent(id: number) {
+    await this.db.transaction(async (dbTxn) => {
+      const evt = await dbTxn.db
+        .selectFrom('repo_push_event')
+        .selectAll()
+        .forUpdate()
+        .skipLocked()
+        .where('id', '=', id)
+        .where('confirmedAt', 'is', null)
+        .executeTakeFirst()
+      if (!evt) return
+      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
+      assert(service)
+      const subject = {
+        $type: 'com.atproto.admin.defs#repoRef',
+        did: evt.subjectDid,
+      }
+      const succeeded = await this.updateSubjectOnService(
+        service,
+        subject,
+        evt.takedownRef,
+      )
+      await dbTxn.db
+        .updateTable('repo_push_event')
+        .set(
+          succeeded
+            ? { confirmedAt: new Date() }
+            : {
+                lastAttempted: new Date(),
+                attempts: evt.attempts ?? 0 + 1,
+              },
+        )
+        .where('subjectDid', '=', evt.subjectDid)
+        .where('eventType', '=', evt.eventType)
+        .execute()
+    })
+  }
+
+  async attemptRecordEvent(id: number) {
+    await this.db.transaction(async (dbTxn) => {
+      const evt = await dbTxn.db
+        .selectFrom('record_push_event')
+        .selectAll()
+        .forUpdate()
+        .skipLocked()
+        .where('id', '=', id)
+        .where('confirmedAt', 'is', null)
+        .executeTakeFirst()
+      if (!evt) return
+      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
+      assert(service)
+      const subject = {
+        $type: 'com.atproto.repo.strongRef',
+        uri: evt.subjectUri,
+        cid: evt.subjectCid,
+      }
+      const succeeded = await this.updateSubjectOnService(
+        service,
+        subject,
+        evt.takedownRef,
+      )
+      await dbTxn.db
+        .updateTable('record_push_event')
+        .set(
+          succeeded
+            ? { confirmedAt: new Date() }
+            : {
+                lastAttempted: new Date(),
+                attempts: evt.attempts ?? 0 + 1,
+              },
+        )
+        .where('subjectUri', '=', evt.subjectUri)
+        .where('eventType', '=', evt.eventType)
+        .execute()
+    })
+  }
+
+  async attemptBlobEvent(id: number) {
+    await this.db.transaction(async (dbTxn) => {
+      const evt = await dbTxn.db
+        .selectFrom('blob_push_event')
+        .selectAll()
+        .forUpdate()
+        .skipLocked()
+        .where('id', '=', id)
+        .where('confirmedAt', 'is', null)
+        .executeTakeFirst()
+      if (!evt) return
+
+      const service = evt.eventType === 'pds_takedown' ? this.pds : this.appview
+      assert(service)
+      const subject = {
+        $type: 'com.atproto.admin.defs#repoBlobRef',
+        did: evt.subjectDid,
+        cid: evt.subjectBlobCid,
+      }
+      const succeeded = await this.updateSubjectOnService(
+        service,
+        subject,
+        evt.takedownRef,
+      )
+      await dbTxn.db
+        .updateTable('blob_push_event')
+        .set(
+          succeeded
+            ? { confirmedAt: new Date() }
+            : {
+                lastAttempted: new Date(),
+                attempts: evt.attempts ?? 0 + 1,
+              },
+        )
+        .where('subjectDid', '=', evt.subjectDid)
+        .where('subjectBlobCid', '=', evt.subjectBlobCid)
+        .where('eventType', '=', evt.eventType)
+        .execute()
+    })
+  }
+}

package/src/daemon/event-reverser.ts

@@ -0,0 +1,74 @@
+import { MINUTE } from '@atproto/common'
+import { dbLogger } from '../logger'
+import { ModerationServiceCreator, ReversalSubject } from '../mod-service'
+import Database from '../db'
+
+export class EventReverser {
+  destroyed = false
+  reversalPromise: Promise<void> = Promise.resolve()
+  timer: NodeJS.Timer | undefined
+
+  constructor(
+    private db: Database,
+    private modService: ModerationServiceCreator,
+  ) {}
+
+  start() {
+    this.poll()
+  }
+
+  poll() {
+    if (this.destroyed) return
+    this.reversalPromise = this.findAndRevertDueActions()
+      .catch((err) =>
+        dbLogger.error({ err }, 'moderation action reversal errored'),
+      )
+      .finally(() => {
+        this.timer = setTimeout(() => this.poll(), getInterval())
+      })
+  }
+
+  async destroy() {
+    this.destroyed = true
+    if (this.timer) {
+      clearTimeout(this.timer)
+    }
+    await this.reversalPromise
+  }
+
+  async revertState(subject: ReversalSubject) {
+    await this.db.transaction(async (dbTxn) => {
+      const moderationTxn = this.modService(dbTxn)
+      const originalEvent =
+        await moderationTxn.getLastReversibleEventForSubject(subject)
+      if (originalEvent) {
+        await moderationTxn.revertState({
+          action: originalEvent.action,
+          createdBy: originalEvent.createdBy,
+          comment:
+            '[SCHEDULED_REVERSAL] Reverting action as originally scheduled',
+          subject: subject.subject,
+          createdAt: new Date(),
+        })
+      }
+    })
+  }
+
+  async findAndRevertDueActions() {
+    const moderationService = this.modService(this.db)
+    const subjectsDueForReversal =
+      await moderationService.getSubjectsDueForReversal()
+
+    // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine
+    // Internally, each reversal runs within its own transaction
+    await Promise.all(subjectsDueForReversal.map(this.revertState.bind(this)))
+  }
+}
+
+const getInterval = (): number => {
+  // super basic synchronization by agreeing when the intervals land relative to unix timestamp
+  const now = Date.now()
+  const intervalMs = MINUTE
+  const nextIteration = Math.ceil(now / intervalMs)
+  return nextIteration * intervalMs - now
+}

package/src/daemon/index.ts

@@ -0,0 +1,33 @@
+import { OzoneConfig, OzoneSecrets } from '../config'
+import DaemonContext from './context'
+import { AppContextOptions } from '../context'
+
+export { EventPusher } from './event-pusher'
+export { EventReverser } from './event-reverser'
+
+export class OzoneDaemon {
+  constructor(public ctx: DaemonContext) {}
+  static async create(
+    cfg: OzoneConfig,
+    secrets: OzoneSecrets,
+    overrides?: Partial<AppContextOptions>,
+  ): Promise<OzoneDaemon> {
+    const ctx = await DaemonContext.fromConfig(cfg, secrets, overrides)
+    return new OzoneDaemon(ctx)
+  }
+
+  async start() {
+    this.ctx.eventPusher.start()
+    this.ctx.eventReverser.start()
+  }
+
+  async processAll() {
+    await this.ctx.eventPusher.processAll()
+  }
+
+  async destroy() {
+    await this.ctx.eventReverser.destroy()
+    await this.ctx.eventPusher.destroy()
+    await this.ctx.db.close()
+  }
+}