@atproto/ozone 0.2.9 → 0.2.11

package/src/daemon/event-reverser.ts

@@ -1,128 +0,0 @@
-import { MINUTE } from '@atproto/common'
-import { Database } from '../db/index.js'
-import { dbLogger } from '../logger.js'
-import {
-  deleteExpiringTagsByIds,
-  getExpiredTags,
-} from '../mod-service/expiring-tags.js'
-import {
-  ModerationServiceCreator,
-  ReversalSubject,
-} from '../mod-service/index.js'
-import { subjectFromStatusRow } from '../mod-service/subject.js'
-
-export class EventReverser {
-  destroyed = false
-  reversalPromise: Promise<void> = Promise.resolve()
-  timer?: NodeJS.Timeout
-
-  constructor(
-    private db: Database,
-    private modService: ModerationServiceCreator,
-  ) {}
-
-  start() {
-    this.poll()
-  }
-
-  poll() {
-    if (this.destroyed) return
-    this.reversalPromise = this.findAndRevertDueActions()
-      .catch((err) =>
-        dbLogger.error({ err }, 'moderation action reversal errored'),
-      )
-      .finally(() => {
-        this.timer = setTimeout(() => this.poll(), getInterval())
-      })
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    await this.reversalPromise
-  }
-
-  async revertState(subject: ReversalSubject) {
-    await this.db.transaction(async (dbTxn) => {
-      const moderationTxn = this.modService(dbTxn)
-      const originalEvent =
-        await moderationTxn.getLastReversibleEventForSubject(subject)
-      if (originalEvent) {
-        await moderationTxn.revertState({
-          action: originalEvent.action,
-          createdBy: originalEvent.createdBy,
-          comment:
-            '[SCHEDULED_REVERSAL] Reverting action as originally scheduled',
-          subject: subject.subject,
-          createdAt: new Date(),
-        })
-      }
-    })
-  }
-
-  async findAndRevertDueActions() {
-    const moderationService = this.modService(this.db)
-    const subjectsDueForReversal =
-      await moderationService.getSubjectsDueForReversal()
-
-    // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine
-    // Internally, each reversal runs within its own transaction
-    await Promise.all([
-      ...subjectsDueForReversal.map(this.revertState.bind(this)),
-      this.findAndRevertExpiredTags(),
-    ])
-  }
-
-  async findAndRevertExpiredTags() {
-    const groups = await getExpiredTags(this.db)
-    if (!groups.length) return
-
-    for (const group of groups) {
-      await this.db.transaction(async (dbTxn) => {
-        // Check which tags are still present on the subject
-        const status = await dbTxn.db
-          .selectFrom('moderation_subject_status')
-          .where('did', '=', group.did)
-          .where('recordPath', '=', group.recordPath)
-          .where('convoId', '=', group.convoId)
-          .selectAll()
-          .executeTakeFirst()
-
-        const currentTags: string[] = status?.tags ?? []
-        const tagsToRemove = group.tags.filter((t) => currentTags.includes(t))
-
-        // Delete the expiring_tag rows regardless
-        await deleteExpiringTagsByIds(dbTxn, group.ids)
-
-        // Only emit removal event if there are tags still present to remove
-        if (tagsToRemove.length > 0 && status) {
-          const subject = subjectFromStatusRow(status)
-          const moderationTxn = this.modService(dbTxn)
-          await moderationTxn.logEvent({
-            event: {
-              $type: 'tools.ozone.moderation.defs#modEventTag',
-              add: [],
-              remove: tagsToRemove,
-              comment:
-                '[SCHEDULED_REVERSAL] Reverting temporary tags as originally scheduled',
-            },
-            createdBy: group.createdBy,
-            subject,
-            createdAt: new Date(),
-          })
-        }
-      })
-    }
-  }
-}
-
-const getInterval = (): number => {
-  // super basic synchronization by agreeing when the intervals land relative to unix timestamp
-  const now = Date.now()
-  const intervalMs = MINUTE
-  const nextIteration = Math.ceil(now / intervalMs)
-  return nextIteration * intervalMs - now
-}

package/src/daemon/index.ts

@@ -1,33 +0,0 @@
-import { OzoneConfig, OzoneSecrets } from '../config/index.js'
-import { AppContextOptions } from '../context.js'
-import { DaemonContext } from './context.js'
-
-export { EventPusher } from './event-pusher.js'
-export { BlobDiverter } from './blob-diverter.js'
-export { EventReverser } from './event-reverser.js'
-export { ScheduledActionProcessor } from './scheduled-action-processor.js'
-export { StrikeExpiryProcessor } from './strike-expiry-processor.js'
-
-export class OzoneDaemon {
-  constructor(public ctx: DaemonContext) {}
-  static async create(
-    cfg: OzoneConfig,
-    secrets: OzoneSecrets,
-    overrides?: Partial<AppContextOptions>,
-  ): Promise<OzoneDaemon> {
-    const ctx = await DaemonContext.fromConfig(cfg, secrets, overrides)
-    return new OzoneDaemon(ctx)
-  }
-
-  async start() {
-    await this.ctx.start()
-  }
-
-  async processAll() {
-    await this.ctx.processAll()
-  }
-
-  async destroy() {
-    await this.ctx.destroy()
-  }
-}

package/src/daemon/job-cursor.ts

@@ -1,33 +0,0 @@
-import { Database } from '../db/index.js'
-
-export async function initJobCursor(db: Database, job: string): Promise<void> {
-  await db.db
-    .insertInto('job_cursor')
-    .values({ job, cursor: null })
-    .onConflict((oc) => oc.doNothing())
-    .execute()
-}
-
-export async function getJobCursor(
-  db: Database,
-  job: string,
-): Promise<string | null> {
-  const entry = await db.db
-    .selectFrom('job_cursor')
-    .select('cursor')
-    .where('job', '=', job)
-    .executeTakeFirst()
-  return entry?.cursor ?? null
-}
-
-export async function updateJobCursor(
-  db: Database,
-  job: string,
-  cursor: string,
-): Promise<void> {
-  await db.db
-    .updateTable('job_cursor')
-    .set({ cursor })
-    .where('job', '=', job)
-    .execute()
-}

package/src/daemon/materialized-view-refresher.ts

@@ -1,33 +0,0 @@
-import { sql } from 'kysely'
-import { MINUTE } from '@atproto/common'
-import { BackgroundQueue, PeriodicBackgroundTask } from '../background.js'
-import { dbLogger } from '../logger.js'
-
-export class MaterializedViewRefresher extends PeriodicBackgroundTask {
-  constructor(backgroundQueue: BackgroundQueue, interval = 30 * MINUTE) {
-    super(backgroundQueue, interval, async ({ db }, signal) => {
-      for (const view of [
-        'account_events_stats',
-        'record_events_stats',
-        'account_record_events_stats',
-        'account_record_status_stats',
-      ]) {
-        if (signal.aborted) break
-
-        // Kysely does not provide a way to cancel a running query. Because of
-        // this, killing the process during a refresh will cause the process to
-        // wait for the current refresh to finish before exiting. This is not
-        // ideal, but it is the best we can do until Kysely provides a way to
-        // cancel a query.
-        try {
-          await sql`REFRESH MATERIALIZED VIEW CONCURRENTLY ${sql.id(view)}`.execute(
-            db,
-          )
-          dbLogger.info(`refreshed materialized view ${view}`)
-        } catch (err) {
-          dbLogger.error({ err, view }, 'failed to refresh materialized view')
-        }
-      }
-    })
-  }
-}

package/src/daemon/queue-router.ts

@@ -1,101 +0,0 @@
-import { MINUTE } from '@atproto/common'
-import { Database } from '../db/index.js'
-import { dbLogger } from '../logger.js'
-import { QueueServiceCreator } from '../queue/service.js'
-import { initJobCursor } from './job-cursor.js'
-
-const JOB_NAME = 'queue_router'
-const BATCH_SIZE = 100
-
-export class QueueRouter {
-  destroyed = false
-  processingPromise: Promise<void> = Promise.resolve()
-  timer?: NodeJS.Timeout
-
-  constructor(
-    private db: Database,
-    private queueServiceCreator: QueueServiceCreator,
-  ) {}
-
-  start() {
-    this.initializeCursor().then(() => this.poll())
-  }
-
-  poll() {
-    if (this.destroyed) return
-    this.processingPromise = this.routeReports()
-      .catch((err) => dbLogger.error({ err }, 'queue routing errored'))
-      .finally(() => {
-        this.timer = setTimeout(() => this.poll(), getInterval())
-      })
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    await this.processingPromise
-  }
-
-  async initializeCursor() {
-    await initJobCursor(this.db, JOB_NAME)
-  }
-
-  async getCursor(): Promise<number | null> {
-    const row = await this.db.db
-      .selectFrom('job_cursor')
-      .select('cursor')
-      .where('job', '=', JOB_NAME)
-      .executeTakeFirst()
-    return row?.cursor ? parseInt(row.cursor, 10) : null
-  }
-
-  async routeReports() {
-    await this.db.transaction(async (txn) => {
-      // Acquire row lock on the job_cursor row. A second daemon instance
-      // hitting this same query blocks here until the first transaction
-      // commits, then reads the now-advanced cursor and processes the next
-      // range. The lock is held for the whole batch (~50–200ms).
-      const row = await txn.db
-        .selectFrom('job_cursor')
-        .selectAll()
-        .where('job', '=', JOB_NAME)
-        .forUpdate()
-        .executeTakeFirst()
-      if (!row) return
-      const cursor = row.cursor ? parseInt(row.cursor, 10) : null
-
-      const queueService = this.queueServiceCreator(txn)
-      const result = await queueService.insertReportsFromEvents({
-        cursor,
-        limit: BATCH_SIZE,
-      })
-
-      if (result.processed === 0) {
-        dbLogger.info('no new report events to route')
-        return
-      }
-
-      await txn.db
-        .updateTable('job_cursor')
-        .set({ cursor: String(result.maxEventId) })
-        .where('job', '=', JOB_NAME)
-        .execute()
-
-      dbLogger.info(
-        {
-          processed: result.processed,
-          assigned: result.assigned,
-          unmatched: result.unmatched,
-          maxEventId: result.maxEventId,
-        },
-        'queue routing completed',
-      )
-    })
-  }
-}
-
-// Poll every 1 minute
-const getInterval = (): number => 1 * MINUTE

package/src/daemon/scheduled-action-processor.ts

@@ -1,304 +0,0 @@
-import { Selectable } from 'kysely'
-import { MINUTE, SECOND } from '@atproto/common'
-import {
-  assertProtectedTagAction,
-  getProtectedTags,
-} from '../api/moderation/util.js'
-import { Database } from '../db/index.js'
-import { ScheduledAction } from '../db/schema/scheduled-action.js'
-import {
-  ModEventTakedown,
-  ModTool,
-} from '../lexicon/types/tools/ozone/moderation/defs.js'
-import { dbLogger } from '../logger.js'
-import {
-  ModerationService,
-  ModerationServiceCreator,
-} from '../mod-service/index.js'
-import { RepoSubject } from '../mod-service/subject.js'
-import { ModEventType } from '../mod-service/types.js'
-import { ScheduledActionServiceCreator } from '../scheduled-action/service.js'
-import { SettingService, SettingServiceCreator } from '../setting/service.js'
-import { retryHttp } from '../util.js'
-
-export class ScheduledActionProcessor {
-  destroyed = false
-  processingPromise: Promise<void> = Promise.resolve()
-  timer?: NodeJS.Timeout
-
-  constructor(
-    private db: Database,
-    private serviceDid: string,
-    private settingService: SettingServiceCreator,
-    private modService: ModerationServiceCreator,
-    private scheduledActionService: ScheduledActionServiceCreator,
-  ) {}
-
-  start() {
-    this.poll()
-  }
-
-  poll() {
-    if (this.destroyed) return
-    this.processingPromise = this.findAndExecuteScheduledActions()
-      .catch((err) =>
-        dbLogger.error({ err }, 'scheduled action processing errored'),
-      )
-      .finally(() => {
-        this.timer = setTimeout(() => this.poll(), getInterval())
-      })
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    await this.processingPromise
-  }
-
-  async executeScheduledAction(actionId: number) {
-    await this.db.transaction(async (dbTxn) => {
-      const settingService = this.settingService(dbTxn)
-      const moderationTxn = this.modService(dbTxn)
-      const scheduledActionTxn = this.scheduledActionService(dbTxn)
-
-      try {
-        // maybe overfetching here to get the action again within the transaction to ensure it's still pending
-        const action = await dbTxn.db
-          .selectFrom('scheduled_action')
-          .selectAll()
-          .where('id', '=', actionId)
-          .where('status', '=', 'pending')
-          .executeTakeFirst()
-
-        if (!action) {
-          // already processed or cancelled
-          return
-        }
-
-        let event: ModEventType
-        const email = {
-          subject: '',
-          content: '',
-        }
-        let modTool: ModTool | undefined
-
-        // Create the appropriate moderation action based on the scheduled action type
-        switch (action.action) {
-          case 'takedown':
-            {
-              const eventData = action.eventData as ModEventTakedown & {
-                modTool?: ModTool
-                emailSubject?: string
-                emailContent?: string
-              }
-              modTool = eventData.modTool
-              event = {
-                $type: 'tools.ozone.moderation.defs#modEventTakedown',
-                comment: `[SCHEDULED_ACTION] ${eventData.comment || 'Scheduled takedown executed'}`,
-                durationInHours: eventData.durationInHours,
-                acknowledgeAccountSubjects:
-                  eventData.acknowledgeAccountSubjects,
-                policies: eventData.policies,
-                severityLevel: eventData.severityLevel,
-                strikeCount: eventData.strikeCount,
-              }
-
-              if (eventData.emailSubject && eventData.emailContent) {
-                email.subject = eventData.emailSubject
-                email.content = eventData.emailContent
-              }
-            }
-            break
-          default:
-            throw new Error(
-              `Unsupported scheduled action type: ${action.action}`,
-            )
-        }
-
-        const moderationEvent = await this.performTakedown({
-          action,
-          event,
-          modTool,
-          moderationTxn,
-          settingService,
-          email,
-        })
-
-        // Mark the scheduled action as executed
-        await scheduledActionTxn.markActionAsExecuted(
-          actionId,
-          moderationEvent.event.id,
-        )
-
-        dbLogger.info(
-          {
-            did: action.did,
-            scheduledActionId: actionId,
-            moderationEventId: moderationEvent.event.id,
-          },
-          'executed scheduled action',
-        )
-      } catch (error) {
-        const errorMessage =
-          error instanceof Error ? error.message : 'Unknown error'
-
-        // mark as failed
-        await scheduledActionTxn.markActionAsFailed(actionId, errorMessage)
-
-        dbLogger.error(
-          {
-            scheduledActionId: actionId,
-            error: errorMessage,
-          },
-          'failed to execute scheduled action',
-        )
-      }
-    })
-  }
-
-  async performTakedown({
-    email,
-    action,
-    event,
-    modTool,
-    moderationTxn,
-    settingService,
-  }: {
-    email: { subject: string; content: string }
-    action: Selectable<ScheduledAction>
-    event: ModEventType
-    modTool: ModTool | undefined
-
-    moderationTxn: ModerationService
-    settingService: SettingService
-  }) {
-    const subject = new RepoSubject(action.did)
-
-    const status = await moderationTxn.getStatus(subject)
-
-    if (status?.takendown) {
-      throw new Error(`Account is already taken down`)
-    }
-
-    if (status?.tags?.length) {
-      const protectedTags = await getProtectedTags(
-        settingService,
-        this.serviceDid,
-      )
-
-      if (protectedTags) {
-        assertProtectedTagAction({
-          protectedTags,
-          subjectTags: status.tags,
-          actionAuthor: action.createdBy,
-          isAdmin: true,
-          isModerator: false,
-          isTriage: false,
-        })
-      }
-    }
-
-    // log the event which also applies the necessary state changes to moderation subject
-    const moderationEvent = await moderationTxn.logEvent({
-      event,
-      subject,
-      modTool,
-      createdBy: action.createdBy,
-    })
-
-    // register the takedown in event pusher
-    await moderationTxn.takedownRepo(
-      subject,
-      moderationEvent.event.id,
-      new Set(
-        moderationEvent.event.meta?.targetServices
-          ? `${moderationEvent.event.meta.targetServices}`.split(',')
-          : undefined,
-      ),
-    )
-
-    if (email.content && email.subject) {
-      let isDelivered = false
-      try {
-        await retryHttp(() =>
-          moderationTxn.sendEmail({
-            ...email,
-            recipientDid: action.did,
-          }),
-        )
-        isDelivered = true
-      } catch (err) {
-        dbLogger.error(
-          { err, did: action.did },
-          'failed to send takedown email',
-        )
-      }
-      await moderationTxn.logEvent({
-        event: {
-          content: email.content,
-          subjectLine: email.subject,
-          $type: 'tools.ozone.moderation.defs#modEventEmail',
-          comment: [
-            'Communication attached to scheduled action',
-            isDelivered ? '' : 'Email delivery failed',
-          ].join('.'),
-          isDelivered,
-        },
-        subject,
-        modTool,
-        createdBy: action.createdBy,
-      })
-    }
-
-    return moderationEvent
-  }
-
-  async findAndExecuteScheduledActions() {
-    const scheduledActionService = this.scheduledActionService(this.db)
-    const now = new Date()
-
-    const actionsToExecute =
-      await scheduledActionService.getPendingActionsToExecute(now)
-
-    for (const action of actionsToExecute) {
-      // For randomized execution, check if we should execute now or wait
-      if (action.randomizeExecution && action.executeAfter) {
-        const executeAfter = new Date(action.executeAfter)
-        // Default to a 30 second window for execution
-        const executeUntil = action.executeUntil
-          ? new Date(action.executeUntil)
-          : new Date(executeAfter.getTime() + 30 * SECOND)
-
-        // Only execute if we're past the earliest time
-        if (now < executeAfter) {
-          continue
-        }
-
-        // For randomized scheduling, randomly decide whether to execute now
-        // The probability increases as we get closer to the deadline
-        const timeRange = executeUntil.getTime() - executeAfter.getTime()
-        const timeElapsed = now.getTime() - executeAfter.getTime()
-        const executeProb = Math.min(timeElapsed / timeRange, 1)
-
-        // Execute with increasing probability as we approach the deadline
-        // Always execute if we're at or past the deadline
-        if (now < executeUntil && Math.random() > executeProb * 0.1) {
-          continue
-        }
-      }
-
-      await this.executeScheduledAction(action.id)
-    }
-  }
-}
-
-const getInterval = (): number => {
-  // Process scheduled actions every minute
-  const now = Date.now()
-  const intervalMs = MINUTE
-  const nextIteration = Math.ceil(now / intervalMs)
-  return nextIteration * intervalMs - now
-}