@atproto/ozone 0.2.8 → 0.2.11

package/src/daemon/stats-computer.ts

@@ -1,101 +0,0 @@
-import { sql } from 'kysely'
-import { MINUTE } from '@atproto/common'
-import { Database } from '../db/index.js'
-import { dbLogger } from '../logger.js'
-import { ReportStatsServiceCreator } from '../report/stats.js'
-
-// Stable lock ID for pg_try_advisory_lock across all instances
-const ADVISORY_LOCK_ID = 7_239_401
-
-/**
- * Background daemon that materializes report statistics on an interval (default is 15 minutes).
- *
- * Each cycle computes calendar-day snapshots: today's stats are recomputed (in-progress day),
- * and yesterday's snapshot is finalized if it wasn't already. Historical snapshots (completed
- * days) are write-once and never recomputed unless explicitly refreshed via the API.
- *
- * Query profile per cycle (assuming ~10K reports/day, 10 queues, 20 moderators, 9 type groups):
- * - 7 batched GROUP BY queries against the report table for today's date window
- *   (+ 7 more for yesterday if finalization is needed).
- *   Day-window queries scan ~10K rows. Pending-count queries use partial indexes
- *   (WHERE status != 'closed') so only scan open reports, not the full table.
- *   Expected: ~10-50ms per query, ~100-350ms total report-table time.
- * - ~40 lightweight reads against report_stat for freshness checks (small indexed table).
- * - ~40 lightweight writes to report_stat for upserts.
- *
- * Locking: Uses pg_try_advisory_lock to ensure only one instance materializes at a time
- * when running multiple containers. Advisory locks are cooperative, session-level locks —
- * they do NOT block any table reads, writes, row locks, or transactions from other sessions.
- * Normal application queries (report creation, moderation actions, API reads) are completely
- * unaffected. If another instance already holds the lock, this instance skips the cycle
- * immediately without blocking.
- */
-export class StatsComputer {
-  destroyed = false
-  processingPromise: Promise<void> = Promise.resolve()
-  timer?: NodeJS.Timeout
-
-  constructor(
-    private db: Database,
-    private reportStatsServiceCreator: ReportStatsServiceCreator,
-    /**
-     * Minutes between stats computer cycles.
-     * Defaults to 15. Minimum is 1.
-     * Set to -1 to disable the stats computer.
-     */
-    private intervalMinutes: number,
-  ) {}
-
-  get disabled() {
-    return this.intervalMinutes < 1
-  }
-
-  start() {
-    this.poll()
-  }
-
-  poll() {
-    if (this.destroyed || this.disabled) return
-    this.processingPromise = this.materializeStats()
-      .catch((err) => dbLogger.error({ err }, 'stats materialization errored'))
-      .finally(() => {
-        this.timer = setTimeout(
-          () => this.poll(),
-          this.intervalMinutes * MINUTE,
-        )
-      })
-  }
-
-  private async materializeStats() {
-    const lockResult = await sql<{
-      locked: boolean
-    }>`SELECT pg_try_advisory_lock(${ADVISORY_LOCK_ID}) as locked`.execute(
-      this.db.db,
-    )
-    const acquired = lockResult.rows[0]?.locked === true
-    if (!acquired) {
-      dbLogger.info(
-        'stats materialization skipped, another instance holds lock',
-      )
-      return
-    }
-
-    try {
-      const statsService = this.reportStatsServiceCreator(this.db)
-      await statsService.materializeAll()
-    } finally {
-      await sql`SELECT pg_advisory_unlock(${ADVISORY_LOCK_ID})`.execute(
-        this.db.db,
-      )
-    }
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    await this.processingPromise
-  }
-}

package/src/daemon/strike-expiry-processor.ts

@@ -1,95 +0,0 @@
-import { HOUR } from '@atproto/common'
-import { Database } from '../db/index.js'
-import { dbLogger } from '../logger.js'
-import { StrikeServiceCreator } from '../mod-service/strike.js'
-import { getJobCursor, initJobCursor, updateJobCursor } from './job-cursor.js'
-
-const JOB_NAME = 'strike_expiry'
-
-export class StrikeExpiryProcessor {
-  destroyed = false
-  processingPromise: Promise<void> = Promise.resolve()
-  timer?: NodeJS.Timeout
-
-  constructor(
-    private db: Database,
-    private strikeServiceCreator: StrikeServiceCreator,
-  ) {}
-
-  start() {
-    this.initializeCursor().then(() => this.poll())
-  }
-
-  poll() {
-    if (this.destroyed) return
-    this.processingPromise = this.processExpiredStrikes()
-      .catch((err) =>
-        dbLogger.error({ err }, 'strike expiry processing errored'),
-      )
-      .finally(() => {
-        this.timer = setTimeout(() => this.poll(), getInterval())
-      })
-  }
-
-  async destroy() {
-    this.destroyed = true
-    if (this.timer) {
-      clearTimeout(this.timer)
-      this.timer = undefined
-    }
-    await this.processingPromise
-  }
-
-  async initializeCursor() {
-    await initJobCursor(this.db, JOB_NAME)
-  }
-
-  async getCursor(): Promise<string | null> {
-    return getJobCursor(this.db, JOB_NAME)
-  }
-
-  async updateCursor(cursor: string): Promise<void> {
-    await updateJobCursor(this.db, JOB_NAME, cursor)
-  }
-
-  async processExpiredStrikes() {
-    const now = new Date()
-    const strikeService = this.strikeServiceCreator(this.db)
-    const lastProcessedAt = await this.getCursor()
-    const affectedSubjects = await strikeService.getExpiredStrikeSubjects(
-      lastProcessedAt || undefined,
-    )
-
-    if (!affectedSubjects.length) {
-      dbLogger.info('no expired strikes to process')
-      await this.updateCursor(now.toISOString())
-      return
-    }
-
-    dbLogger.info(
-      { count: affectedSubjects.length },
-      'processing subjects with expired strikes',
-    )
-
-    await Promise.all(
-      affectedSubjects.map(({ subjectDid }) => {
-        return strikeService.updateSubjectStrikeCount(subjectDid)
-      }),
-    )
-
-    await this.updateCursor(now.toISOString())
-
-    dbLogger.info(
-      { processed: affectedSubjects.length },
-      'strike expiry processing completed',
-    )
-  }
-}
-
-const getInterval = (): number => {
-  // Run every hour, synchronized to the hour boundary
-  const now = Date.now()
-  const intervalMs = HOUR
-  const nextIteration = Math.ceil(now / intervalMs)
-  return nextIteration * intervalMs - now
-}

package/src/daemon/team-profile-synchronizer.ts

@@ -1,15 +0,0 @@
-import { HOUR } from '@atproto/common'
-import { BackgroundQueue, PeriodicBackgroundTask } from '../background.js'
-import { TeamService } from '../team/index.js'
-
-export class TeamProfileSynchronizer extends PeriodicBackgroundTask {
-  constructor(
-    backgroundQueue: BackgroundQueue,
-    teamService: TeamService,
-    interval = 24 * HOUR,
-  ) {
-    super(backgroundQueue, interval, async () => {
-      await teamService.syncMemberProfiles()
-    })
-  }
-}

package/src/daemon/verification-listener.ts

@@ -1,169 +0,0 @@
-import { lexicons } from '@atproto/api'
-import { BackgroundQueue } from '../background.js'
-import { Database } from '../db/index.js'
-import { CommitCreateEvent, Jetstream } from '../jetstream/service.js'
-import { verificationLogger } from '../logger.js'
-import { VerificationService } from '../verification/service.js'
-
-type VerificationRecord = {
-  subject: string
-  handle: string
-  displayName: string
-  createdAt: string
-}
-
-export class VerificationListener {
-  destroyed = false
-  private cursor?: number
-  private jetstream: Jetstream | null = null
-  private collection = 'app.bsky.graph.verification'
-  public backgroundQueue = new BackgroundQueue(this.db, { concurrency: 1 })
-  private verificationService = VerificationService.creator()(this.db)
-
-  constructor(
-    private db: Database,
-    private jetstreamUrl: string,
-    private verifierIssuersToIndex?: string[],
-  ) {}
-
-  // When the queue has capacity, this method returns true which means we can continue to handle events
-  // otherwise, it will close jetstream connection and wait for all previously queued events to be processed first
-  // and then start jetstream listener again before returning false. At that point, the previous listeners should
-  // have updates the cursor in db to the last processed event and the new listener will start from that cursor
-  async ensureCoolDown() {
-    const { waitingCount, runningCount } = this.backgroundQueue.getStats()
-    if (waitingCount > 50 || runningCount > 50) {
-      verificationLogger.warn(`Background queue is full, pausing listener`)
-      this.jetstream?.close()
-      await this.backgroundQueue.processAll()
-      await this.start()
-      return false
-    }
-    return true
-  }
-
-  handleNewVerification(
-    issuer: string,
-    uri: string,
-    cid: string,
-    record: VerificationRecord,
-    cursor: number,
-  ) {
-    this.backgroundQueue.add(async () => {
-      try {
-        const { subject, handle, displayName, createdAt } = record
-        await this.verificationService.create([
-          { uri, cid, issuer, subject, handle, displayName, createdAt },
-        ])
-        await this.updateCursor(cursor)
-      } catch (err) {
-        verificationLogger.error(
-          err,
-          'Error handling verification create event',
-        )
-      }
-    })
-  }
-
-  handleDeletedVerification(uri: string, cursor: number) {
-    this.backgroundQueue.add(async () => {
-      try {
-        await this.verificationService.markRevoked({
-          uris: [uri],
-        })
-        await this.updateCursor(cursor)
-      } catch (err) {
-        verificationLogger.error(
-          err,
-          'Error handling verification delete event',
-        )
-      }
-    })
-  }
-
-  async getCursor() {
-    await this.verificationService.createFirehoseCursor()
-    const cursor = await this.verificationService.getFirehoseCursor()
-    if (cursor) {
-      this.cursor = cursor
-    }
-    return this.cursor
-  }
-
-  async updateCursor(cursor: number) {
-    // Assuming cursors are always incremental, if we have processed an event with higher value cursor, let's not update to a lower value
-    if (this.cursor && this.cursor >= cursor) {
-      return
-    }
-
-    // This will only update if the cursor is higher than the current one in db
-    const updatedCursor =
-      await this.verificationService.updateFirehoseCursor(cursor)
-
-    if (updatedCursor) {
-      this.cursor = updatedCursor
-    }
-  }
-
-  async start() {
-    await this.getCursor()
-
-    this.jetstream = new Jetstream({
-      endpoint: this.jetstreamUrl,
-      cursor: this.cursor || undefined,
-      wantedCollections: [this.collection],
-      wantedDids: this.verifierIssuersToIndex?.length
-        ? this.verifierIssuersToIndex
-        : undefined,
-    })
-
-    await this.jetstream.start({
-      onCreate: {
-        [this.collection]: async (e: CommitCreateEvent<VerificationRecord>) => {
-          const recordValidity = lexicons.validate(
-            this.collection,
-            e.commit.record,
-          )
-
-          if (!recordValidity.success) {
-            verificationLogger.error(
-              recordValidity.error,
-              'Invalid verification record in the firehose',
-            )
-            return
-          }
-
-          const hasCapacity = await this.ensureCoolDown()
-          if (hasCapacity) {
-            const issuer = e.did
-            const { record, rkey, collection, cid } = e.commit
-            const uri = `at://${issuer}/${collection}/${rkey}`
-            this.handleNewVerification(issuer, uri, cid, record, e.time_us)
-          }
-        },
-      },
-      onDelete: {
-        [this.collection]: async (e) => {
-          const hasCapacity = await this.ensureCoolDown()
-          if (hasCapacity) {
-            this.handleDeletedVerification(
-              `at://${e.did}/${e.commit.collection}/${e.commit.rkey}`,
-              e.time_us,
-            )
-          }
-        },
-      },
-    })
-  }
-
-  async stop() {
-    if (!this.destroyed) {
-      this.destroyed = true
-      try {
-        await this.jetstream?.close()
-      } finally {
-        await this.backgroundQueue.destroy()
-      }
-    }
-  }
-}

package/src/db/index.ts

@@ -1,203 +0,0 @@
-import assert from 'node:assert'
-import { EventEmitter } from 'node:events'
-import {
-  Kysely,
-  KyselyPlugin,
-  PluginTransformQueryArgs,
-  PluginTransformResultArgs,
-  PostgresDialect,
-  QueryResult,
-  RootOperationNode,
-  UnknownRow,
-} from 'kysely'
-import { Migrator } from 'kysely/migration'
-// eslint-disable-next-line import/default
-import pg from 'pg'
-// eslint-disable-next-line import/no-named-as-default-member
-const { Pool: PgPool, types: pgTypes } = pg
-type PgPool = InstanceType<typeof PgPool>
-import type TypedEmitter from 'typed-emitter'
-import { dbLogger } from '../logger.js'
-import * as migrations from './migrations/index.js'
-import { CtxMigrationProvider } from './migrations/provider.js'
-import { DatabaseSchema, DatabaseSchemaType } from './schema/index.js'
-import { PgOptions } from './types.js'
-
-export class Database {
-  pool: PgPool
-  db: DatabaseSchema
-  migrator: Migrator
-  txEvt = new EventEmitter() as TxnEmitter
-  destroyed = false
-  isPrimary = false
-
-  constructor(
-    public opts: PgOptions,
-    instances?: { db: DatabaseSchema; pool: PgPool },
-  ) {
-    // if instances are provided, use those
-    if (instances) {
-      this.db = instances.db
-      this.pool = instances.pool
-    } else {
-      // else create a pool & connect
-      const { schema, url } = opts
-      const pool =
-        opts.pool ??
-        new PgPool({
-          connectionString: url,
-          max: opts.poolSize,
-          maxUses: opts.poolMaxUses,
-          idleTimeoutMillis: opts.poolIdleTimeoutMs,
-        })
-
-      // Select count(*) and other pg bigints as js integer
-      pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
-
-      // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
-      if (schema && !/^[a-z_]+$/i.test(schema)) {
-        throw new Error(
-          `Postgres schema must only contain [A-Za-z_]: ${schema}`,
-        )
-      }
-
-      pool.on('error', onPoolError)
-      pool.on('connect', (client) => {
-        client.on('error', onClientError)
-        // Used for trigram indexes, e.g. on actor search
-        client.query('SET pg_trgm.word_similarity_threshold TO .4;')
-        if (schema) {
-          // Shared objects such as extensions will go in the public schema
-          client.query(`SET search_path TO "${schema}",public;`)
-        }
-      })
-
-      this.pool = pool
-      this.db = new Kysely<DatabaseSchemaType>({
-        dialect: new PostgresDialect({ pool }),
-      })
-    }
-
-    this.migrator = new Migrator({
-      db: this.db,
-      migrationTableSchema: opts.schema,
-      provider: new CtxMigrationProvider(migrations, 'pg'),
-    })
-  }
-
-  get schema(): string | undefined {
-    return this.opts.schema
-  }
-
-  get isTransaction() {
-    return this.db.isTransaction
-  }
-
-  assertTransaction() {
-    assert(this.isTransaction, 'Transaction required')
-  }
-
-  assertNotTransaction() {
-    assert(!this.isTransaction, 'Cannot be in a transaction')
-  }
-
-  async transaction<T>(fn: (db: Database) => Promise<T>): Promise<T> {
-    const leakyTxPlugin = new LeakyTxPlugin()
-    const { dbTxn, txRes } = await this.db
-      .withPlugin(leakyTxPlugin)
-      .transaction()
-      .execute(async (txn) => {
-        const dbTxn = new Database(this.opts, {
-          db: txn,
-          pool: this.pool,
-        })
-        const txRes = await fn(dbTxn)
-          .catch(async (err) => {
-            leakyTxPlugin.endTx()
-            // ensure that all in-flight queries are flushed & the connection is open
-            await dbTxn.db.getExecutor().provideConnection(noopAsync)
-            throw err
-          })
-          .finally(() => leakyTxPlugin.endTx())
-        return { dbTxn, txRes }
-      })
-    dbTxn?.txEvt.emit('commit')
-    return txRes
-  }
-
-  onCommit(fn: () => void) {
-    this.assertTransaction()
-    this.txEvt.once('commit', fn)
-  }
-
-  async close(): Promise<void> {
-    if (this.destroyed) return
-    await this.db.destroy()
-    this.destroyed = true
-  }
-
-  async migrateToOrThrow(migration: string) {
-    if (this.schema) {
-      await this.db.schema.createSchema(this.schema).ifNotExists().execute()
-    }
-    const { error, results } = await this.migrator.migrateTo(migration)
-    if (error) {
-      throw error
-    }
-    if (!results) {
-      throw new Error('An unknown failure occurred while migrating')
-    }
-    return results
-  }
-
-  async migrateToLatestOrThrow() {
-    if (this.schema) {
-      await this.db.schema.createSchema(this.schema).ifNotExists().execute()
-    }
-    const { error, results } = await this.migrator.migrateToLatest()
-    if (error) {
-      throw error
-    }
-    if (!results) {
-      throw new Error('An unknown failure occurred while migrating')
-    }
-    return results
-  }
-}
-
-export default Database
-
-const onPoolError = (err: Error) => dbLogger.error({ err }, 'db pool error')
-const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
-
-// utils
-// -------
-
-class LeakyTxPlugin implements KyselyPlugin {
-  private txOver = false
-
-  endTx() {
-    this.txOver = true
-  }
-
-  transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
-    if (this.txOver) {
-      throw new Error('tx already failed')
-    }
-    return args.node
-  }
-
-  async transformResult(
-    args: PluginTransformResultArgs,
-  ): Promise<QueryResult<UnknownRow>> {
-    return args.result
-  }
-}
-
-type TxnEmitter = TypedEmitter.default<TxnEvents>
-
-type TxnEvents = {
-  commit: () => void
-}
-
-const noopAsync = async () => {}