@atproto/ozone 0.0.11 → 0.0.13

package/src/mod-service/index.ts

@@ -43,6 +43,7 @@ import { BlobPushEvent } from '../db/schema/blob_push_event'
 import { BackgroundQueue } from '../background'
 import { EventPusher } from '../daemon'
 import { jsonb } from '../db/types'
+import { LabelChannel } from '../db/schema/label'
 
 export type ModerationServiceCreator = (db: Database) => ModerationService
 
@@ -438,24 +439,22 @@ export class ModerationService {
       takedownRef,
     }))
 
-    const [repoEvts] = await Promise.all([
-      this.db.db
-        .insertInto('repo_push_event')
-        .values(values)
-        .onConflict((oc) =>
-          oc.columns(['subjectDid', 'eventType']).doUpdateSet({
-            takedownRef,
-            confirmedAt: null,
-            attempts: 0,
-            lastAttempted: null,
-          }),
-        )
-        .returning('id')
-        .execute(),
-      this.formatAndCreateLabels(subject.did, null, {
-        create: [UNSPECCED_TAKEDOWN_LABEL],
-      }),
-    ])
+    const repoEvts = await this.db.db
+      .insertInto('repo_push_event')
+      .values(values)
+      .onConflict((oc) =>
+        oc.columns(['subjectDid', 'eventType']).doUpdateSet({
+          takedownRef,
+          confirmedAt: null,
+          attempts: 0,
+          lastAttempted: null,
+        }),
+      )
+      .returning('id')
+      .execute()
+    await this.formatAndCreateLabels(subject.did, null, {
+      create: [UNSPECCED_TAKEDOWN_LABEL],
+    })
 
     this.db.onCommit(() => {
       this.backgroundQueue.add(async () => {
@@ -467,23 +466,21 @@ export class ModerationService {
   }
 
   async reverseTakedownRepo(subject: RepoSubject) {
-    const [repoEvts] = await Promise.all([
-      this.db.db
-        .updateTable('repo_push_event')
-        .where('eventType', 'in', TAKEDOWNS)
-        .where('subjectDid', '=', subject.did)
-        .set({
-          takedownRef: null,
-          confirmedAt: null,
-          attempts: 0,
-          lastAttempted: null,
-        })
-        .returning('id')
-        .execute(),
-      this.formatAndCreateLabels(subject.did, null, {
-        negate: [UNSPECCED_TAKEDOWN_LABEL],
-      }),
-    ])
+    const repoEvts = await this.db.db
+      .updateTable('repo_push_event')
+      .where('eventType', 'in', TAKEDOWNS)
+      .where('subjectDid', '=', subject.did)
+      .set({
+        takedownRef: null,
+        confirmedAt: null,
+        attempts: 0,
+        lastAttempted: null,
+      })
+      .returning('id')
+      .execute()
+    await this.formatAndCreateLabels(subject.did, null, {
+      negate: [UNSPECCED_TAKEDOWN_LABEL],
+    })
 
     this.db.onCommit(() => {
       this.backgroundQueue.add(async () => {
@@ -509,22 +506,22 @@ export class ModerationService {
     if (blobCids && blobCids.length > 0) {
       labels.push(UNSPECCED_TAKEDOWN_BLOBS_LABEL)
     }
-    const [recordEvts] = await Promise.all([
-      this.db.db
-        .insertInto('record_push_event')
-        .values(values)
-        .onConflict((oc) =>
-          oc.columns(['subjectUri', 'eventType']).doUpdateSet({
-            takedownRef,
-            confirmedAt: null,
-            attempts: 0,
-            lastAttempted: null,
-          }),
-        )
-        .returning('id')
-        .execute(),
-      this.formatAndCreateLabels(subject.uri, subject.cid, { create: labels }),
-    ])
+    const recordEvts = await this.db.db
+      .insertInto('record_push_event')
+      .values(values)
+      .onConflict((oc) =>
+        oc.columns(['subjectUri', 'eventType']).doUpdateSet({
+          takedownRef,
+          confirmedAt: null,
+          attempts: 0,
+          lastAttempted: null,
+        }),
+      )
+      .returning('id')
+      .execute()
+    await this.formatAndCreateLabels(subject.uri, subject.cid, {
+      create: labels,
+    })
 
     this.db.onCommit(() => {
       this.backgroundQueue.add(async () => {
@@ -579,29 +576,31 @@ export class ModerationService {
     if (blobCids && blobCids.length > 0) {
       labels.push(UNSPECCED_TAKEDOWN_BLOBS_LABEL)
     }
-    const [recordEvts] = await Promise.all([
-      this.db.db
-        .updateTable('record_push_event')
-        .where('eventType', 'in', TAKEDOWNS)
-        .where('subjectDid', '=', subject.did)
-        .where('subjectUri', '=', subject.uri)
-        .set({
-          takedownRef: null,
-          confirmedAt: null,
-          attempts: 0,
-          lastAttempted: null,
+    const recordEvts = await this.db.db
+      .updateTable('record_push_event')
+      .where('eventType', 'in', TAKEDOWNS)
+      .where('subjectDid', '=', subject.did)
+      .where('subjectUri', '=', subject.uri)
+      .set({
+        takedownRef: null,
+        confirmedAt: null,
+        attempts: 0,
+        lastAttempted: null,
+      })
+      .returning('id')
+      .execute()
+    await this.formatAndCreateLabels(subject.uri, subject.cid, {
+      negate: labels,
+    }),
+      this.db.onCommit(() => {
+        this.backgroundQueue.add(async () => {
+          await Promise.all(
+            recordEvts.map((evt) =>
+              this.eventPusher.attemptRecordEvent(evt.id),
+            ),
+          )
         })
-        .returning('id')
-        .execute(),
-      this.formatAndCreateLabels(subject.uri, subject.cid, { negate: labels }),
-    ])
-    this.db.onCommit(() => {
-      this.backgroundQueue.add(async () => {
-        await Promise.all(
-          recordEvts.map((evt) => this.eventPusher.attemptRecordEvent(evt.id)),
-        )
       })
-    })
 
     if (blobCids && blobCids.length > 0) {
       const blobEvts = await this.db.db
@@ -858,12 +857,14 @@ export class ModerationService {
       neg: !!l.neg,
     }))
     const { ref } = this.db.db.dynamic
+    await sql`notify ${ref(LabelChannel)}`.execute(this.db.db)
     const excluded = (col: string) => ref(`excluded.${col}`)
     await this.db.db
       .insertInto('label')
       .values(dbVals)
       .onConflict((oc) =>
         oc.columns(['src', 'uri', 'cid', 'val']).doUpdateSet({
+          id: sql`${excluded('id')}`,
           neg: sql`${excluded('neg')}`,
           cts: sql`${excluded('cts')}`,
         }),

package/src/mod-service/status.ts

@@ -179,6 +179,9 @@ export const adjustModerationSubjectStatus = async (
     subjectStatus.appealed = true
     newStatus.lastAppealedAt = createdAt
     subjectStatus.lastAppealedAt = createdAt
+    // Set reviewState to escalated when appeal events are emitted
+    subjectStatus.reviewState = REVIEWESCALATED
+    newStatus.reviewState = REVIEWESCALATED
   }
 
   if (

package/src/mod-service/util.ts

@@ -0,0 +1,17 @@
+import { LabelRow } from '../db/schema/label'
+import { Label } from '../lexicon/types/com/atproto/label/defs'
+
+export const formatLabel = (row: LabelRow): Label => {
+  const label: Label = {
+    src: row.src,
+    uri: row.uri,
+    val: row.val,
+    neg: row.neg,
+    cts: row.cts,
+  }
+  if (row.cid !== '') {
+    // @NOTE avoiding undefined values on label, which dag-cbor chokes on when serializing.
+    label.cid = row.cid
+  }
+  return label
+}

package/src/mod-service/views.ts

@@ -24,6 +24,7 @@ import {
 } from './types'
 import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
 import { subjectFromEventRow, subjectFromStatusRow } from './subject'
+import { formatLabel } from './util'
 
 export type AppviewAuth = () => Promise<
   | {
@@ -403,11 +404,7 @@ export class ModerationViews {
       .if(!includeNeg, (qb) => qb.where('neg', '=', false))
       .selectAll()
       .execute()
-    return res.map((l) => ({
-      ...l,
-      cid: l.cid === '' ? undefined : l.cid,
-      neg: l.neg,
-    }))
+    return res.map((l) => formatLabel(l))
   }
 
   async getSubjectStatus(

package/src/sequencer/index.ts

@@ -0,0 +1,2 @@
+export * from './sequencer'
+export * from './outbox'

package/src/sequencer/outbox.ts

@@ -0,0 +1,122 @@
+import { AsyncBuffer, AsyncBufferFullError } from '@atproto/common'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import { Sequencer, LabelsEvt } from './sequencer'
+
+export type OutboxOpts = {
+  maxBufferSize: number
+}
+
+export class Outbox {
+  private caughtUp = false
+  lastSeen = -1
+
+  cutoverBuffer: LabelsEvt[]
+  outBuffer: AsyncBuffer<LabelsEvt>
+
+  constructor(public sequencer: Sequencer, opts: Partial<OutboxOpts> = {}) {
+    const { maxBufferSize = 500 } = opts
+    this.cutoverBuffer = []
+    this.outBuffer = new AsyncBuffer<LabelsEvt>(maxBufferSize)
+  }
+
+  // event stream occurs in 3 phases
+  // 1. backfill events: events that have been added to the DB since the last time a connection was open.
+  // The outbox is not yet listening for new events from the sequencer
+  // 2. cutover: the outbox has caught up with where the sequencer purports to be,
+  // but the sequencer might already be halfway through sending out a round of updates.
+  // Therefore, we start accepting the sequencer's events in a buffer, while making our own request to the
+  // database to ensure we're caught up. We then dedupe the query & the buffer & stream the events in order
+  // 3. streaming: we're all caught up on historic state, so the sequencer outputs events and we
+  // immediately yield them
+  async *events(
+    backfillCursor?: number,
+    signal?: AbortSignal,
+  ): AsyncGenerator<LabelsEvt> {
+    // catch up as much as we can
+    if (backfillCursor !== undefined) {
+      for await (const evt of this.getBackfill(backfillCursor)) {
+        if (signal?.aborted) return
+        this.lastSeen = evt.seq
+        yield evt
+      }
+    } else {
+      // if not backfill, we don't need to cutover, just start streaming
+      this.caughtUp = true
+    }
+
+    // streams updates from sequencer, but buffers them for cutover as it makes a last request
+
+    const addToBuffer = (evts) => {
+      if (this.caughtUp) {
+        this.outBuffer.pushMany(evts)
+      } else {
+        this.cutoverBuffer = [...this.cutoverBuffer, ...evts]
+      }
+    }
+
+    if (!signal?.aborted) {
+      this.sequencer.on('events', addToBuffer)
+    }
+    signal?.addEventListener('abort', () =>
+      this.sequencer.off('events', addToBuffer),
+    )
+
+    const cutover = async () => {
+      // only need to perform cutover if we've been backfilling
+      if (backfillCursor !== undefined) {
+        const cutoverEvts = await this.sequencer.requestLabelRange({
+          earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
+        })
+        this.outBuffer.pushMany(cutoverEvts)
+        // dont worry about dupes, we ensure order on yield
+        this.outBuffer.pushMany(this.cutoverBuffer)
+        this.caughtUp = true
+        this.cutoverBuffer = []
+      } else {
+        this.caughtUp = true
+      }
+    }
+    cutover()
+
+    while (true) {
+      try {
+        for await (const evt of this.outBuffer.events()) {
+          if (signal?.aborted) return
+          if (evt.seq > this.lastSeen) {
+            this.lastSeen = evt.seq
+            yield evt
+          }
+        }
+      } catch (err) {
+        if (err instanceof AsyncBufferFullError) {
+          throw new InvalidRequestError(
+            'Stream consumer too slow',
+            'ConsumerTooSlow',
+          )
+        } else {
+          throw err
+        }
+      }
+    }
+  }
+
+  // yields only historical events
+  async *getBackfill(backfillCursor: number) {
+    const PAGE_SIZE = 500
+    while (true) {
+      const evts = await this.sequencer.requestLabelRange({
+        earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
+        limit: PAGE_SIZE,
+      })
+      for (const evt of evts) {
+        yield evt
+      }
+      // if we're within half a pagesize of the sequencer, we call it good & switch to cutover
+      const seqCursor = this.sequencer.lastSeen ?? -1
+      if (seqCursor - this.lastSeen < PAGE_SIZE / 2) break
+      if (evts.length < 1) break
+    }
+  }
+}
+
+export default Outbox

package/src/sequencer/sequencer.ts

@@ -0,0 +1,143 @@
+import EventEmitter from 'events'
+import TypedEmitter from 'typed-emitter'
+import { seqLogger as log } from '../logger'
+import Database from '../db'
+import { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels'
+import { LabelChannel, Label as LabelTable } from '../db/schema/label'
+import { Selectable } from 'kysely'
+import { formatLabel } from '../mod-service/util'
+import { PoolClient } from 'pg'
+
+export type { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels'
+type LabelRow = Selectable<LabelTable>
+
+export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
+  destroyed = false
+  pollPromise: Promise<void> | undefined
+  queued = false
+  conn: PoolClient | undefined
+
+  constructor(public db: Database, public lastSeen = 0) {
+    super()
+    // note: this does not err when surpassed, just prints a warning to stderr
+    this.setMaxListeners(100)
+  }
+
+  async start() {
+    const curr = await this.curr()
+    this.lastSeen = curr ?? 0
+    this.poll()
+    this.conn = await this.db.pool.connect()
+    this.conn.query(`listen ${LabelChannel}`) // if this errors, unhandled rejection should cause process to exit
+    this.conn.on('notification', (notif) => {
+      if (notif.channel === LabelChannel) {
+        this.poll()
+      }
+    })
+  }
+
+  async destroy() {
+    if (this.destroyed) return
+    this.destroyed = true
+    if (this.conn) {
+      this.conn.release()
+      this.conn = undefined
+    }
+    if (this.pollPromise) {
+      await this.pollPromise
+    }
+    this.emit('close')
+  }
+
+  async curr(): Promise<number | null> {
+    const got = await this.db.db
+      .selectFrom('label')
+      .selectAll()
+      .orderBy('id', 'desc')
+      .limit(1)
+      .executeTakeFirst()
+    return got?.id ?? null
+  }
+
+  async next(cursor: number): Promise<LabelRow | null> {
+    const got = await this.db.db
+      .selectFrom('label')
+      .selectAll()
+      .where('id', '>', cursor)
+      .limit(1)
+      .orderBy('id', 'asc')
+      .executeTakeFirst()
+    return got || null
+  }
+
+  async requestLabelRange(opts: {
+    earliestId?: number
+    limit?: number
+  }): Promise<LabelsEvt[]> {
+    const { earliestId, limit } = opts
+
+    let seqQb = this.db.db.selectFrom('label').selectAll().orderBy('id', 'asc')
+    if (earliestId !== undefined) {
+      seqQb = seqQb.where('id', '>', earliestId)
+    }
+    if (limit !== undefined) {
+      seqQb = seqQb.limit(limit)
+    }
+
+    const rows = await seqQb.execute()
+    if (rows.length < 1) {
+      return []
+    }
+
+    const evts: LabelsEvt[] = []
+    for (const row of rows) {
+      evts.push({
+        seq: row.id,
+        labels: [formatLabel(row)],
+      })
+    }
+
+    return evts
+  }
+
+  private poll() {
+    if (this.destroyed) return
+    if (this.pollPromise) {
+      this.queued = true
+      return
+    }
+    this.queued = false
+    this.pollPromise = this.requestLabelRange({
+      earliestId: this.lastSeen,
+      limit: 500,
+    })
+      .then((evts) => {
+        this.emit('events', evts)
+        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen
+        if (evts.length > 0) {
+          this.queued = true
+        }
+      })
+      .catch((err) => {
+        log.error(
+          { err, lastSeen: this.lastSeen },
+          'sequencer failed to poll db',
+        )
+      })
+      .finally(() => {
+        this.pollPromise = undefined
+        if (this.queued) {
+          this.poll()
+        }
+      })
+  }
+}
+
+type SequencerEvents = {
+  events: (evts: LabelsEvt[]) => void
+  close: () => void
+}
+
+export type SequencerEmitter = TypedEmitter<SequencerEvents>
+
+export default Sequencer