@atproto/ozone 0.0.12 → 0.0.14

package/src/mod-service/views.ts

@@ -24,6 +24,7 @@ import {
 } from './types'
 import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
 import { subjectFromEventRow, subjectFromStatusRow } from './subject'
+import { formatLabel } from './util'
 
 export type AppviewAuth = () => Promise<
   | {
@@ -403,11 +404,7 @@ export class ModerationViews {
       .if(!includeNeg, (qb) => qb.where('neg', '=', false))
       .selectAll()
       .execute()
-    return res.map((l) => ({
-      ...l,
-      cid: l.cid === '' ? undefined : l.cid,
-      neg: l.neg,
-    }))
+    return res.map((l) => formatLabel(l))
   }
 
   async getSubjectStatus(

package/src/sequencer/index.ts

@@ -0,0 +1,2 @@
+export * from './sequencer'
+export * from './outbox'

package/src/sequencer/outbox.ts

@@ -0,0 +1,122 @@
+import { AsyncBuffer, AsyncBufferFullError } from '@atproto/common'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import { Sequencer, LabelsEvt } from './sequencer'
+
+export type OutboxOpts = {
+  maxBufferSize: number
+}
+
+export class Outbox {
+  private caughtUp = false
+  lastSeen = -1
+
+  cutoverBuffer: LabelsEvt[]
+  outBuffer: AsyncBuffer<LabelsEvt>
+
+  constructor(public sequencer: Sequencer, opts: Partial<OutboxOpts> = {}) {
+    const { maxBufferSize = 500 } = opts
+    this.cutoverBuffer = []
+    this.outBuffer = new AsyncBuffer<LabelsEvt>(maxBufferSize)
+  }
+
+  // event stream occurs in 3 phases
+  // 1. backfill events: events that have been added to the DB since the last time a connection was open.
+  // The outbox is not yet listening for new events from the sequencer
+  // 2. cutover: the outbox has caught up with where the sequencer purports to be,
+  // but the sequencer might already be halfway through sending out a round of updates.
+  // Therefore, we start accepting the sequencer's events in a buffer, while making our own request to the
+  // database to ensure we're caught up. We then dedupe the query & the buffer & stream the events in order
+  // 3. streaming: we're all caught up on historic state, so the sequencer outputs events and we
+  // immediately yield them
+  async *events(
+    backfillCursor?: number,
+    signal?: AbortSignal,
+  ): AsyncGenerator<LabelsEvt> {
+    // catch up as much as we can
+    if (backfillCursor !== undefined) {
+      for await (const evt of this.getBackfill(backfillCursor)) {
+        if (signal?.aborted) return
+        this.lastSeen = evt.seq
+        yield evt
+      }
+    } else {
+      // if not backfill, we don't need to cutover, just start streaming
+      this.caughtUp = true
+    }
+
+    // streams updates from sequencer, but buffers them for cutover as it makes a last request
+
+    const addToBuffer = (evts) => {
+      if (this.caughtUp) {
+        this.outBuffer.pushMany(evts)
+      } else {
+        this.cutoverBuffer = [...this.cutoverBuffer, ...evts]
+      }
+    }
+
+    if (!signal?.aborted) {
+      this.sequencer.on('events', addToBuffer)
+    }
+    signal?.addEventListener('abort', () =>
+      this.sequencer.off('events', addToBuffer),
+    )
+
+    const cutover = async () => {
+      // only need to perform cutover if we've been backfilling
+      if (backfillCursor !== undefined) {
+        const cutoverEvts = await this.sequencer.requestLabelRange({
+          earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
+        })
+        this.outBuffer.pushMany(cutoverEvts)
+        // dont worry about dupes, we ensure order on yield
+        this.outBuffer.pushMany(this.cutoverBuffer)
+        this.caughtUp = true
+        this.cutoverBuffer = []
+      } else {
+        this.caughtUp = true
+      }
+    }
+    cutover()
+
+    while (true) {
+      try {
+        for await (const evt of this.outBuffer.events()) {
+          if (signal?.aborted) return
+          if (evt.seq > this.lastSeen) {
+            this.lastSeen = evt.seq
+            yield evt
+          }
+        }
+      } catch (err) {
+        if (err instanceof AsyncBufferFullError) {
+          throw new InvalidRequestError(
+            'Stream consumer too slow',
+            'ConsumerTooSlow',
+          )
+        } else {
+          throw err
+        }
+      }
+    }
+  }
+
+  // yields only historical events
+  async *getBackfill(backfillCursor: number) {
+    const PAGE_SIZE = 500
+    while (true) {
+      const evts = await this.sequencer.requestLabelRange({
+        earliestId: this.lastSeen > -1 ? this.lastSeen : backfillCursor,
+        limit: PAGE_SIZE,
+      })
+      for (const evt of evts) {
+        yield evt
+      }
+      // if we're within half a pagesize of the sequencer, we call it good & switch to cutover
+      const seqCursor = this.sequencer.lastSeen ?? -1
+      if (seqCursor - this.lastSeen < PAGE_SIZE / 2) break
+      if (evts.length < 1) break
+    }
+  }
+}
+
+export default Outbox

package/src/sequencer/sequencer.ts

@@ -0,0 +1,143 @@
+import EventEmitter from 'events'
+import TypedEmitter from 'typed-emitter'
+import { seqLogger as log } from '../logger'
+import Database from '../db'
+import { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels'
+import { LabelChannel, Label as LabelTable } from '../db/schema/label'
+import { Selectable } from 'kysely'
+import { formatLabel } from '../mod-service/util'
+import { PoolClient } from 'pg'
+
+export type { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels'
+type LabelRow = Selectable<LabelTable>
+
+export class Sequencer extends (EventEmitter as new () => SequencerEmitter) {
+  destroyed = false
+  pollPromise: Promise<void> | undefined
+  queued = false
+  conn: PoolClient | undefined
+
+  constructor(public db: Database, public lastSeen = 0) {
+    super()
+    // note: this does not err when surpassed, just prints a warning to stderr
+    this.setMaxListeners(100)
+  }
+
+  async start() {
+    const curr = await this.curr()
+    this.lastSeen = curr ?? 0
+    this.poll()
+    this.conn = await this.db.pool.connect()
+    this.conn.query(`listen ${LabelChannel}`) // if this errors, unhandled rejection should cause process to exit
+    this.conn.on('notification', (notif) => {
+      if (notif.channel === LabelChannel) {
+        this.poll()
+      }
+    })
+  }
+
+  async destroy() {
+    if (this.destroyed) return
+    this.destroyed = true
+    if (this.conn) {
+      this.conn.release()
+      this.conn = undefined
+    }
+    if (this.pollPromise) {
+      await this.pollPromise
+    }
+    this.emit('close')
+  }
+
+  async curr(): Promise<number | null> {
+    const got = await this.db.db
+      .selectFrom('label')
+      .selectAll()
+      .orderBy('id', 'desc')
+      .limit(1)
+      .executeTakeFirst()
+    return got?.id ?? null
+  }
+
+  async next(cursor: number): Promise<LabelRow | null> {
+    const got = await this.db.db
+      .selectFrom('label')
+      .selectAll()
+      .where('id', '>', cursor)
+      .limit(1)
+      .orderBy('id', 'asc')
+      .executeTakeFirst()
+    return got || null
+  }
+
+  async requestLabelRange(opts: {
+    earliestId?: number
+    limit?: number
+  }): Promise<LabelsEvt[]> {
+    const { earliestId, limit } = opts
+
+    let seqQb = this.db.db.selectFrom('label').selectAll().orderBy('id', 'asc')
+    if (earliestId !== undefined) {
+      seqQb = seqQb.where('id', '>', earliestId)
+    }
+    if (limit !== undefined) {
+      seqQb = seqQb.limit(limit)
+    }
+
+    const rows = await seqQb.execute()
+    if (rows.length < 1) {
+      return []
+    }
+
+    const evts: LabelsEvt[] = []
+    for (const row of rows) {
+      evts.push({
+        seq: row.id,
+        labels: [formatLabel(row)],
+      })
+    }
+
+    return evts
+  }
+
+  private poll() {
+    if (this.destroyed) return
+    if (this.pollPromise) {
+      this.queued = true
+      return
+    }
+    this.queued = false
+    this.pollPromise = this.requestLabelRange({
+      earliestId: this.lastSeen,
+      limit: 500,
+    })
+      .then((evts) => {
+        this.emit('events', evts)
+        this.lastSeen = evts.at(-1)?.seq ?? this.lastSeen
+        if (evts.length > 0) {
+          this.queued = true
+        }
+      })
+      .catch((err) => {
+        log.error(
+          { err, lastSeen: this.lastSeen },
+          'sequencer failed to poll db',
+        )
+      })
+      .finally(() => {
+        this.pollPromise = undefined
+        if (this.queued) {
+          this.poll()
+        }
+      })
+  }
+}
+
+type SequencerEvents = {
+  events: (evts: LabelsEvt[]) => void
+  close: () => void
+}
+
+export type SequencerEmitter = TypedEmitter<SequencerEvents>
+
+export default Sequencer

package/tests/__snapshots__/moderation-events.test.ts.snap

@@ -5,85 +5,63 @@ Object {
   "createdAt": "1970-01-01T00:00:00.000Z",
   "createdBy": "user(2)",
   "event": Object {
-    "$type": "com.atproto.admin.defs#modEventLabel",
-    "comment": "[AutoModerator]: Applying labels",
-    "createLabelVals": Array [
-      "test-label",
-    ],
-    "negateLabelVals": Array [],
+    "$type": "com.atproto.admin.defs#modEventReport",
+    "comment": "X",
+    "reportType": "com.atproto.moderation.defs#reasonMisleading",
   },
   "id": 1,
   "subject": Object {
-    "$type": "com.atproto.admin.defs#recordView",
-    "blobCids": Array [],
-    "cid": "cids(0)",
+    "$type": "com.atproto.admin.defs#repoView",
+    "did": "user(0)",
+    "handle": "alice.test",
     "indexedAt": "1970-01-01T00:00:00.000Z",
-    "moderation": Object {},
-    "repo": Object {
-      "did": "user(0)",
-      "handle": "alice.test",
-      "indexedAt": "1970-01-01T00:00:00.000Z",
-      "moderation": Object {
-        "subjectStatus": Object {
-          "createdAt": "1970-01-01T00:00:00.000Z",
-          "id": 1,
-          "lastReportedAt": "1970-01-01T00:00:00.000Z",
-          "lastReviewedAt": "1970-01-01T00:00:00.000Z",
-          "lastReviewedBy": "user(1)",
-          "reviewState": "com.atproto.admin.defs#reviewEscalated",
-          "subject": Object {
-            "$type": "com.atproto.admin.defs#repoRef",
-            "did": "user(0)",
-          },
-          "subjectBlobCids": Array [],
-          "subjectRepoHandle": "alice.test",
-          "tags": Array [
-            "lang:und",
-          ],
-          "takendown": false,
-          "updatedAt": "1970-01-01T00:00:00.000Z",
+    "moderation": Object {
+      "subjectStatus": Object {
+        "createdAt": "1970-01-01T00:00:00.000Z",
+        "id": 1,
+        "lastReportedAt": "1970-01-01T00:00:00.000Z",
+        "lastReviewedAt": "1970-01-01T00:00:00.000Z",
+        "lastReviewedBy": "user(1)",
+        "reviewState": "com.atproto.admin.defs#reviewEscalated",
+        "subject": Object {
+          "$type": "com.atproto.admin.defs#repoRef",
+          "did": "user(0)",
         },
+        "subjectBlobCids": Array [],
+        "subjectRepoHandle": "alice.test",
+        "tags": Array [
+          "lang:und",
+        ],
+        "takendown": false,
+        "updatedAt": "1970-01-01T00:00:00.000Z",
       },
-      "relatedRecords": Array [
-        Object {
-          "$type": "app.bsky.actor.profile",
-          "avatar": Object {
-            "$type": "blob",
-            "mimeType": "image/jpeg",
-            "ref": Object {
-              "$link": "cids(2)",
-            },
-            "size": 3976,
-          },
-          "description": "its me!",
-          "displayName": "ali",
-          "labels": Object {
-            "$type": "com.atproto.label.defs#selfLabels",
-            "values": Array [
-              Object {
-                "val": "self-label-a",
-              },
-              Object {
-                "val": "self-label-b",
-              },
-            ],
+    },
+    "relatedRecords": Array [
+      Object {
+        "$type": "app.bsky.actor.profile",
+        "avatar": Object {
+          "$type": "blob",
+          "mimeType": "image/jpeg",
+          "ref": Object {
+            "$link": "cids(0)",
           },
+          "size": 3976,
         },
-      ],
-    },
-    "uri": "record(0)",
-    "value": Object {
-      "$type": "app.bsky.feed.post",
-      "createdAt": "1970-01-01T00:00:00.000Z",
-      "embed": Object {
-        "$type": "app.bsky.embed.record",
-        "record": Object {
-          "cid": "cids(1)",
-          "uri": "record(1)",
+        "description": "its me!",
+        "displayName": "ali",
+        "labels": Object {
+          "$type": "com.atproto.label.defs#selfLabels",
+          "values": Array [
+            Object {
+              "val": "self-label-a",
+            },
+            Object {
+              "val": "self-label-b",
+            },
+          ],
         },
       },
-      "text": "yoohoo label_me",
-    },
+    ],
   },
   "subjectBlobCids": Array [],
   "subjectBlobs": Array [],
@@ -101,7 +79,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 13,
+    "id": 11,
     "subject": Object {
       "$type": "com.atproto.admin.defs#repoRef",
       "did": "user(0)",
@@ -120,7 +98,7 @@ Array [
       ],
       "remove": Array [],
     },
-    "id": 8,
+    "id": 6,
     "subject": Object {
       "$type": "com.atproto.admin.defs#repoRef",
       "did": "user(0)",
@@ -137,7 +115,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 7,
+    "id": 5,
     "subject": Object {
       "$type": "com.atproto.admin.defs#repoRef",
       "did": "user(0)",
@@ -159,7 +137,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 12,
+    "id": 10,
     "subject": Object {
       "$type": "com.atproto.repo.strongRef",
       "cid": "cids(0)",
@@ -178,7 +156,7 @@ Array [
       ],
       "remove": Array [],
     },
-    "id": 6,
+    "id": 4,
     "subject": Object {
       "$type": "com.atproto.repo.strongRef",
       "cid": "cids(0)",
@@ -196,7 +174,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 5,
+    "id": 3,
     "subject": Object {
       "$type": "com.atproto.repo.strongRef",
       "cid": "cids(0)",

package/tests/__snapshots__/moderation.test.ts.snap

@@ -4,7 +4,7 @@ exports[`moderation reporting creates reports of a record. 1`] = `
 Array [
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 7,
+    "id": 5,
     "reasonType": "com.atproto.moderation.defs#reasonSpam",
     "reportedBy": "user(0)",
     "subject": Object {
@@ -15,7 +15,7 @@ Array [
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 9,
+    "id": 7,
     "reason": "defamation",
     "reasonType": "com.atproto.moderation.defs#reasonOther",
     "reportedBy": "user(1)",
@@ -32,7 +32,7 @@ exports[`moderation reporting creates reports of a repo. 1`] = `
 Array [
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 3,
+    "id": 1,
     "reasonType": "com.atproto.moderation.defs#reasonSpam",
     "reportedBy": "user(0)",
     "subject": Object {
@@ -42,7 +42,7 @@ Array [
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 5,
+    "id": 3,
     "reason": "impersonation",
     "reasonType": "com.atproto.moderation.defs#reasonOther",
     "reportedBy": "user(2)",

package/tests/moderation-appeals.test.ts

@@ -118,7 +118,7 @@ describe('moderation-appeals', () => {
       })
 
       // Verify that appeal status changed when appeal report was emitted by moderator
-      const status = await assertBobsPostStatus(REVIEWOPEN, true)
+      const status = await assertBobsPostStatus(REVIEWESCALATED, true)
       expect(status?.appealedAt).not.toBeNull()
 
       // Create a report as normal user for carol's post
@@ -143,7 +143,11 @@ describe('moderation-appeals', () => {
         subject: getCarolPostSubject(),
       })
       // Verify that the appeal status on carol's post is true
-      await assertSubjectStatus(getCarolPostSubject().uri, REVIEWOPEN, true)
+      await assertSubjectStatus(
+        getCarolPostSubject().uri,
+        REVIEWESCALATED,
+        true,
+      )
     })
     it('allows multiple appeals and updates last appealed timestamp', async () => {
       // Resolve appeal with acknowledge
@@ -155,7 +159,7 @@ describe('moderation-appeals', () => {
         createdBy: sc.dids.carol,
       })
 
-      const previousStatus = await assertBobsPostStatus(REVIEWOPEN, false)
+      const previousStatus = await assertBobsPostStatus(REVIEWESCALATED, false)
 
       await emitModerationEvent({
         event: {
@@ -167,7 +171,7 @@ describe('moderation-appeals', () => {
       })
 
       // Verify that even after the appeal event by bob for his post, the appeal status is true again with new timestamp
-      const newStatus = await assertBobsPostStatus(REVIEWOPEN, true)
+      const newStatus = await assertBobsPostStatus(REVIEWESCALATED, true)
       expect(
         new Date(`${previousStatus?.lastAppealedAt}`).getTime(),
       ).toBeLessThan(new Date(`${newStatus?.lastAppealedAt}`).getTime())
@@ -210,7 +214,11 @@ describe('moderation-appeals', () => {
         createdBy: sc.dids.alice,
       })
 
-      await assertSubjectStatus(getAlicesPostSubject().uri, REVIEWOPEN, true)
+      await assertSubjectStatus(
+        getAlicesPostSubject().uri,
+        REVIEWESCALATED,
+        true,
+      )
 
       // Bob reports it again
       await emitModerationEvent({
@@ -222,8 +230,12 @@ describe('moderation-appeals', () => {
         createdBy: sc.dids.bob,
       })
 
-      // Assert that the status is still REVIEWOPEN, as report events are meant to do
-      await assertSubjectStatus(getAlicesPostSubject().uri, REVIEWOPEN, true)
+      // Assert that the status is still REVIEWESCALATED, as report events are meant to do
+      await assertSubjectStatus(
+        getAlicesPostSubject().uri,
+        REVIEWESCALATED,
+        true,
+      )
 
       // Emit an escalation event
       await emitModerationEvent({

package/tests/moderation-events.test.ts

@@ -22,13 +22,13 @@ describe('moderation-events', () => {
   ) => {
     return pdsAgent.api.com.atproto.admin.emitModerationEvent(eventData, {
       encoding: 'application/json',
-      headers: network.bsky.adminAuthHeaders('moderator'),
+      headers: network.ozone.adminAuthHeaders('moderator'),
     })
   }
 
   const queryModerationEvents = (eventQuery) =>
     agent.api.com.atproto.admin.queryModerationEvents(eventQuery, {
-      headers: network.bsky.adminAuthHeaders('moderator'),
+      headers: network.ozone.adminAuthHeaders('moderator'),
     })
 
   const seedEvents = async () => {
@@ -203,11 +203,11 @@ describe('moderation-events', () => {
       const defaultEvents = await getPaginatedEvents()
       const reversedEvents = await getPaginatedEvents('asc')
 
-      expect(allEvents.data.events.length).toEqual(7)
+      expect(allEvents.data.events.length).toEqual(6)
       expect(defaultEvents.length).toEqual(allEvents.data.events.length)
       expect(reversedEvents.length).toEqual(allEvents.data.events.length)
       // First event in the reversed list is the last item in the default list
-      expect(reversedEvents[0].id).toEqual(defaultEvents[6].id)
+      expect(reversedEvents[0].id).toEqual(defaultEvents[5].id)
     })
 
     it('returns report events matching reportType filters', async () => {
@@ -240,7 +240,7 @@ describe('moderation-events', () => {
 
       expect(eventsWithX.data.events.length).toEqual(10)
       expect(eventsWithTest.data.events.length).toEqual(0)
-      expect(eventsWithComment.data.events.length).toEqual(12)
+      expect(eventsWithComment.data.events.length).toEqual(10)
     })
 
     it('returns events matching filter params for labels', async () => {
@@ -325,7 +325,7 @@ describe('moderation-events', () => {
         })
       const addEvent = await tagEvent({ add: ['L1', 'L2'], remove: [] })
       const addAndRemoveEvent = await tagEvent({ add: ['L3'], remove: ['L2'] })
-      const [addFinder, addAndRemoveFinder, removeFinder] = await Promise.all([
+      const [addFinder, addAndRemoveFinder, _removeFinder] = await Promise.all([
         queryModerationEvents({
           addedTags: ['L1'],
         }),
@@ -356,7 +356,7 @@ describe('moderation-events', () => {
     it('gets an event by specific id', async () => {
       const { data } = await pdsAgent.api.com.atproto.admin.getModerationEvent(
         { id: 1 },
-        { headers: network.bsky.adminAuthHeaders('moderator') },
+        { headers: network.ozone.adminAuthHeaders('moderator') },
       )
       expect(forSnapshot(data)).toMatchSnapshot()
     })

package/tests/moderation-statuses.test.ts

@@ -19,13 +19,13 @@ describe('moderation-statuses', () => {
   const emitModerationEvent = async (eventData) => {
     return pdsAgent.api.com.atproto.admin.emitModerationEvent(eventData, {
       encoding: 'application/json',
-      headers: network.bsky.adminAuthHeaders('moderator'),
+      headers: network.ozone.adminAuthHeaders('moderator'),
     })
   }
 
   const queryModerationStatuses = (statusQuery) =>
     agent.api.com.atproto.admin.queryModerationStatuses(statusQuery, {
-      headers: network.bsky.adminAuthHeaders('moderator'),
+      headers: network.ozone.adminAuthHeaders('moderator'),
     })
 
   const seedEvents = async () => {