@atproto/ozone 0.1.107 → 0.1.109

package/src/api/verification/grantVerifications.ts

@@ -0,0 +1,90 @@
+import { Selectable } from 'kysely'
+import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import { AppContext } from '../../context'
+import { Verification } from '../../db/schema/verification'
+import { Server } from '../../lexicon'
+import { getReposForVerifications } from '../../verification/util'
+
+export default function (server: Server, ctx: AppContext) {
+  server.tools.ozone.verification.grantVerifications({
+    auth: ctx.authVerifier.modOrAdminToken,
+    handler: async ({ input, auth, req }) => {
+      if (!ctx.cfg.verifier) {
+        throw new InvalidRequestError('Verifier not configured')
+      }
+
+      if (!auth.credentials.isVerifier) {
+        throw new AuthRequiredError(
+          'Must be an admin or verifier to grant verifications',
+        )
+      }
+
+      const modViews = ctx.modService(ctx.db).views
+      const profilesBefore = await modViews.getProfiles(
+        input.body.verifications.map((v) => v.subject),
+      )
+
+      // Filter out any subject for which, the current issuer already has a valid verification record indexed
+      const verificationsToBeGranted = input.body.verifications.filter(
+        (verificationInput) => {
+          const hasValidVerification = profilesBefore
+            .get(verificationInput.subject)
+            ?.verification?.verifications.find(
+              (v) => v.issuer === ctx.cfg.verifier?.did && v.isValid,
+            )
+          return !hasValidVerification
+        },
+      )
+
+      const verificationIssuer = ctx.verificationIssuer(ctx.cfg.verifier)
+      const verificationService = ctx.verificationService(ctx.db)
+      const { grantedVerifications, failedVerifications } =
+        await verificationIssuer.verify(verificationsToBeGranted)
+
+      if (!grantedVerifications.length) {
+        return {
+          encoding: 'application/json',
+          body: {
+            verifications: [],
+            failedVerifications,
+          },
+        }
+      }
+
+      const createdVerifications: Selectable<Verification>[] = []
+      const verificationEntries =
+        await verificationService.create(grantedVerifications)
+
+      const dids = new Set<string>([ctx.cfg.verifier.did])
+
+      for (const verification of verificationEntries) {
+        createdVerifications.push(verification)
+        dids.add(verification.subject)
+      }
+
+      const didsArr = Array.from(dids)
+      const [repos, profiles] = await Promise.all([
+        getReposForVerifications(
+          ctx,
+          ctx.reqLabelers(req),
+          ctx.modService(ctx.db),
+          didsArr,
+          auth.credentials.isModerator,
+        ),
+        modViews.getProfiles(didsArr),
+      ])
+      const verifications = verificationService.view(
+        createdVerifications,
+        repos,
+        profiles,
+      )
+      return {
+        encoding: 'application/json',
+        body: {
+          verifications,
+          failedVerifications,
+        },
+      }
+    },
+  })
+}

package/src/api/verification/listVerifications.ts

@@ -0,0 +1,44 @@
+import { AppContext } from '../../context'
+import { Server } from '../../lexicon'
+import { getReposForVerifications } from '../../verification/util'
+
+export default function (server: Server, ctx: AppContext) {
+  server.tools.ozone.verification.listVerifications({
+    auth: ctx.authVerifier.modOrAdminToken,
+    handler: async ({ req, params, auth }) => {
+      const modViews = ctx.modService(ctx.db).views
+      const verificationService = ctx.verificationService(ctx.db)
+      const { verifications, cursor } = await verificationService.list(params)
+
+      const dids = new Set<string>()
+      for (const verification of verifications) {
+        dids.add(verification.subject)
+        dids.add(verification.issuer)
+      }
+
+      const didsArr = Array.from(dids)
+      const [repos, profiles] = await Promise.all([
+        getReposForVerifications(
+          ctx,
+          ctx.reqLabelers(req),
+          ctx.modService(ctx.db),
+          didsArr,
+          auth.credentials.isModerator,
+        ),
+        modViews.getProfiles(didsArr),
+      ])
+
+      return {
+        encoding: 'application/json',
+        body: {
+          cursor,
+          verifications: verificationService.view(
+            verifications,
+            repos,
+            profiles,
+          ),
+        },
+      }
+    },
+  })
+}

package/src/api/verification/revokeVerifications.ts

@@ -0,0 +1,43 @@
+import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
+import { AppContext } from '../../context'
+import { Server } from '../../lexicon'
+
+export default function (server: Server, ctx: AppContext) {
+  server.tools.ozone.verification.revokeVerifications({
+    auth: ctx.authVerifier.modOrAdminToken,
+    handler: async ({ input, auth }) => {
+      if (!ctx.cfg.verifier) {
+        throw new InvalidRequestError('Verifier not configured')
+      }
+
+      if (!auth.credentials.isVerifier) {
+        throw new AuthRequiredError(
+          'Must be an admin or verifier to revoke verifications',
+        )
+      }
+
+      const verificationIssuer = ctx.verificationIssuer(ctx.cfg.verifier)
+      const { uris, revokeReason } = input.body
+      const { revokedVerifications, failedRevocations } =
+        await verificationIssuer.revoke({ uris })
+
+      if (revokedVerifications.length) {
+        const verificationService = ctx.verificationService(ctx.db)
+        await verificationService.markRevoked({
+          uris: revokedVerifications,
+          revokeReason,
+          revokedBy:
+            'iss' in auth.credentials ? auth.credentials.iss : undefined,
+        })
+      }
+
+      return {
+        encoding: 'application/json',
+        body: {
+          revokedVerifications,
+          failedRevocations,
+        },
+      }
+    },
+  })
+}

package/src/auth-verifier.ts

@@ -18,6 +18,7 @@ export type AdminTokenOutput = {
     isAdmin: true
     isModerator: true
     isTriage: true
+    isVerifier: true
   }
 }
 
@@ -28,7 +29,8 @@ export type ModeratorOutput = {
     iss: string
     isAdmin: boolean
     isModerator: boolean
-    isTriage: true
+    isTriage: boolean
+    isVerifier: boolean
   }
 }
 
@@ -40,6 +42,7 @@ type StandardOutput = {
     isAdmin: boolean
     isModerator: boolean
     isTriage: boolean
+    isVerifier: boolean
   }
 }
 
@@ -82,14 +85,13 @@ export class AuthVerifier {
 
   moderator = async (reqCtx: ReqCtx): Promise<ModeratorOutput> => {
     const creds = await this.standard(reqCtx)
-    if (!creds.credentials.isTriage) {
+    if (!creds.credentials.isTriage && !creds.credentials.isVerifier) {
       throw new AuthRequiredError('not a moderator account')
     }
     return {
       credentials: {
         ...creds.credentials,
         type: 'moderator',
-        isTriage: true,
       },
     }
   }
@@ -125,7 +127,7 @@ export class AuthVerifier {
       throw new AuthRequiredError('member is disabled', 'MemberDisabled')
     }
 
-    const { isAdmin, isModerator, isTriage } =
+    const { isAdmin, isModerator, isTriage, isVerifier } =
       this.teamService.getMemberRole(member)
 
     return {
@@ -136,6 +138,7 @@ export class AuthVerifier {
         isAdmin,
         isModerator,
         isTriage,
+        isVerifier,
       },
     }
   }
@@ -173,6 +176,7 @@ export class AuthVerifier {
         isAdmin: true,
         isModerator: true,
         isTriage: true,
+        isVerifier: true,
       },
     }
   }

package/src/background.ts

@@ -10,7 +10,7 @@ type Task = (db: Database, signal: AbortSignal) => Promise<void>
  */
 export class BackgroundQueue {
   private abortController = new AbortController()
-  private queue = new PQueue({ concurrency: 20 })
+  private queue: PQueue
 
   public get signal() {
     return this.abortController.signal
@@ -20,7 +20,12 @@ export class BackgroundQueue {
     return this.signal.aborted
   }
 
-  constructor(protected db: Database) {}
+  constructor(
+    protected db: Database,
+    queueOpts?: { concurrency?: number },
+  ) {
+    this.queue = new PQueue(queueOpts ?? { concurrency: 20 })
+  }
 
   getStats() {
     return {
@@ -58,7 +63,7 @@ export class BackgroundQueue {
         await task(this.db, abortController.signal)
       } catch (err) {
         if (!isCausedBySignal(err, abortController.signal)) {
-          dbLogger.error(err, 'background queue task failed')
+          dbLogger.error({ err }, 'background queue task failed')
         }
       } finally {
         abortController.abort()

package/src/config/config.ts

@@ -79,6 +79,15 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     moderators: env.moderatorDids,
     triage: env.triageDids,
   }
+  const verifierCfg: OzoneConfig['verifier'] =
+    env.verifierUrl && env.verifierDid && env.verifierPassword
+      ? {
+          url: env.verifierUrl,
+          did: env.verifierDid,
+          password: env.verifierPassword,
+          issuersToIndex: env.verifierIssuersToIndex,
+        }
+      : null
 
   return {
     service: serviceCfg,
@@ -90,6 +99,8 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     identity: identityCfg,
     blobDivert: blobDivertServiceCfg,
     access: accessCfg,
+    verifier: verifierCfg,
+    jetstreamUrl: env.jetstreamUrl,
   }
 }
 
@@ -103,6 +114,8 @@ export type OzoneConfig = {
   identity: IdentityConfig
   blobDivert: BlobDivertConfig | null
   access: AccessConfig
+  jetstreamUrl?: string
+  verifier: VerifierConfig | null
 }
 
 export type ServiceConfig = {
@@ -159,3 +172,11 @@ export type AccessConfig = {
   moderators: string[]
   triage: string[]
 }
+
+export type VerifierConfig = {
+  url: string
+  did: string
+  password: string
+  jetstreamUrl?: string
+  issuersToIndex?: string[]
+}

package/src/config/env.ts

@@ -37,6 +37,11 @@ export const readEnv = (): OzoneEnvironment => {
     signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'),
     blobDivertUrl: envStr('OZONE_BLOB_DIVERT_URL'),
     blobDivertAdminPassword: envStr('OZONE_BLOB_DIVERT_ADMIN_PASSWORD'),
+    verifierUrl: envStr('OZONE_VERIFIER_URL'),
+    verifierDid: envStr('OZONE_VERIFIER_DID'),
+    verifierPassword: envStr('OZONE_VERIFIER_PASSWORD'),
+    verifierIssuersToIndex: envList('OZONE_VERIFIER_ISSUERS_TO_INDEX'),
+    jetstreamUrl: envStr('OZONE_JETSTREAM_URL'),
   }
 }
 
@@ -72,4 +77,9 @@ export type OzoneEnvironment = {
   signingKeyHex?: string
   blobDivertUrl?: string
   blobDivertAdminPassword?: string
+  verifierUrl?: string
+  verifierDid?: string
+  verifierPassword?: string
+  verifierIssuersToIndex?: string[]
+  jetstreamUrl?: string
 }

package/src/context.ts

@@ -28,6 +28,14 @@ import {
   getSigningKeyId,
   parseLabelerHeader,
 } from './util'
+import {
+  VerificationIssuer,
+  VerificationIssuerCreator,
+} from './verification/issuer'
+import {
+  VerificationService,
+  VerificationServiceCreator,
+} from './verification/service'
 
 export type AppContextOptions = {
   db: Database
@@ -49,6 +57,8 @@ export type AppContextOptions = {
   backgroundQueue: BackgroundQueue
   sequencer: Sequencer
   authVerifier: AuthVerifier
+  verificationService: VerificationServiceCreator
+  verificationIssuer: VerificationIssuerCreator
 }
 
 export class AppContext {
@@ -127,6 +137,8 @@ export class AppContext {
     )
     const setService = SetService.creator()
     const settingService = SettingService.creator()
+    const verificationService = VerificationService.creator()
+    const verificationIssuer = VerificationIssuer.creator()
 
     const sequencer = new Sequencer(modService(db))
 
@@ -156,6 +168,8 @@ export class AppContext {
         sequencer,
         authVerifier,
         blobDiverter,
+        verificationService,
+        verificationIssuer,
         ...(overrides ?? {}),
       },
       secrets,
@@ -202,6 +216,14 @@ export class AppContext {
     return this.opts.settingService
   }
 
+  get verificationService(): VerificationServiceCreator {
+    return this.opts.verificationService
+  }
+
+  get verificationIssuer(): VerificationIssuerCreator {
+    return this.opts.verificationIssuer
+  }
+
   get appviewAgent(): AtpAgent {
     return this.opts.appviewAgent
   }

package/src/daemon/context.ts

@@ -13,6 +13,7 @@ import { EventPusher } from './event-pusher'
 import { EventReverser } from './event-reverser'
 import { MaterializedViewRefresher } from './materialized-view-refresher'
 import { TeamProfileSynchronizer } from './team-profile-synchronizer'
+import { VerificationListener } from './verification-listener'
 
 export type DaemonContextOptions = {
   db: Database
@@ -23,6 +24,7 @@ export type DaemonContextOptions = {
   eventReverser: EventReverser
   materializedViewRefresher: MaterializedViewRefresher
   teamProfileSynchronizer: TeamProfileSynchronizer
+  verificationListener?: VerificationListener
 }
 
 export class DaemonContext {
@@ -88,6 +90,16 @@ export class DaemonContext {
       cfg.db.materializedViewRefreshIntervalMs,
     )
 
+    // Only spawn the listener if verifier config exists and a jetstream URL is provided
+    const verificationListener =
+      cfg.verifier && cfg.jetstreamUrl
+        ? new VerificationListener(
+            db,
+            cfg.jetstreamUrl,
+            cfg.verifier?.issuersToIndex,
+          )
+        : undefined
+
     return new DaemonContext({
       db,
       cfg,
@@ -97,6 +109,7 @@ export class DaemonContext {
       eventReverser,
       materializedViewRefresher,
       teamProfileSynchronizer,
+      verificationListener,
       ...(overrides ?? {}),
     })
   }
@@ -129,11 +142,16 @@ export class DaemonContext {
     return this.opts.teamProfileSynchronizer
   }
 
+  get verificationListener(): VerificationListener | undefined {
+    return this.opts.verificationListener
+  }
+
   async start() {
     this.eventPusher.start()
     this.eventReverser.start()
     this.materializedViewRefresher.start()
     this.teamProfileSynchronizer.start()
+    this.verificationListener?.start()
   }
 
   async processAll() {
@@ -150,6 +168,7 @@ export class DaemonContext {
         this.eventPusher.destroy(),
         this.materializedViewRefresher.destroy(),
         this.teamProfileSynchronizer.destroy(),
+        this.verificationListener?.stop(),
       ])
     } finally {
       await this.backgroundQueue.destroy()

package/src/daemon/verification-listener.ts

@@ -0,0 +1,164 @@
+import { lexicons } from '@atproto/api'
+import { BackgroundQueue } from '../background'
+import { Database } from '../db'
+import { CommitCreateEvent, Jetstream } from '../jetstream/service'
+import { verificationLogger } from '../logger'
+import { VerificationService } from '../verification/service'
+
+type VerificationRecord = {
+  subject: string
+  handle: string
+  displayName: string
+  createdAt: string
+}
+
+export class VerificationListener {
+  destroyed = false
+  private cursor?: number
+  private jetstream: Jetstream | null = null
+  private collection = 'app.bsky.graph.verification'
+  public backgroundQueue = new BackgroundQueue(this.db, { concurrency: 1 })
+  private verificationService = VerificationService.creator()(this.db)
+
+  constructor(
+    private db: Database,
+    private jetstreamUrl: string,
+    private verifierIssuersToIndex?: string[],
+  ) {}
+
+  // When the queue has capacity, this method returns true which means we can continue to handle events
+  // otherwise, it will close jetstream connection and wait for all previously queued events to be processed first
+  // and then start jetstream listener again before returning false. At that point, the previous listeners should
+  // have updates the cursor in db to the last processed event and the new listener will start from that cursor
+  async ensureCoolDown() {
+    const { waitingCount, runningCount } = this.backgroundQueue.getStats()
+    if (waitingCount > 50 || runningCount > 50) {
+      verificationLogger.warn(`Background queue is full, pausing listener`)
+      this.jetstream?.close()
+      await this.backgroundQueue.processAll()
+      await this.start()
+      return false
+    }
+    return true
+  }
+
+  handleNewVerification(
+    issuer: string,
+    uri: string,
+    cid: string,
+    record: VerificationRecord,
+    cursor: number,
+  ) {
+    this.backgroundQueue.add(async () => {
+      try {
+        const { subject, handle, displayName, createdAt } = record
+        await this.verificationService.create([
+          { uri, cid, issuer, subject, handle, displayName, createdAt },
+        ])
+        await this.updateCursor(cursor)
+      } catch (err) {
+        verificationLogger.error(
+          err,
+          'Error handling verification create event',
+        )
+      }
+    })
+  }
+
+  handleDeletedVerification(uri: string, cursor: number) {
+    this.backgroundQueue.add(async () => {
+      try {
+        await this.verificationService.markRevoked({
+          uris: [uri],
+        })
+        await this.updateCursor(cursor)
+      } catch (err) {
+        verificationLogger.error(
+          err,
+          'Error handling verification delete event',
+        )
+      }
+    })
+  }
+
+  async getCursor() {
+    await this.verificationService.createFirehoseCursor()
+    const cursor = await this.verificationService.getFirehoseCursor()
+    if (cursor) {
+      this.cursor = cursor
+    }
+    return this.cursor
+  }
+
+  async updateCursor(cursor: number) {
+    // Assuming cursors are always incremental, if we have processed an event with higher value cursor, let's not update to a lower value
+    if (this.cursor && this.cursor >= cursor) {
+      return
+    }
+
+    // This will only update if the cursor is higher than the current one in db
+    const updatedCursor =
+      await this.verificationService.updateFirehoseCursor(cursor)
+
+    if (updatedCursor) {
+      this.cursor = updatedCursor
+    }
+  }
+
+  async start() {
+    await this.getCursor()
+
+    this.jetstream = new Jetstream({
+      endpoint: this.jetstreamUrl,
+      cursor: this.cursor || undefined,
+      wantedCollections: [this.collection],
+      wantedDids: this.verifierIssuersToIndex?.length
+        ? this.verifierIssuersToIndex
+        : undefined,
+    })
+
+    await this.jetstream.start({
+      onCreate: {
+        [this.collection]: async (e: CommitCreateEvent<VerificationRecord>) => {
+          const recordValidity = lexicons.validate(
+            this.collection,
+            e.commit.record,
+          )
+
+          if (!recordValidity.success) {
+            verificationLogger.error(
+              recordValidity.error,
+              'Invalid verification record in the firehose',
+            )
+            return
+          }
+
+          const hasCapacity = await this.ensureCoolDown()
+          if (hasCapacity) {
+            const issuer = e.did
+            const { record, rkey, collection, cid } = e.commit
+            const uri = `at://${issuer}/${collection}/${rkey}`
+            this.handleNewVerification(issuer, uri, cid, record, e.time_us)
+          }
+        },
+      },
+      onDelete: {
+        [this.collection]: async (e) => {
+          const hasCapacity = await this.ensureCoolDown()
+          if (hasCapacity) {
+            this.handleDeletedVerification(
+              `at://${e.did}/${e.commit.collection}/${e.commit.rkey}`,
+              e.time_us,
+            )
+          }
+        },
+      },
+    })
+  }
+
+  stop() {
+    this.jetstream?.close()
+    this.backgroundQueue.destroy()
+    this.destroyed = true
+  }
+}

package/src/db/migrations/20250415T201720309Z-verification.ts

@@ -0,0 +1,34 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('verification')
+    .addColumn('uri', 'text', (col) => col.notNull().primaryKey())
+    .addColumn('cid', 'text', (col) => col.notNull())
+    .addColumn('issuer', 'text', (col) => col.notNull())
+    .addColumn('subject', 'text', (col) => col.notNull())
+    .addColumn('handle', 'text', (col) => col.notNull())
+    .addColumn('displayName', 'text', (col) => col.notNull())
+    .addColumn('revokeReason', 'text')
+    .addColumn('revokedBy', 'text')
+    .addColumn('revokedAt', 'text')
+    .addColumn('createdAt', 'text', (col) => col.notNull())
+    .addColumn('updatedAt', 'text', (col) =>
+      col.defaultTo(sql`now()`).notNull(),
+    )
+    .execute()
+  await db.schema
+    .createIndex('verification_issuer_idx')
+    .on('verification')
+    .column('issuer')
+    .execute()
+  await db.schema
+    .createIndex('verification_createdat_uri_idx')
+    .on('verification')
+    .columns(['createdAt', 'uri'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('verification').execute()
+}

package/src/db/migrations/20250417T201720309Z-firehose-cursor.ts

@@ -0,0 +1,16 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('firehose_cursor')
+    .addColumn('service', 'text', (col) => col.primaryKey())
+    .addColumn('cursor', 'bigint')
+    .addColumn('updatedAt', 'text', (col) =>
+      col.defaultTo(sql`now()`).notNull(),
+    )
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('firehose_cursor').execute()
+}

package/src/db/migrations/index.ts

@@ -23,3 +23,5 @@ export * as _20250211T003647759Z from './20250211T003647759Z-add-reporter-stats-
 export * as _20250211T132135150Z from './20250211T132135150Z-moderation-event-message-partial-idx'
 export * as _20250221T132135150Z from './20250221T132135150Z-member-details'
 export * as _20250404T201720309Z from './20250404T201720309Z-subject-status-sort-idxs'
+export * as _20250415T201720309Z from './20250415T201720309Z-verification'
+export * as _20250417T201720309Z from './20250417T201720309Z-firehose-cursor'