@atproto/ozone 0.0.16 → 0.0.17-next.1

package/src/auth-verifier.ts

@@ -7,16 +7,16 @@ type ReqCtx = {
   req: express.Request
 }
 
-type RoleOutput = {
+export type AdminTokenOutput = {
   credentials: {
-    type: 'role'
-    isAdmin: boolean
-    isModerator: boolean
+    type: 'admin_token'
+    isAdmin: true
+    isModerator: true
     isTriage: true
   }
 }
 
-type ModeratorOutput = {
+export type ModeratorOutput = {
   credentials: {
     type: 'moderator'
     aud: string
@@ -51,8 +51,6 @@ export type AuthVerifierOpts = {
   moderators: string[]
   triage: string[]
   adminPassword: string
-  moderatorPassword: string
-  triagePassword: string
 }
 
 export class AuthVerifier {
@@ -61,8 +59,6 @@ export class AuthVerifier {
   moderators: string[]
   triage: string[]
   private adminPassword: string
-  private moderatorPassword: string
-  private triagePassword: string
 
   constructor(public idResolver: IdResolver, opts: AuthVerifierOpts) {
     this.serviceDid = opts.serviceDid
@@ -70,15 +66,15 @@ export class AuthVerifier {
     this.moderators = opts.moderators
     this.triage = opts.triage
     this.adminPassword = opts.adminPassword
-    this.moderatorPassword = opts.moderatorPassword
-    this.triagePassword = opts.triagePassword
   }
 
-  modOrRole = async (reqCtx: ReqCtx): Promise<ModeratorOutput | RoleOutput> => {
-    if (isBearerToken(reqCtx.req)) {
-      return this.moderator(reqCtx)
+  modOrAdminToken = async (
+    reqCtx: ReqCtx,
+  ): Promise<ModeratorOutput | AdminTokenOutput> => {
+    if (isBasicToken(reqCtx.req)) {
+      return this.adminToken(reqCtx)
     } else {
-      return this.role(reqCtx)
+      return this.moderator(reqCtx)
     }
   }
 
@@ -138,36 +134,30 @@ export class AuthVerifier {
     return this.nullCreds()
   }
 
-  standardOptionalOrRole = async (
+  standardOptionalOrAdminToken = async (
     reqCtx: ReqCtx,
-  ): Promise<StandardOutput | RoleOutput | NullOutput> => {
+  ): Promise<StandardOutput | AdminTokenOutput | NullOutput> => {
     if (isBearerToken(reqCtx.req)) {
       return this.standard(reqCtx)
     } else if (isBasicToken(reqCtx.req)) {
-      return this.role(reqCtx)
+      return this.adminToken(reqCtx)
     } else {
       return this.nullCreds()
     }
   }
 
-  role = async (reqCtx: ReqCtx): Promise<RoleOutput> => {
+  adminToken = async (reqCtx: ReqCtx): Promise<AdminTokenOutput> => {
     const parsed = parseBasicAuth(reqCtx.req.headers.authorization ?? '')
     const { username, password } = parsed ?? {}
-    if (username !== 'admin') {
-      throw new AuthRequiredError()
-    }
-    const isAdmin = password === this.adminPassword
-    const isModerator = isAdmin || password === this.moderatorPassword
-    const isTriage = isModerator || password === this.triagePassword
-    if (!isTriage) {
+    if (username !== 'admin' || password !== this.adminPassword) {
       throw new AuthRequiredError()
     }
     return {
       credentials: {
-        type: 'role',
-        isAdmin,
-        isModerator,
-        isTriage,
+        type: 'admin_token',
+        isAdmin: true,
+        isModerator: true,
+        isTriage: true,
       },
     }
   }

package/src/communication-service/template.ts

@@ -1,7 +1,7 @@
 import Database from '../db'
 import { Selectable } from 'kysely'
 import { CommunicationTemplate } from '../db/schema/communication_template'
-import { CommunicationTemplateView } from '../lexicon/types/com/atproto/admin/defs'
+import { TemplateView } from '../lexicon/types/tools/ozone/communication/defs'
 
 export type CommunicationTemplateServiceCreator = (
   db: Database,
@@ -90,7 +90,7 @@ export class CommunicationTemplateService {
       .execute()
   }
 
-  view(template: Selectable<CommunicationTemplate>): CommunicationTemplateView {
+  view(template: Selectable<CommunicationTemplate>): TemplateView {
     return {
       id: `${template.id}`,
       name: template.name,

package/src/config/config.ts

@@ -25,18 +25,20 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs,
   }
 
-  assert(env.appviewUrl)
-  assert(env.appviewDid)
+  assert(env.appviewUrl && env.appviewDid)
   const appviewCfg: OzoneConfig['appview'] = {
     url: env.appviewUrl,
     did: env.appviewDid,
+    pushEvents: !!env.appviewPushEvents,
   }
 
-  assert(env.pdsUrl)
-  assert(env.pdsDid)
-  const pdsCfg: OzoneConfig['pds'] = {
-    url: env.pdsUrl,
-    did: env.pdsDid,
+  let pdsCfg: OzoneConfig['pds'] = null
+  if (env.pdsUrl || env.pdsDid) {
+    assert(env.pdsUrl && env.pdsDid)
+    pdsCfg = {
+      url: env.pdsUrl,
+      did: env.pdsDid,
+    }
   }
 
   const cdnCfg: OzoneConfig['cdn'] = {
@@ -48,6 +50,13 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     plcUrl: env.didPlcUrl,
   }
 
+  const blobDivertServiceCfg =
+    env.blobDivertUrl && env.blobDivertAdminPassword
+      ? {
+          url: env.blobDivertUrl,
+          adminPassword: env.blobDivertAdminPassword,
+        }
+      : null
   const accessCfg: OzoneConfig['access'] = {
     admins: env.adminDids,
     moderators: env.moderatorDids,
@@ -61,6 +70,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     pds: pdsCfg,
     cdn: cdnCfg,
     identity: identityCfg,
+    blobDivert: blobDivertServiceCfg,
     access: accessCfg,
   }
 }
@@ -72,6 +82,7 @@ export type OzoneConfig = {
   pds: PdsConfig | null
   cdn: CdnConfig
   identity: IdentityConfig
+  blobDivert: BlobDivertConfig | null
   access: AccessConfig
 }
 
@@ -83,6 +94,11 @@ export type ServiceConfig = {
   devMode?: boolean
 }
 
+export type BlobDivertConfig = {
+  url: string
+  adminPassword: string
+}
+
 export type DatabaseConfig = {
   postgresUrl: string
   postgresSchema?: string
@@ -94,6 +110,7 @@ export type DatabaseConfig = {
 export type AppviewConfig = {
   url: string
   did: string
+  pushEvents: boolean
 }
 
 export type PdsConfig = {

package/src/config/env.ts

@@ -10,6 +10,7 @@ export const readEnv = (): OzoneEnvironment => {
     serverDid: envStr('OZONE_SERVER_DID'),
     appviewUrl: envStr('OZONE_APPVIEW_URL'),
     appviewDid: envStr('OZONE_APPVIEW_DID'),
+    appviewPushEvents: envBool('OZONE_APPVIEW_PUSH_EVENTS'),
     pdsUrl: envStr('OZONE_PDS_URL'),
     pdsDid: envStr('OZONE_PDS_DID'),
     dbPostgresUrl: envStr('OZONE_DB_POSTGRES_URL'),
@@ -23,9 +24,9 @@ export const readEnv = (): OzoneEnvironment => {
     moderatorDids: envList('OZONE_MODERATOR_DIDS'),
     triageDids: envList('OZONE_TRIAGE_DIDS'),
     adminPassword: envStr('OZONE_ADMIN_PASSWORD'),
-    moderatorPassword: envStr('OZONE_MODERATOR_PASSWORD'),
-    triagePassword: envStr('OZONE_TRIAGE_PASSWORD'),
     signingKeyHex: envStr('OZONE_SIGNING_KEY_HEX'),
+    blobDivertUrl: envStr('OZONE_BLOB_DIVERT_URL'),
+    blobDivertAdminPassword: envStr('OZONE_BLOB_DIVERT_ADMIN_PASSWORD'),
   }
 }
 
@@ -38,6 +39,7 @@ export type OzoneEnvironment = {
   serverDid?: string
   appviewUrl?: string
   appviewDid?: string
+  appviewPushEvents?: boolean
   pdsUrl?: string
   pdsDid?: string
   dbPostgresUrl?: string
@@ -51,7 +53,7 @@ export type OzoneEnvironment = {
   moderatorDids: string[]
   triageDids: string[]
   adminPassword?: string
-  moderatorPassword?: string
-  triagePassword?: string
   signingKeyHex?: string
+  blobDivertUrl?: string
+  blobDivertAdminPassword?: string
 }

package/src/config/secrets.ts

@@ -3,21 +3,15 @@ import { OzoneEnvironment } from './env'
 
 export const envToSecrets = (env: OzoneEnvironment): OzoneSecrets => {
   assert(env.adminPassword)
-  assert(env.moderatorPassword)
-  assert(env.triagePassword)
   assert(env.signingKeyHex)
 
   return {
     adminPassword: env.adminPassword,
-    moderatorPassword: env.moderatorPassword,
-    triagePassword: env.triagePassword,
     signingKeyHex: env.signingKeyHex,
   }
 }
 
 export type OzoneSecrets = {
   adminPassword: string
-  moderatorPassword: string
-  triagePassword: string
   signingKeyHex: string
 }

package/src/context.ts

@@ -14,8 +14,10 @@ import {
   CommunicationTemplateService,
   CommunicationTemplateServiceCreator,
 } from './communication-service/template'
+import { BlobDiverter } from './daemon/blob-diverter'
 import { AuthVerifier } from './auth-verifier'
 import { ImageInvalidator } from './image-invalidator'
+import { getSigningKeyId } from './util'
 
 export type AppContextOptions = {
   db: Database
@@ -24,7 +26,9 @@ export type AppContextOptions = {
   communicationTemplateService: CommunicationTemplateServiceCreator
   appviewAgent: AtpAgent
   pdsAgent: AtpAgent | undefined
+  blobDiverter?: BlobDiverter
   signingKey: Keypair
+  signingKeyId: number
   idResolver: IdResolver
   imgInvalidator?: ImageInvalidator
   backgroundQueue: BackgroundQueue
@@ -48,11 +52,16 @@ export class AppContext {
       poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
     })
     const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+    const signingKeyId = await getSigningKeyId(db, signingKey.did())
     const appviewAgent = new AtpAgent({ service: cfg.appview.url })
     const pdsAgent = cfg.pds
       ? new AtpAgent({ service: cfg.pds.url })
       : undefined
 
+    const idResolver = new IdResolver({
+      plcUrl: cfg.identity.plcUrl,
+    })
+
     const createAuthHeaders = (aud: string) =>
       createServiceAuthHeaders({
         iss: `${cfg.service.did}#atproto_labeler`,
@@ -61,30 +70,31 @@ export class AppContext {
       })
 
     const backgroundQueue = new BackgroundQueue(db)
+    const blobDiverter = cfg.blobDivert
+      ? new BlobDiverter(db, {
+          idResolver,
+          serviceConfig: cfg.blobDivert,
+        })
+      : undefined
     const eventPusher = new EventPusher(db, createAuthHeaders, {
-      appview: cfg.appview,
+      appview: cfg.appview.pushEvents ? cfg.appview : undefined,
       pds: cfg.pds ?? undefined,
     })
-
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-    })
-
     const modService = ModerationService.creator(
+      signingKey,
+      signingKeyId,
       cfg,
       backgroundQueue,
       idResolver,
       eventPusher,
       appviewAgent,
       createAuthHeaders,
-      cfg.service.did,
       overrides?.imgInvalidator,
-      cfg.cdn.paths,
     )
 
     const communicationTemplateService = CommunicationTemplateService.creator()
 
-    const sequencer = new Sequencer(db)
+    const sequencer = new Sequencer(modService(db))
 
     const authVerifier = new AuthVerifier(idResolver, {
       serviceDid: cfg.service.did,
@@ -92,8 +102,6 @@ export class AppContext {
       moderators: cfg.access.moderators,
       triage: cfg.access.triage,
       adminPassword: secrets.adminPassword,
-      moderatorPassword: secrets.moderatorPassword,
-      triagePassword: secrets.triagePassword,
     })
 
     return new AppContext(
@@ -105,10 +113,12 @@ export class AppContext {
         appviewAgent,
         pdsAgent,
         signingKey,
+        signingKeyId,
         idResolver,
         backgroundQueue,
         sequencer,
         authVerifier,
+        blobDiverter,
         ...(overrides ?? {}),
       },
       secrets,
@@ -135,6 +145,10 @@ export class AppContext {
     return this.opts.modService
   }
 
+  get blobDiverter(): BlobDiverter | undefined {
+    return this.opts.blobDiverter
+  }
+
   get communicationTemplateService(): CommunicationTemplateServiceCreator {
     return this.opts.communicationTemplateService
   }
@@ -151,6 +165,10 @@ export class AppContext {
     return this.opts.signingKey
   }
 
+  get signingKeyId(): number {
+    return this.opts.signingKeyId
+  }
+
   get plcClient(): plc.Client {
     return new plc.Client(this.cfg.identity.plcUrl)
   }
@@ -190,6 +208,12 @@ export class AppContext {
   async appviewAuth() {
     return this.serviceAuthHeaders(this.cfg.appview.did)
   }
-}
 
+  devOverride(overrides: Partial<AppContextOptions>) {
+    this.opts = {
+      ...this.opts,
+      ...overrides,
+    }
+  }
+}
 export default AppContext

package/src/daemon/blob-diverter.ts

@@ -0,0 +1,150 @@
+import {
+  VerifyCidTransform,
+  forwardStreamErrors,
+  getPdsEndpoint,
+} from '@atproto/common'
+import { IdResolver } from '@atproto/identity'
+import axios from 'axios'
+import { Readable } from 'stream'
+import { CID } from 'multiformats/cid'
+
+import Database from '../db'
+import { retryHttp } from '../util'
+import { BlobDivertConfig } from '../config'
+
+export class BlobDiverter {
+  serviceConfig: BlobDivertConfig
+  idResolver: IdResolver
+
+  constructor(
+    public db: Database,
+    services: {
+      idResolver: IdResolver
+      serviceConfig: BlobDivertConfig
+    },
+  ) {
+    this.serviceConfig = services.serviceConfig
+    this.idResolver = services.idResolver
+  }
+
+  private async getBlob({
+    pds,
+    did,
+    cid,
+  }: {
+    pds: string
+    did: string
+    cid: string
+  }) {
+    const blobResponse = await axios.get(
+      `${pds}/xrpc/com.atproto.sync.getBlob`,
+      {
+        params: { did, cid },
+        decompress: true,
+        responseType: 'stream',
+        timeout: 5000, // 5sec of inactivity on the connection
+      },
+    )
+    const imageStream: Readable = blobResponse.data
+    const verifyCid = new VerifyCidTransform(CID.parse(cid))
+    forwardStreamErrors(imageStream, verifyCid)
+
+    return {
+      contentType:
+        blobResponse.headers['content-type'] || 'application/octet-stream',
+      imageStream: imageStream.pipe(verifyCid),
+    }
+  }
+
+  async sendImage({
+    url,
+    imageStream,
+    contentType,
+  }: {
+    url: string
+    imageStream: Readable
+    contentType: string
+  }) {
+    const result = await axios(url, {
+      method: 'POST',
+      data: imageStream,
+      headers: {
+        Authorization: basicAuth('admin', this.serviceConfig.adminPassword),
+        'Content-Type': contentType,
+      },
+    })
+
+    return result.status === 200
+  }
+
+  private async uploadBlob(
+    {
+      imageStream,
+      contentType,
+    }: { imageStream: Readable; contentType: string },
+    {
+      subjectDid,
+      subjectUri,
+    }: { subjectDid: string; subjectUri: string | null },
+  ) {
+    const url = new URL(this.serviceConfig.url)
+    url.searchParams.set('did', subjectDid)
+    if (subjectUri) url.searchParams.set('uri', subjectUri)
+    const result = await this.sendImage({
+      url: url.toString(),
+      imageStream,
+      contentType,
+    })
+
+    return result
+  }
+
+  async uploadBlobOnService({
+    subjectDid,
+    subjectUri,
+    subjectBlobCids,
+  }: {
+    subjectDid: string
+    subjectUri: string
+    subjectBlobCids: string[]
+  }): Promise<boolean> {
+    const didDoc = await this.idResolver.did.resolve(subjectDid)
+
+    if (!didDoc) {
+      throw new Error('Error resolving DID')
+    }
+
+    const pds = getPdsEndpoint(didDoc)
+
+    if (!pds) {
+      throw new Error('Error resolving PDS')
+    }
+
+    // attempt to download and upload within the same retry block since the imageStream is not reusable
+    const uploadResult = await Promise.all(
+      subjectBlobCids.map((cid) =>
+        retryHttp(async () => {
+          const { imageStream, contentType } = await this.getBlob({
+            pds,
+            cid,
+            did: subjectDid,
+          })
+          return this.uploadBlob(
+            { imageStream, contentType },
+            { subjectDid, subjectUri },
+          )
+        }),
+      ),
+    )
+
+    if (uploadResult.includes(false)) {
+      throw new Error(`Error uploading blob ${subjectUri}`)
+    }
+
+    return true
+  }
+}
+
+const basicAuth = (username: string, password: string) => {
+  return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64')
+}

package/src/daemon/context.ts

@@ -1,5 +1,6 @@
 import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
 import { createServiceAuthHeaders } from '@atproto/xrpc-server'
+import { IdResolver } from '@atproto/identity'
 import AtpAgent from '@atproto/api'
 import { OzoneConfig, OzoneSecrets } from '../config'
 import { Database } from '../db'
@@ -7,7 +8,7 @@ import { EventPusher } from './event-pusher'
 import { EventReverser } from './event-reverser'
 import { ModerationService, ModerationServiceCreator } from '../mod-service'
 import { BackgroundQueue } from '../background'
-import { IdResolver } from '@atproto/identity'
+import { getSigningKeyId } from '../util'
 
 export type DaemonContextOptions = {
   db: Database
@@ -31,33 +32,36 @@ export class DaemonContext {
       schema: cfg.db.postgresSchema,
     })
     const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+    const signingKeyId = await getSigningKeyId(db, signingKey.did())
+
+    const idResolver = new IdResolver({
+      plcUrl: cfg.identity.plcUrl,
+    })
 
     const appviewAgent = new AtpAgent({ service: cfg.appview.url })
     const createAuthHeaders = (aud: string) =>
       createServiceAuthHeaders({
-        iss: cfg.service.did,
+        iss: `${cfg.service.did}#atproto_labeler`,
         aud,
         keypair: signingKey,
       })
 
     const eventPusher = new EventPusher(db, createAuthHeaders, {
-      appview: cfg.appview,
+      appview: cfg.appview.pushEvents ? cfg.appview : undefined,
       pds: cfg.pds ?? undefined,
     })
 
     const backgroundQueue = new BackgroundQueue(db)
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-    })
 
     const modService = ModerationService.creator(
+      signingKey,
+      signingKeyId,
       cfg,
       backgroundQueue,
       idResolver,
       eventPusher,
       appviewAgent,
       createAuthHeaders,
-      cfg.service.did,
     )
 
     const eventReverser = new EventReverser(db, modService)

package/src/daemon/event-pusher.ts

@@ -1,10 +1,13 @@
+import assert from 'node:assert'
 import AtpAgent from '@atproto/api'
 import { SECOND } from '@atproto/common'
 import Database from '../db'
+import { RepoPushEventType } from '../db/schema/repo_push_event'
 import { retryHttp } from '../util'
 import { dbLogger } from '../logger'
 import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus'
-import assert from 'assert'
+import { BlobPushEvent } from '../db/schema/blob_push_event'
+import { Insertable, Selectable } from 'kysely'
 
 type EventSubject = InputSchema['subject']
 
@@ -74,6 +77,13 @@ export class EventPusher {
     this.poll(this.blobPollState, () => this.pushBlobEvents())
   }
 
+  get takedowns(): RepoPushEventType[] {
+    const takedowns: RepoPushEventType[] = []
+    if (this.pds) takedowns.push('pds_takedown')
+    if (this.appview) takedowns.push('appview_takedown')
+    return takedowns
+  }
+
   poll(state: PollState, fn: () => Promise<void>) {
     if (this.destroyed) return
     state.promise = fn()
@@ -277,20 +287,53 @@ export class EventPusher {
         subject,
         evt.takedownRef,
       )
-      await dbTxn.db
-        .updateTable('blob_push_event')
-        .set(
-          succeeded
-            ? { confirmedAt: new Date() }
-            : {
-                lastAttempted: new Date(),
-                attempts: (evt.attempts ?? 0) + 1,
-              },
-        )
-        .where('subjectDid', '=', evt.subjectDid)
-        .where('subjectBlobCid', '=', evt.subjectBlobCid)
-        .where('eventType', '=', evt.eventType)
-        .execute()
+      await this.markBlobEventAttempt(dbTxn, evt, succeeded)
     })
   }
+
+  async markBlobEventAttempt(
+    dbTxn: Database,
+    event: Selectable<BlobPushEvent>,
+    succeeded: boolean,
+  ) {
+    await dbTxn.db
+      .updateTable('blob_push_event')
+      .set(
+        succeeded
+          ? { confirmedAt: new Date() }
+          : {
+              lastAttempted: new Date(),
+              attempts: (event.attempts ?? 0) + 1,
+            },
+      )
+      .where('subjectDid', '=', event.subjectDid)
+      .where('subjectBlobCid', '=', event.subjectBlobCid)
+      .where('eventType', '=', event.eventType)
+      .execute()
+  }
+
+  async logBlobPushEvent(
+    blobValues: Insertable<BlobPushEvent>[],
+    takedownRef?: string | null,
+  ) {
+    return this.db.db
+      .insertInto('blob_push_event')
+      .values(blobValues)
+      .onConflict((oc) =>
+        oc.columns(['subjectDid', 'subjectBlobCid', 'eventType']).doUpdateSet({
+          takedownRef,
+          confirmedAt: null,
+          attempts: 0,
+          lastAttempted: null,
+        }),
+      )
+      .returning([
+        'id',
+        'subjectDid',
+        'subjectUri',
+        'subjectBlobCid',
+        'eventType',
+      ])
+      .execute()
+  }
 }