@atproto/ozone 0.0.17-next.0 → 0.0.17-next.1

package/src/context.ts

@@ -14,8 +14,10 @@ import {
   CommunicationTemplateService,
   CommunicationTemplateServiceCreator,
 } from './communication-service/template'
+import { BlobDiverter } from './daemon/blob-diverter'
 import { AuthVerifier } from './auth-verifier'
 import { ImageInvalidator } from './image-invalidator'
+import { getSigningKeyId } from './util'
 
 export type AppContextOptions = {
   db: Database
@@ -24,7 +26,9 @@ export type AppContextOptions = {
   communicationTemplateService: CommunicationTemplateServiceCreator
   appviewAgent: AtpAgent
   pdsAgent: AtpAgent | undefined
+  blobDiverter?: BlobDiverter
   signingKey: Keypair
+  signingKeyId: number
   idResolver: IdResolver
   imgInvalidator?: ImageInvalidator
   backgroundQueue: BackgroundQueue
@@ -48,11 +52,16 @@ export class AppContext {
       poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
     })
     const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+    const signingKeyId = await getSigningKeyId(db, signingKey.did())
     const appviewAgent = new AtpAgent({ service: cfg.appview.url })
     const pdsAgent = cfg.pds
       ? new AtpAgent({ service: cfg.pds.url })
       : undefined
 
+    const idResolver = new IdResolver({
+      plcUrl: cfg.identity.plcUrl,
+    })
+
     const createAuthHeaders = (aud: string) =>
       createServiceAuthHeaders({
         iss: `${cfg.service.did}#atproto_labeler`,
@@ -61,30 +70,31 @@ export class AppContext {
       })
 
     const backgroundQueue = new BackgroundQueue(db)
+    const blobDiverter = cfg.blobDivert
+      ? new BlobDiverter(db, {
+          idResolver,
+          serviceConfig: cfg.blobDivert,
+        })
+      : undefined
     const eventPusher = new EventPusher(db, createAuthHeaders, {
       appview: cfg.appview.pushEvents ? cfg.appview : undefined,
       pds: cfg.pds ?? undefined,
     })
-
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-    })
-
     const modService = ModerationService.creator(
+      signingKey,
+      signingKeyId,
       cfg,
       backgroundQueue,
       idResolver,
       eventPusher,
       appviewAgent,
       createAuthHeaders,
-      cfg.service.did,
       overrides?.imgInvalidator,
-      cfg.cdn.paths,
     )
 
     const communicationTemplateService = CommunicationTemplateService.creator()
 
-    const sequencer = new Sequencer(db)
+    const sequencer = new Sequencer(modService(db))
 
     const authVerifier = new AuthVerifier(idResolver, {
       serviceDid: cfg.service.did,
@@ -103,10 +113,12 @@ export class AppContext {
         appviewAgent,
         pdsAgent,
         signingKey,
+        signingKeyId,
         idResolver,
         backgroundQueue,
         sequencer,
         authVerifier,
+        blobDiverter,
         ...(overrides ?? {}),
       },
       secrets,
@@ -133,6 +145,10 @@ export class AppContext {
     return this.opts.modService
   }
 
+  get blobDiverter(): BlobDiverter | undefined {
+    return this.opts.blobDiverter
+  }
+
   get communicationTemplateService(): CommunicationTemplateServiceCreator {
     return this.opts.communicationTemplateService
   }
@@ -149,6 +165,10 @@ export class AppContext {
     return this.opts.signingKey
   }
 
+  get signingKeyId(): number {
+    return this.opts.signingKeyId
+  }
+
   get plcClient(): plc.Client {
     return new plc.Client(this.cfg.identity.plcUrl)
   }
@@ -188,6 +208,12 @@ export class AppContext {
   async appviewAuth() {
     return this.serviceAuthHeaders(this.cfg.appview.did)
   }
-}
 
+  devOverride(overrides: Partial<AppContextOptions>) {
+    this.opts = {
+      ...this.opts,
+      ...overrides,
+    }
+  }
+}
 export default AppContext

package/src/daemon/blob-diverter.ts

@@ -0,0 +1,150 @@
+import {
+  VerifyCidTransform,
+  forwardStreamErrors,
+  getPdsEndpoint,
+} from '@atproto/common'
+import { IdResolver } from '@atproto/identity'
+import axios from 'axios'
+import { Readable } from 'stream'
+import { CID } from 'multiformats/cid'
+
+import Database from '../db'
+import { retryHttp } from '../util'
+import { BlobDivertConfig } from '../config'
+
+export class BlobDiverter {
+  serviceConfig: BlobDivertConfig
+  idResolver: IdResolver
+
+  constructor(
+    public db: Database,
+    services: {
+      idResolver: IdResolver
+      serviceConfig: BlobDivertConfig
+    },
+  ) {
+    this.serviceConfig = services.serviceConfig
+    this.idResolver = services.idResolver
+  }
+
+  private async getBlob({
+    pds,
+    did,
+    cid,
+  }: {
+    pds: string
+    did: string
+    cid: string
+  }) {
+    const blobResponse = await axios.get(
+      `${pds}/xrpc/com.atproto.sync.getBlob`,
+      {
+        params: { did, cid },
+        decompress: true,
+        responseType: 'stream',
+        timeout: 5000, // 5sec of inactivity on the connection
+      },
+    )
+    const imageStream: Readable = blobResponse.data
+    const verifyCid = new VerifyCidTransform(CID.parse(cid))
+    forwardStreamErrors(imageStream, verifyCid)
+
+    return {
+      contentType:
+        blobResponse.headers['content-type'] || 'application/octet-stream',
+      imageStream: imageStream.pipe(verifyCid),
+    }
+  }
+
+  async sendImage({
+    url,
+    imageStream,
+    contentType,
+  }: {
+    url: string
+    imageStream: Readable
+    contentType: string
+  }) {
+    const result = await axios(url, {
+      method: 'POST',
+      data: imageStream,
+      headers: {
+        Authorization: basicAuth('admin', this.serviceConfig.adminPassword),
+        'Content-Type': contentType,
+      },
+    })
+
+    return result.status === 200
+  }
+
+  private async uploadBlob(
+    {
+      imageStream,
+      contentType,
+    }: { imageStream: Readable; contentType: string },
+    {
+      subjectDid,
+      subjectUri,
+    }: { subjectDid: string; subjectUri: string | null },
+  ) {
+    const url = new URL(this.serviceConfig.url)
+    url.searchParams.set('did', subjectDid)
+    if (subjectUri) url.searchParams.set('uri', subjectUri)
+    const result = await this.sendImage({
+      url: url.toString(),
+      imageStream,
+      contentType,
+    })
+
+    return result
+  }
+
+  async uploadBlobOnService({
+    subjectDid,
+    subjectUri,
+    subjectBlobCids,
+  }: {
+    subjectDid: string
+    subjectUri: string
+    subjectBlobCids: string[]
+  }): Promise<boolean> {
+    const didDoc = await this.idResolver.did.resolve(subjectDid)
+
+    if (!didDoc) {
+      throw new Error('Error resolving DID')
+    }
+
+    const pds = getPdsEndpoint(didDoc)
+
+    if (!pds) {
+      throw new Error('Error resolving PDS')
+    }
+
+    // attempt to download and upload within the same retry block since the imageStream is not reusable
+    const uploadResult = await Promise.all(
+      subjectBlobCids.map((cid) =>
+        retryHttp(async () => {
+          const { imageStream, contentType } = await this.getBlob({
+            pds,
+            cid,
+            did: subjectDid,
+          })
+          return this.uploadBlob(
+            { imageStream, contentType },
+            { subjectDid, subjectUri },
+          )
+        }),
+      ),
+    )
+
+    if (uploadResult.includes(false)) {
+      throw new Error(`Error uploading blob ${subjectUri}`)
+    }
+
+    return true
+  }
+}
+
+const basicAuth = (username: string, password: string) => {
+  return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64')
+}

package/src/daemon/context.ts

@@ -1,5 +1,6 @@
 import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
 import { createServiceAuthHeaders } from '@atproto/xrpc-server'
+import { IdResolver } from '@atproto/identity'
 import AtpAgent from '@atproto/api'
 import { OzoneConfig, OzoneSecrets } from '../config'
 import { Database } from '../db'
@@ -7,7 +8,7 @@ import { EventPusher } from './event-pusher'
 import { EventReverser } from './event-reverser'
 import { ModerationService, ModerationServiceCreator } from '../mod-service'
 import { BackgroundQueue } from '../background'
-import { IdResolver } from '@atproto/identity'
+import { getSigningKeyId } from '../util'
 
 export type DaemonContextOptions = {
   db: Database
@@ -31,6 +32,11 @@ export class DaemonContext {
       schema: cfg.db.postgresSchema,
     })
     const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
+    const signingKeyId = await getSigningKeyId(db, signingKey.did())
+
+    const idResolver = new IdResolver({
+      plcUrl: cfg.identity.plcUrl,
+    })
 
     const appviewAgent = new AtpAgent({ service: cfg.appview.url })
     const createAuthHeaders = (aud: string) =>
@@ -46,18 +52,16 @@ export class DaemonContext {
     })
 
     const backgroundQueue = new BackgroundQueue(db)
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-    })
 
     const modService = ModerationService.creator(
+      signingKey,
+      signingKeyId,
       cfg,
       backgroundQueue,
       idResolver,
       eventPusher,
       appviewAgent,
       createAuthHeaders,
-      cfg.service.did,
     )
 
     const eventReverser = new EventReverser(db, modService)

package/src/daemon/event-pusher.ts

@@ -6,6 +6,8 @@ import { RepoPushEventType } from '../db/schema/repo_push_event'
 import { retryHttp } from '../util'
 import { dbLogger } from '../logger'
 import { InputSchema } from '../lexicon/types/com/atproto/admin/updateSubjectStatus'
+import { BlobPushEvent } from '../db/schema/blob_push_event'
+import { Insertable, Selectable } from 'kysely'
 
 type EventSubject = InputSchema['subject']
 
@@ -285,20 +287,53 @@ export class EventPusher {
         subject,
         evt.takedownRef,
       )
-      await dbTxn.db
-        .updateTable('blob_push_event')
-        .set(
-          succeeded
-            ? { confirmedAt: new Date() }
-            : {
-                lastAttempted: new Date(),
-                attempts: (evt.attempts ?? 0) + 1,
-              },
-        )
-        .where('subjectDid', '=', evt.subjectDid)
-        .where('subjectBlobCid', '=', evt.subjectBlobCid)
-        .where('eventType', '=', evt.eventType)
-        .execute()
+      await this.markBlobEventAttempt(dbTxn, evt, succeeded)
     })
   }
+
+  async markBlobEventAttempt(
+    dbTxn: Database,
+    event: Selectable<BlobPushEvent>,
+    succeeded: boolean,
+  ) {
+    await dbTxn.db
+      .updateTable('blob_push_event')
+      .set(
+        succeeded
+          ? { confirmedAt: new Date() }
+          : {
+              lastAttempted: new Date(),
+              attempts: (event.attempts ?? 0) + 1,
+            },
+      )
+      .where('subjectDid', '=', event.subjectDid)
+      .where('subjectBlobCid', '=', event.subjectBlobCid)
+      .where('eventType', '=', event.eventType)
+      .execute()
+  }
+
+  async logBlobPushEvent(
+    blobValues: Insertable<BlobPushEvent>[],
+    takedownRef?: string | null,
+  ) {
+    return this.db.db
+      .insertInto('blob_push_event')
+      .values(blobValues)
+      .onConflict((oc) =>
+        oc.columns(['subjectDid', 'subjectBlobCid', 'eventType']).doUpdateSet({
+          takedownRef,
+          confirmedAt: null,
+          attempts: 0,
+          lastAttempted: null,
+        }),
+      )
+      .returning([
+        'id',
+        'subjectDid',
+        'subjectUri',
+        'subjectBlobCid',
+        'eventType',
+      ])
+      .execute()
+  }
 }

package/src/daemon/index.ts

@@ -3,6 +3,7 @@ import DaemonContext from './context'
 import { AppContextOptions } from '../context'
 
 export { EventPusher } from './event-pusher'
+export { BlobDiverter } from './blob-diverter'
 export { EventReverser } from './event-reverser'
 
 export class OzoneDaemon {

package/src/db/migrations/20240228T003647759Z-add-label-sigs.ts

@@ -0,0 +1,25 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('label').addColumn('exp', 'varchar').execute()
+  await db.schema
+    .alterTable('label')
+    .addColumn('sig', sql`bytea`)
+    .execute()
+  await db.schema
+    .alterTable('label')
+    .addColumn('signingKeyId', 'integer')
+    .execute()
+  await db.schema
+    .createTable('signing_key')
+    .addColumn('id', 'serial', (col) => col.primaryKey())
+    .addColumn('key', 'varchar', (col) => col.notNull().unique())
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('signing_key')
+  await db.schema.alterTable('label').dropColumn('exp').execute()
+  await db.schema.alterTable('label').dropColumn('sig').execute()
+  await db.schema.alterTable('label').dropColumn('signingKey').execute()
+}

package/src/db/migrations/index.ts

@@ -6,3 +6,4 @@ export * as _20231219T205730722Z from './20231219T205730722Z-init'
 export * as _20240116T085607200Z from './20240116T085607200Z-communication-template'
 export * as _20240201T051104136Z from './20240201T051104136Z-mod-event-blobs'
 export * as _20240208T213404429Z from './20240208T213404429Z-add-tags-column-to-moderation-subject'
+export * as _20240228T003647759Z from './20240228T003647759Z-add-label-sigs'

package/src/db/schema/index.ts

@@ -5,11 +5,13 @@ import * as repoPushEvent from './repo_push_event'
 import * as recordPushEvent from './record_push_event'
 import * as blobPushEvent from './blob_push_event'
 import * as label from './label'
+import * as signingKey from './signing_key'
 import * as communicationTemplate from './communication_template'
 
 export type DatabaseSchemaType = modEvent.PartialDB &
   modSubjectStatus.PartialDB &
   label.PartialDB &
+  signingKey.PartialDB &
   repoPushEvent.PartialDB &
   recordPushEvent.PartialDB &
   blobPushEvent.PartialDB &

package/src/db/schema/label.ts

@@ -10,6 +10,9 @@ export interface Label {
   val: string
   neg: boolean
   cts: string
+  exp: string | null
+  sig: Buffer | null
+  signingKeyId: number | null
 }
 
 export type LabelRow = Selectable<Label>

package/src/db/schema/moderation_event.ts

@@ -5,17 +5,17 @@ export const eventTableName = 'moderation_event'
 export interface ModerationEvent {
   id: Generated<number>
   action:
-    | 'com.atproto.admin.defs#modEventTakedown'
-    | 'com.atproto.admin.defs#modEventAcknowledge'
-    | 'com.atproto.admin.defs#modEventEscalate'
-    | 'com.atproto.admin.defs#modEventComment'
-    | 'com.atproto.admin.defs#modEventLabel'
-    | 'com.atproto.admin.defs#modEventReport'
-    | 'com.atproto.admin.defs#modEventMute'
-    | 'com.atproto.admin.defs#modEventReverseTakedown'
-    | 'com.atproto.admin.defs#modEventEmail'
-    | 'com.atproto.admin.defs#modEventResolveAppeal'
-    | 'com.atproto.admin.defs#modEventTag'
+    | 'tools.ozone.moderation.defs#modEventTakedown'
+    | 'tools.ozone.moderation.defs#modEventAcknowledge'
+    | 'tools.ozone.moderation.defs#modEventEscalate'
+    | 'tools.ozone.moderation.defs#modEventComment'
+    | 'tools.ozone.moderation.defs#modEventLabel'
+    | 'tools.ozone.moderation.defs#modEventReport'
+    | 'tools.ozone.moderation.defs#modEventMute'
+    | 'tools.ozone.moderation.defs#modEventReverseTakedown'
+    | 'tools.ozone.moderation.defs#modEventEmail'
+    | 'tools.ozone.moderation.defs#modEventResolveAppeal'
+    | 'tools.ozone.moderation.defs#modEventTag'
   subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
   subjectDid: string
   subjectUri: string | null

package/src/db/schema/moderation_subject_status.ts

@@ -4,7 +4,7 @@ import {
   REVIEWOPEN,
   REVIEWESCALATED,
   REVIEWNONE,
-} from '../../lexicon/types/com/atproto/admin/defs'
+} from '../../lexicon/types/tools/ozone/moderation/defs'
 
 export const subjectStatusTableName = 'moderation_subject_status'
 

package/src/db/schema/signing_key.ts

@@ -0,0 +1,10 @@
+import { Generated } from 'kysely'
+
+export const tableName = 'signing_key'
+
+export interface SigningKey {
+  id: Generated<number>
+  key: string
+}
+
+export type PartialDB = { [tableName]: SigningKey }