@atproto/ozone 0.0.13 → 0.0.15

package/dist/mod-service/index.d.ts

@@ -11,6 +11,7 @@ import { Label } from '../lexicon/types/com/atproto/label/defs';
 import { ModSubject, RecordSubject, RepoSubject } from './subject';
 import { BackgroundQueue } from '../background';
 import { EventPusher } from '../daemon';
+import { ImageInvalidator } from '../image-invalidator';
 export type ModerationServiceCreator = (db: Database) => ModerationService;
 export declare class ModerationService {
     db: Database;
@@ -19,8 +20,10 @@ export declare class ModerationService {
     appviewAgent: AtpAgent;
     private appviewAuth;
     serverDid: string;
-    constructor(db: Database, backgroundQueue: BackgroundQueue, eventPusher: EventPusher, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string);
-    static creator(backgroundQueue: BackgroundQueue, eventPusher: EventPusher, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string): (db: Database) => ModerationService;
+    imgInvalidator?: ImageInvalidator | undefined;
+    cdnPaths?: string[] | undefined;
+    constructor(db: Database, backgroundQueue: BackgroundQueue, eventPusher: EventPusher, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string, imgInvalidator?: ImageInvalidator | undefined, cdnPaths?: string[] | undefined);
+    static creator(backgroundQueue: BackgroundQueue, eventPusher: EventPusher, appviewAgent: AtpAgent, appviewAuth: AppviewAuth, serverDid: string, imgInvalidator?: ImageInvalidator, cdnPaths?: string[]): (db: Database) => ModerationService;
     views: ModerationViews;
     getEvent(id: number): Promise<ModerationEventRow | undefined>;
     getEventOrThrow(id: number): Promise<ModerationEventRow>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/ozone",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "license": "MIT",
   "description": "Backend service for moderating the Bluesky network.",
   "keywords": [
@@ -30,7 +30,7 @@
     "pino-http": "^8.2.1",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
-    "@atproto/api": "^0.10.2",
+    "@atproto/api": "^0.10.4",
     "@atproto/common": "^0.3.3",
     "@atproto/crypto": "^0.3.0",
     "@atproto/identity": "^0.3.2",
@@ -46,10 +46,10 @@
     "@types/pg": "^8.6.6",
     "@types/qs": "^6.9.7",
     "axios": "^0.27.2",
-    "@atproto/api": "^0.10.2",
-    "@atproto/dev-env": "^0.2.34",
+    "@atproto/api": "^0.10.4",
+    "@atproto/dev-env": "^0.2.36",
     "@atproto/lex-cli": "^0.3.1",
-    "@atproto/pds": "^0.4.2",
+    "@atproto/pds": "^0.4.4",
     "@atproto/xrpc": "^0.4.2"
   },
   "scripts": {

package/src/config/config.ts

@@ -38,6 +38,10 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     did: env.pdsDid,
   }
 
+  const cdnCfg: OzoneConfig['cdn'] = {
+    paths: env.cdnPaths,
+  }
+
   assert(env.didPlcUrl)
   const identityCfg: OzoneConfig['identity'] = {
     plcUrl: env.didPlcUrl,
@@ -48,6 +52,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
     db: dbCfg,
     appview: appviewCfg,
     pds: pdsCfg,
+    cdn: cdnCfg,
     identity: identityCfg,
   }
 }
@@ -57,6 +62,7 @@ export type OzoneConfig = {
   db: DatabaseConfig
   appview: AppviewConfig
   pds: PdsConfig | null
+  cdn: CdnConfig
   identity: IdentityConfig
 }
 
@@ -88,3 +94,7 @@ export type PdsConfig = {
 export type IdentityConfig = {
   plcUrl: string
 }
+
+export type CdnConfig = {
+  paths?: string[]
+}

package/src/config/env.ts

@@ -1,4 +1,4 @@
-import { envInt, envStr } from '@atproto/common'
+import { envInt, envList, envStr } from '@atproto/common'
 
 export const readEnv = (): OzoneEnvironment => {
   return {
@@ -17,6 +17,7 @@ export const readEnv = (): OzoneEnvironment => {
     dbPoolMaxUses: envInt('OZONE_DB_POOL_MAX_USES'),
     dbPoolIdleTimeoutMs: envInt('OZONE_DB_POOL_IDLE_TIMEOUT_MS'),
     didPlcUrl: envStr('OZONE_DID_PLC_URL'),
+    cdnPaths: envList('OZONE_CDN_PATHS'),
     adminPassword: envStr('OZONE_ADMIN_PASSWORD'),
     moderatorPassword: envStr('OZONE_MODERATOR_PASSWORD'),
     triagePassword: envStr('OZONE_TRIAGE_PASSWORD'),
@@ -40,6 +41,7 @@ export type OzoneEnvironment = {
   dbPoolMaxUses?: number
   dbPoolIdleTimeoutMs?: number
   didPlcUrl?: string
+  cdnPaths?: string[]
   adminPassword?: string
   moderatorPassword?: string
   triagePassword?: string

package/src/context.ts

@@ -15,6 +15,7 @@ import {
   CommunicationTemplateService,
   CommunicationTemplateServiceCreator,
 } from './communication-service/template'
+import { ImageInvalidator } from './image-invalidator'
 
 export type AppContextOptions = {
   db: Database
@@ -25,6 +26,7 @@ export type AppContextOptions = {
   pdsAgent: AtpAgent | undefined
   signingKey: Keypair
   idResolver: IdResolver
+  imgInvalidator?: ImageInvalidator
   backgroundQueue: BackgroundQueue
   sequencer: Sequencer
 }
@@ -71,6 +73,8 @@ export class AppContext {
       appviewAgent,
       appviewAuth,
       cfg.service.did,
+      overrides?.imgInvalidator,
+      cfg.cdn.paths,
     )
 
     const communicationTemplateService = CommunicationTemplateService.creator()

package/src/image-invalidator.ts

@@ -0,0 +1,7 @@
+// Invalidation is a general interface for propagating an image blob
+// takedown through any caches where a representation of it may be stored.
+// @NOTE this does not remove the blob from storage: just invalidates it from caches.
+// @NOTE keep in sync with same interface in aws/src/cloudfront.ts
+export interface ImageInvalidator {
+  invalidate(subject: string, paths: string[]): Promise<void>
+}

package/src/index.ts

@@ -13,6 +13,7 @@ import { createServer } from './lexicon'
 import AppContext, { AppContextOptions } from './context'
 
 export * from './config'
+export { type ImageInvalidator } from './image-invalidator'
 export { Database } from './db'
 export { OzoneDaemon, EventPusher, EventReverser } from './daemon'
 export { AppContext } from './context'

package/src/mod-service/index.ts

@@ -39,11 +39,13 @@ import {
   RepoSubject,
   subjectFromStatusRow,
 } from './subject'
+import { jsonb } from '../db/types'
+import { LabelChannel } from '../db/schema/label'
 import { BlobPushEvent } from '../db/schema/blob_push_event'
 import { BackgroundQueue } from '../background'
 import { EventPusher } from '../daemon'
-import { jsonb } from '../db/types'
-import { LabelChannel } from '../db/schema/label'
+import { ImageInvalidator } from '../image-invalidator'
+import { httpLogger as log } from '../logger'
 
 export type ModerationServiceCreator = (db: Database) => ModerationService
 
@@ -55,6 +57,8 @@ export class ModerationService {
     public appviewAgent: AtpAgent,
     private appviewAuth: AppviewAuth,
     public serverDid: string,
+    public imgInvalidator?: ImageInvalidator,
+    public cdnPaths?: string[],
   ) {}
 
   static creator(
@@ -63,6 +67,8 @@ export class ModerationService {
     appviewAgent: AtpAgent,
     appviewAuth: AppviewAuth,
     serverDid: string,
+    imgInvalidator?: ImageInvalidator,
+    cdnPaths?: string[],
   ) {
     return (db: Database) =>
       new ModerationService(
@@ -72,6 +78,8 @@ export class ModerationService {
         appviewAgent,
         appviewAuth,
         serverDid,
+        imgInvalidator,
+        cdnPaths,
       )
   }
 
@@ -556,14 +564,38 @@ export class ModerationService {
               lastAttempted: null,
             }),
         )
-        .returning('id')
+        .returning(['id', 'subjectDid', 'subjectBlobCid', 'eventType'])
         .execute()
 
       this.db.onCommit(() => {
         this.backgroundQueue.add(async () => {
-          await Promise.all(
-            blobEvts.map((evt) => this.eventPusher.attemptBlobEvent(evt.id)),
+          await Promise.allSettled(
+            blobEvts.map((evt) =>
+              this.eventPusher
+                .attemptBlobEvent(evt.id)
+                .catch((err) =>
+                  log.error({ err, ...evt }, 'failed to push blob event'),
+                ),
+            ),
           )
+
+          if (this.imgInvalidator) {
+            await Promise.allSettled(
+              (subject.blobCids ?? []).map((cid) => {
+                const paths = (this.cdnPaths ?? []).map((path) =>
+                  path.replace('%s', subject.did).replace('%s', cid),
+                )
+                return this.imgInvalidator
+                  ?.invalidate(cid, paths)
+                  .catch((err) =>
+                    log.error(
+                      { err, paths, cid },
+                      'failed to invalidate blob on cdn',
+                    ),
+                  )
+              }),
+            )
+          }
         })
       })
     }

package/tests/moderation.test.ts

@@ -25,6 +25,7 @@ import {
 } from '../src/lexicon/types/com/atproto/admin/defs'
 import { EventReverser } from '../src'
 import { TestOzone } from '@atproto/dev-env/src/ozone'
+import { ImageInvalidator } from '../src/image-invalidator'
 import {
   UNSPECCED_TAKEDOWN_BLOBS_LABEL,
   UNSPECCED_TAKEDOWN_LABEL,
@@ -43,6 +44,7 @@ type TakedownParams = BaseCreateReportParams &
 describe('moderation', () => {
   let network: TestNetwork
   let ozone: TestOzone
+  let mockInvalidator: MockInvalidator
   let agent: AtpAgent
   let bskyAgent: AtpAgent
   let pdsAgent: AtpAgent
@@ -155,8 +157,13 @@ describe('moderation', () => {
   }
 
   beforeAll(async () => {
+    mockInvalidator = new MockInvalidator()
     network = await TestNetwork.create({
       dbPostgresSchema: 'ozone_moderation',
+      ozone: {
+        imgInvalidator: mockInvalidator,
+        cdnPaths: ['/path1/%s/%s', '/path2/%s/%s'],
+      },
     })
     ozone = network.ozone
     agent = network.ozone.getClient()
@@ -981,6 +988,18 @@ describe('moderation', () => {
       expect(await fetchImage.json()).toEqual({ message: 'Image not found' })
     })
 
+    it('invalidates the image in the cdn', async () => {
+      const blobCid = blob.image.ref.toString()
+      expect(mockInvalidator.invalidated.length).toBe(1)
+      expect(mockInvalidator.invalidated.at(0)?.subject).toBe(blobCid)
+      expect(mockInvalidator.invalidated.at(0)?.paths.at(0)).toEqual(
+        `/path1/${sc.dids.carol}/${blobCid}`,
+      )
+      expect(mockInvalidator.invalidated.at(0)?.paths.at(1)).toEqual(
+        `/path2/${sc.dids.carol}/${blobCid}`,
+      )
+    })
+
     it('fans takedown out to pds', async () => {
       const res = await pdsAgent.api.com.atproto.admin.getSubjectStatus(
         {
@@ -1059,3 +1078,11 @@ describe('moderation', () => {
     })
   })
 })
+
+class MockInvalidator implements ImageInvalidator {
+  invalidated: { subject: string; paths: string[] }[] = []
+
+  async invalidate(subject: string, paths: string[]) {
+    this.invalidated.push({ subject, paths })
+  }
+}