@atproto/ozone 0.2.8 → 0.2.11

package/src/context.ts

@@ -1,399 +0,0 @@
-import assert from 'node:assert'
-import * as plc from '@did-plc/lib'
-import express from 'express'
-import { AtpAgent } from '@atproto/api'
-import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
-import { DidCache, IdResolver, MemoryCache } from '@atproto/identity'
-import { createServiceAuthHeaders } from '@atproto/xrpc-server'
-import { AssignmentService } from './assignment/index.js'
-import { AuthVerifier } from './auth-verifier.js'
-import { BackgroundQueue } from './background.js'
-import {
-  CommunicationTemplateService,
-  CommunicationTemplateServiceCreator,
-} from './communication-service/template.js'
-import { OzoneConfig, OzoneSecrets } from './config/index.js'
-import { BlobDiverter } from './daemon/blob-diverter.js'
-import { EventPusher } from './daemon/index.js'
-import { Database } from './db/index.js'
-import { ImageInvalidator } from './image-invalidator.js'
-import {
-  ModerationService,
-  ModerationServiceCreator,
-} from './mod-service/index.js'
-import {
-  ModerationServiceProfile,
-  ModerationServiceProfileCreator,
-} from './mod-service/profile.js'
-import { StrikeService, StrikeServiceCreator } from './mod-service/strike.js'
-import { QueueService, QueueServiceCreator } from './queue/service.js'
-import {
-  ReportStatsService,
-  ReportStatsServiceCreator,
-} from './report/stats.js'
-import {
-  SafelinkRuleService,
-  SafelinkRuleServiceCreator,
-} from './safelink/service.js'
-import {
-  ScheduledActionService,
-  ScheduledActionServiceCreator,
-} from './scheduled-action/service.js'
-import { Sequencer } from './sequencer/sequencer.js'
-import { SetService, SetServiceCreator } from './set/service.js'
-import { SettingService, SettingServiceCreator } from './setting/service.js'
-import { TeamService, TeamServiceCreator } from './team/index.js'
-import {
-  LABELER_HEADER_NAME,
-  ParsedLabelers,
-  defaultLabelerHeader,
-  getSigningKeyId,
-  parseLabelerHeader,
-} from './util.js'
-import {
-  VerificationIssuer,
-  VerificationIssuerCreator,
-} from './verification/issuer.js'
-import {
-  VerificationService,
-  VerificationServiceCreator,
-} from './verification/service.js'
-
-export type AppContextOptions = {
-  db: Database
-  cfg: OzoneConfig
-  modService: ModerationServiceCreator
-  moderationServiceProfile: ModerationServiceProfileCreator
-  communicationTemplateService: CommunicationTemplateServiceCreator
-  safelinkRuleService: SafelinkRuleServiceCreator
-  scheduledActionService: ScheduledActionServiceCreator
-  queueService: QueueServiceCreator
-  reportStatsService: ReportStatsServiceCreator
-  setService: SetServiceCreator
-  settingService: SettingServiceCreator
-  strikeService: StrikeServiceCreator
-  teamService: TeamServiceCreator
-  appviewAgent: AtpAgent
-  pdsAgent: AtpAgent | undefined
-  chatAgent: AtpAgent | undefined
-  blobDiverter?: BlobDiverter
-  signingKey: Keypair
-  signingKeyId: number
-  didCache: DidCache
-  idResolver: IdResolver
-  imgInvalidator?: ImageInvalidator
-  backgroundQueue: BackgroundQueue
-  sequencer: Sequencer
-  assignmentService: AssignmentService
-  authVerifier: AuthVerifier
-  verificationService: VerificationServiceCreator
-  verificationIssuer: VerificationIssuerCreator
-}
-
-export class AppContext {
-  constructor(
-    private opts: AppContextOptions,
-    private secrets: OzoneSecrets,
-  ) {}
-
-  static async fromConfig(
-    cfg: OzoneConfig,
-    secrets: OzoneSecrets,
-    overrides?: Partial<AppContextOptions>,
-  ): Promise<AppContext> {
-    const db = new Database({
-      url: cfg.db.postgresUrl,
-      schema: cfg.db.postgresSchema,
-      poolSize: cfg.db.poolSize,
-      poolMaxUses: cfg.db.poolMaxUses,
-      poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
-    })
-    const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
-    const signingKeyId = await getSigningKeyId(db, signingKey.did())
-    const appviewAgent = new AtpAgent({ service: cfg.appview.url })
-    const pdsAgent = cfg.pds
-      ? new AtpAgent({ service: cfg.pds.url })
-      : undefined
-    const chatAgent = cfg.chat
-      ? new AtpAgent({ service: cfg.chat.url })
-      : undefined
-
-    const didCache = new MemoryCache(
-      cfg.identity.cacheStaleTTL,
-      cfg.identity.cacheMaxTTL,
-    )
-    const idResolver = new IdResolver({
-      plcUrl: cfg.identity.plcUrl,
-      didCache,
-    })
-
-    const createAuthHeaders = (aud: string, lxm: string) =>
-      createServiceAuthHeaders({
-        iss: `${cfg.service.did}#atproto_labeler`,
-        aud,
-        lxm,
-        keypair: signingKey,
-      })
-
-    const backgroundQueue = new BackgroundQueue(db, { concurrency: 20 })
-    const blobDiverter = cfg.blobDivert
-      ? new BlobDiverter(db, {
-          idResolver,
-          serviceConfig: cfg.blobDivert,
-        })
-      : undefined
-    const eventPusher = new EventPusher(db, createAuthHeaders, {
-      appview: cfg.appview.pushEvents ? cfg.appview : undefined,
-      pds: cfg.pds ?? undefined,
-    })
-
-    const communicationTemplateService = CommunicationTemplateService.creator()
-    const safelinkRuleService = SafelinkRuleService.creator()
-    const scheduledActionService = ScheduledActionService.creator()
-    const teamService = TeamService.creator(
-      appviewAgent,
-      cfg.appview.did,
-      createAuthHeaders,
-    )
-    const queueService = QueueService.creator()
-    const reportStatsService = ReportStatsService.creator()
-    const setService = SetService.creator()
-    const settingService = SettingService.creator()
-    const strikeService = StrikeService.creator()
-    const verificationService = VerificationService.creator()
-    const verificationIssuer = VerificationIssuer.creator()
-    const moderationServiceProfile = ModerationServiceProfile.creator(
-      cfg,
-      appviewAgent,
-    )
-    const modService = ModerationService.creator(
-      signingKey,
-      signingKeyId,
-      cfg,
-      backgroundQueue,
-      idResolver,
-      eventPusher,
-      appviewAgent,
-      createAuthHeaders,
-      strikeService,
-      overrides?.imgInvalidator,
-    )
-    const assignmentService = AssignmentService.creator(
-      {
-        queueDurationMs: cfg.assignments.queueDurationMs,
-        reportDurationMs: cfg.assignments.reportDurationMs,
-      },
-      queueService,
-      teamService,
-    )(db)
-
-    const sequencer = new Sequencer(modService(db))
-
-    const authVerifier = new AuthVerifier(idResolver, {
-      serviceDid: cfg.service.did,
-      adminPassword: secrets.adminPassword,
-      teamService: teamService(db),
-    })
-
-    return new AppContext(
-      {
-        db,
-        cfg,
-        modService,
-        moderationServiceProfile,
-        communicationTemplateService,
-        safelinkRuleService,
-        scheduledActionService,
-        teamService,
-        queueService,
-        reportStatsService,
-        setService,
-        settingService,
-        strikeService,
-        appviewAgent,
-        pdsAgent,
-        chatAgent,
-        signingKey,
-        signingKeyId,
-        didCache,
-        idResolver,
-        backgroundQueue,
-        sequencer,
-        assignmentService,
-        authVerifier,
-        blobDiverter,
-        verificationService,
-        verificationIssuer,
-        ...(overrides ?? {}),
-      },
-      secrets,
-    )
-  }
-
-  assignPort(port: number) {
-    assert(
-      !this.cfg.service.port || this.cfg.service.port === port,
-      'Conflicting port in config',
-    )
-    this.opts.cfg.service.port = port
-  }
-
-  get db(): Database {
-    return this.opts.db
-  }
-
-  get cfg(): OzoneConfig {
-    return this.opts.cfg
-  }
-
-  get modService(): ModerationServiceCreator {
-    return this.opts.modService
-  }
-
-  get blobDiverter(): BlobDiverter | undefined {
-    return this.opts.blobDiverter
-  }
-
-  get communicationTemplateService(): CommunicationTemplateServiceCreator {
-    return this.opts.communicationTemplateService
-  }
-
-  get safelinkRuleService(): SafelinkRuleServiceCreator {
-    return this.opts.safelinkRuleService
-  }
-
-  get scheduledActionService(): ScheduledActionServiceCreator {
-    return this.opts.scheduledActionService
-  }
-
-  get teamService(): TeamServiceCreator {
-    return this.opts.teamService
-  }
-
-  get queueService(): QueueServiceCreator {
-    return this.opts.queueService
-  }
-
-  get reportStatsService(): ReportStatsServiceCreator {
-    return this.opts.reportStatsService
-  }
-
-  get setService(): SetServiceCreator {
-    return this.opts.setService
-  }
-
-  get settingService(): SettingServiceCreator {
-    return this.opts.settingService
-  }
-
-  get strikeService(): StrikeServiceCreator {
-    return this.opts.strikeService
-  }
-
-  get verificationService(): VerificationServiceCreator {
-    return this.opts.verificationService
-  }
-
-  get verificationIssuer(): VerificationIssuerCreator {
-    return this.opts.verificationIssuer
-  }
-
-  get moderationServiceProfile(): ModerationServiceProfileCreator {
-    return this.opts.moderationServiceProfile
-  }
-
-  get appviewAgent(): AtpAgent {
-    return this.opts.appviewAgent
-  }
-
-  get pdsAgent(): AtpAgent | undefined {
-    return this.opts.pdsAgent
-  }
-
-  get chatAgent(): AtpAgent | undefined {
-    return this.opts.chatAgent
-  }
-
-  get signingKey(): Keypair {
-    return this.opts.signingKey
-  }
-
-  get signingKeyId(): number {
-    return this.opts.signingKeyId
-  }
-
-  get plcClient(): plc.Client {
-    return new plc.Client(this.cfg.identity.plcUrl)
-  }
-
-  get didCache(): DidCache {
-    return this.opts.didCache
-  }
-
-  get idResolver(): IdResolver {
-    return this.opts.idResolver
-  }
-
-  get backgroundQueue(): BackgroundQueue {
-    return this.opts.backgroundQueue
-  }
-
-  get sequencer(): Sequencer {
-    return this.opts.sequencer
-  }
-
-  get assignmentService(): AssignmentService {
-    return this.opts.assignmentService
-  }
-
-  get authVerifier(): AuthVerifier {
-    return this.opts.authVerifier
-  }
-
-  async serviceAuthHeaders(aud: string, lxm: string) {
-    const iss = `${this.cfg.service.did}#atproto_labeler`
-    return createServiceAuthHeaders({
-      iss,
-      aud,
-      lxm,
-      keypair: this.signingKey,
-    })
-  }
-
-  async pdsAuth(lxm: string) {
-    if (!this.cfg.pds) {
-      return undefined
-    }
-    return this.serviceAuthHeaders(this.cfg.pds.did, lxm)
-  }
-
-  async appviewAuth(lxm: string) {
-    return this.serviceAuthHeaders(this.cfg.appview.did, lxm)
-  }
-
-  async chatAuth(lxm: string) {
-    if (!this.cfg.chat) {
-      throw new Error('No chat service configured')
-    }
-    return this.serviceAuthHeaders(this.cfg.chat.did, lxm)
-  }
-
-  devOverride(overrides: Partial<AppContextOptions>) {
-    this.opts = {
-      ...this.opts,
-      ...overrides,
-    }
-  }
-
-  reqLabelers(req: express.Request): ParsedLabelers {
-    const val = req.header(LABELER_HEADER_NAME)
-    let parsed: ParsedLabelers | null
-    try {
-      parsed = parseLabelerHeader(val, this.cfg.service.did)
-    } catch (err) {
-      parsed = null
-    }
-    if (!parsed) return defaultLabelerHeader([])
-    return parsed
-  }
-}

package/src/daemon/blob-diverter.ts

@@ -1,186 +0,0 @@
-import { Readable } from 'node:stream'
-import { finished, pipeline } from 'node:stream/promises'
-import { CID } from 'multiformats/cid'
-import * as undici from 'undici'
-import {
-  VerifyCidTransform,
-  allFulfilled,
-  createDecoders,
-  getPdsEndpoint,
-} from '@atproto/common'
-import { IdResolver } from '@atproto/identity'
-import { ResponseType, XRPCError } from '@atproto/xrpc'
-import { BlobDivertConfig } from '../config/index.js'
-import { Database } from '../db/index.js'
-import { retryHttp } from '../util.js'
-
-export class BlobDiverter {
-  serviceConfig: BlobDivertConfig
-  idResolver: IdResolver
-
-  constructor(
-    public db: Database,
-    services: {
-      idResolver: IdResolver
-      serviceConfig: BlobDivertConfig
-    },
-  ) {
-    this.serviceConfig = services.serviceConfig
-    this.idResolver = services.idResolver
-  }
-
-  /**
-   * @throws {XRPCError} so that retryHttp can handle retries
-   */
-  async getBlob(options: GetBlobOptions): Promise<Blob> {
-    const blobUrl = getBlobUrl(options)
-
-    const blobResponse = await undici
-      .request(blobUrl, {
-        headersTimeout: 10e3,
-        bodyTimeout: 30e3,
-      })
-      .catch((err) => {
-        throw asXrpcClientError(err, `Error fetching blob ${options.cid}`)
-      })
-
-    if (blobResponse.statusCode !== 200) {
-      await blobResponse.body.dump()
-      throw new XRPCError(
-        blobResponse.statusCode,
-        undefined,
-        `Error downloading blob ${options.cid}`,
-      )
-    }
-
-    try {
-      const type = blobResponse.headers['content-type']
-      const encoding = blobResponse.headers['content-encoding']
-
-      const verifier = new VerifyCidTransform(CID.parse(options.cid))
-
-      void pipeline([
-        blobResponse.body,
-        ...createDecoders(encoding),
-        verifier,
-      ]).catch((_err) => {})
-
-      return {
-        type: typeof type === 'string' ? type : 'application/octet-stream',
-        stream: verifier,
-      }
-    } catch (err) {
-      // Typically un-supported content encoding
-      await blobResponse.body.dump()
-      throw err
-    }
-  }
-
-  /**
-   * @throws {XRPCError} so that retryHttp can handle retries
-   */
-  async uploadBlob(blob: Blob, report: ReportBlobOptions) {
-    const uploadUrl = reportBlobUrl(this.serviceConfig.url, report)
-
-    const result = await undici
-      .request(uploadUrl, {
-        method: 'POST',
-        body: blob.stream,
-        headersTimeout: 30e3,
-        bodyTimeout: 10e3,
-        headers: {
-          Authorization: basicAuth('admin', this.serviceConfig.adminPassword),
-          'content-type': blob.type,
-        },
-      })
-      .catch((err) => {
-        throw asXrpcClientError(err, `Error uploading blob ${report.did}`)
-      })
-
-    if (result.statusCode !== 200) {
-      await result.body.dump()
-      throw new XRPCError(
-        result.statusCode,
-        undefined,
-        `Error uploading blob ${report.did}`,
-      )
-    }
-
-    await finished(result.body.resume())
-  }
-
-  async uploadBlobOnService({
-    subjectDid: did,
-    subjectUri: uri,
-    subjectBlobCids,
-  }: {
-    subjectDid: string
-    subjectUri: string | null
-    subjectBlobCids: string[]
-  }): Promise<void> {
-    const didDoc = await this.idResolver.did.resolve(did)
-    if (!didDoc) throw new Error('Error resolving DID')
-
-    const pds = getPdsEndpoint(didDoc)
-    if (!pds) throw new Error('Error resolving PDS')
-
-    await allFulfilled(
-      subjectBlobCids.map((cid) =>
-        retryHttp(async () => {
-          // attempt to download and upload within the same retry block since
-          // the blob stream is not reusable
-          const blob = await this.getBlob({ pds, cid, did })
-          return this.uploadBlob(blob, { did, uri })
-        }),
-      ),
-    ).catch((err) => {
-      throw new XRPCError(
-        ResponseType.UpstreamFailure,
-        undefined,
-        'Failed to process blobs',
-        undefined,
-        { cause: err },
-      )
-    })
-  }
-}
-
-const basicAuth = (username: string, password: string) => {
-  return 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64')
-}
-
-type Blob = {
-  type: string
-  stream: Readable
-}
-
-type GetBlobOptions = {
-  pds: string
-  did: string
-  cid: string
-}
-
-function getBlobUrl({ pds, did, cid }: GetBlobOptions): URL {
-  const url = new URL(`/xrpc/com.atproto.sync.getBlob`, pds)
-  url.searchParams.set('did', did)
-  url.searchParams.set('cid', cid)
-  return url
-}
-
-type ReportBlobOptions = {
-  did: string
-  uri: string | null
-}
-
-function reportBlobUrl(service: string, { did, uri }: ReportBlobOptions): URL {
-  const url = new URL(`/xrpc/com.atproto.unspecced.reportBlob`, service)
-  url.searchParams.set('did', did)
-  if (uri != null) url.searchParams.set('uri', uri)
-  return url
-}
-
-function asXrpcClientError(err: unknown, message: string) {
-  return new XRPCError(ResponseType.Unknown, undefined, message, undefined, {
-    cause: err,
-  })
-}