@atproto/bsky 0.0.15 → 0.0.17

package/src/api/com/atproto/moderation/util.ts

@@ -1,16 +1,8 @@
 import { CID } from 'multiformats/cid'
 import { InvalidRequestError } from '@atproto/xrpc-server'
 import { AtUri } from '@atproto/syntax'
-import { ModerationAction } from '../../../../db/tables/moderation'
-import { ModerationReport } from '../../../../db/tables/moderation'
 import { InputSchema as ReportInput } from '../../../../lexicon/types/com/atproto/moderation/createReport'
-import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/takeModerationAction'
-import {
-  ACKNOWLEDGE,
-  FLAG,
-  TAKEDOWN,
-  ESCALATE,
-} from '../../../../lexicon/types/com/atproto/admin/defs'
+import { InputSchema as ActionInput } from '../../../../lexicon/types/com/atproto/admin/emitModerationEvent'
 import {
   REASONOTHER,
   REASONSPAM,
@@ -19,6 +11,13 @@ import {
   REASONSEXUAL,
   REASONVIOLATION,
 } from '../../../../lexicon/types/com/atproto/moderation/defs'
+import {
+  REVIEWCLOSED,
+  REVIEWESCALATED,
+  REVIEWOPEN,
+} from '../../../../lexicon/types/com/atproto/admin/defs'
+import { ModerationEvent } from '../../../../db/tables/moderation'
+import { ModerationSubjectStatusRow } from '../../../../services/moderation/types'
 
 type SubjectInput = ReportInput['subject'] | ActionInput['subject']
 
@@ -34,8 +33,9 @@ export const getSubject = (subject: SubjectInput) => {
     typeof subject.uri === 'string' &&
     typeof subject.cid === 'string'
   ) {
+    const uri = new AtUri(subject.uri)
     return {
-      uri: new AtUri(subject.uri),
+      uri,
       cid: CID.parse(subject.cid),
     }
   }
@@ -44,23 +44,28 @@ export const getSubject = (subject: SubjectInput) => {
 
 export const getReasonType = (reasonType: ReportInput['reasonType']) => {
   if (reasonTypes.has(reasonType)) {
-    return reasonType as ModerationReport['reasonType']
+    return reasonType as NonNullable<ModerationEvent['meta']>['reportType']
   }
   throw new InvalidRequestError('Invalid reason type')
 }
 
-export const getAction = (action: ActionInput['action']) => {
-  if (
-    action === TAKEDOWN ||
-    action === FLAG ||
-    action === ACKNOWLEDGE ||
-    action === ESCALATE
-  ) {
-    return action as ModerationAction['action']
+export const getEventType = (type: string) => {
+  if (eventTypes.has(type)) {
+    return type as ModerationEvent['action']
   }
-  throw new InvalidRequestError('Invalid action')
+  throw new InvalidRequestError('Invalid event type')
 }
 
+export const getReviewState = (reviewState?: string) => {
+  if (!reviewState) return undefined
+  if (reviewStates.has(reviewState)) {
+    return reviewState as ModerationSubjectStatusRow['reviewState']
+  }
+  throw new InvalidRequestError('Invalid review state')
+}
+
+const reviewStates = new Set([REVIEWCLOSED, REVIEWESCALATED, REVIEWOPEN])
+
 const reasonTypes = new Set([
   REASONOTHER,
   REASONSPAM,
@@ -69,3 +74,16 @@ const reasonTypes = new Set([
   REASONSEXUAL,
   REASONVIOLATION,
 ])
+
+const eventTypes = new Set([
+  'com.atproto.admin.defs#modEventTakedown',
+  'com.atproto.admin.defs#modEventAcknowledge',
+  'com.atproto.admin.defs#modEventEscalate',
+  'com.atproto.admin.defs#modEventComment',
+  'com.atproto.admin.defs#modEventLabel',
+  'com.atproto.admin.defs#modEventReport',
+  'com.atproto.admin.defs#modEventMute',
+  'com.atproto.admin.defs#modEventUnmute',
+  'com.atproto.admin.defs#modEventReverseTakedown',
+  'com.atproto.admin.defs#modEventEmail',
+])

package/src/api/index.ts

@@ -41,18 +41,15 @@ import registerPush from './app/bsky/notification/registerPush'
 import getPopularFeedGenerators from './app/bsky/unspecced/getPopularFeedGenerators'
 import getTimelineSkeleton from './app/bsky/unspecced/getTimelineSkeleton'
 import createReport from './com/atproto/moderation/createReport'
-import resolveModerationReports from './com/atproto/admin/resolveModerationReports'
-import reverseModerationAction from './com/atproto/admin/reverseModerationAction'
-import takeModerationAction from './com/atproto/admin/takeModerationAction'
+import emitModerationEvent from './com/atproto/admin/emitModerationEvent'
 import searchRepos from './com/atproto/admin/searchRepos'
 import adminGetRecord from './com/atproto/admin/getRecord'
 import getRepo from './com/atproto/admin/getRepo'
-import getModerationAction from './com/atproto/admin/getModerationAction'
-import getModerationActions from './com/atproto/admin/getModerationActions'
-import getModerationReport from './com/atproto/admin/getModerationReport'
-import getModerationReports from './com/atproto/admin/getModerationReports'
+import queryModerationStatuses from './com/atproto/admin/queryModerationStatuses'
 import resolveHandle from './com/atproto/identity/resolveHandle'
 import getRecord from './com/atproto/repo/getRecord'
+import queryModerationEvents from './com/atproto/admin/queryModerationEvents'
+import getModerationEvent from './com/atproto/admin/getModerationEvent'
 import fetchLabels from './com/atproto/temp/fetchLabels'
 
 export * as health from './health'
@@ -105,16 +102,13 @@ export default function (server: Server, ctx: AppContext) {
   getTimelineSkeleton(server, ctx)
   // com.atproto
   createReport(server, ctx)
-  resolveModerationReports(server, ctx)
-  reverseModerationAction(server, ctx)
-  takeModerationAction(server, ctx)
+  emitModerationEvent(server, ctx)
   searchRepos(server, ctx)
   adminGetRecord(server, ctx)
   getRepo(server, ctx)
-  getModerationAction(server, ctx)
-  getModerationActions(server, ctx)
-  getModerationReport(server, ctx)
-  getModerationReports(server, ctx)
+  getModerationEvent(server, ctx)
+  queryModerationEvents(server, ctx)
+  queryModerationStatuses(server, ctx)
   resolveHandle(server, ctx)
   getRecord(server, ctx)
   fetchLabels(server, ctx)

package/src/auth.ts

@@ -7,35 +7,43 @@ import { ServerConfig } from './config'
 const BASIC = 'Basic '
 const BEARER = 'Bearer '
 
-export const authVerifier =
-  (idResolver: IdResolver, opts: { aud: string | null }) =>
-  async (reqCtx: { req: express.Request; res: express.Response }) => {
+export const authVerifier = (
+  idResolver: IdResolver,
+  opts: { aud: string | null },
+) => {
+  const getSigningKey = async (
+    did: string,
+    forceRefresh: boolean,
+  ): Promise<string> => {
+    const atprotoData = await idResolver.did.resolveAtprotoData(
+      did,
+      forceRefresh,
+    )
+    return atprotoData.signingKey
+  }
+
+  return async (reqCtx: { req: express.Request; res: express.Response }) => {
     const jwtStr = getJwtStrFromReq(reqCtx.req)
     if (!jwtStr) {
       throw new AuthRequiredError('missing jwt', 'MissingJwt')
     }
-    const payload = await verifyJwt(
-      jwtStr,
-      opts.aud,
-      async (did, forceRefresh) => {
-        const atprotoData = await idResolver.did.resolveAtprotoData(
-          did,
-          forceRefresh,
-        )
-        return atprotoData.signingKey
-      },
-    )
+    const payload = await verifyJwt(jwtStr, opts.aud, getSigningKey)
     return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } }
   }
+}
 
-export const authOptionalVerifier =
-  (idResolver: IdResolver, opts: { aud: string | null }) =>
-  async (reqCtx: { req: express.Request; res: express.Response }) => {
+export const authOptionalVerifier = (
+  idResolver: IdResolver,
+  opts: { aud: string | null },
+) => {
+  const verifyAccess = authVerifier(idResolver, opts)
+  return async (reqCtx: { req: express.Request; res: express.Response }) => {
     if (!reqCtx.req.headers.authorization) {
       return { credentials: { did: null } }
     }
-    return authVerifier(idResolver, opts)(reqCtx)
+    return verifyAccess(reqCtx)
   }
+}
 
 export const authOptionalAccessOrRoleVerifier = (
   idResolver: IdResolver,
@@ -127,9 +135,9 @@ export const buildBasicAuth = (username: string, password: string): string => {
 }
 
 export const getJwtStrFromReq = (req: express.Request): string | null => {
-  const { authorization = '' } = req.headers
-  if (!authorization.startsWith(BEARER)) {
+  const { authorization } = req.headers
+  if (!authorization?.startsWith(BEARER)) {
     return null
   }
-  return authorization.replace(BEARER, '').trim()
+  return authorization.slice(BEARER.length).trim()
 }

package/src/auto-moderator/index.ts

@@ -61,7 +61,6 @@ export class AutoModerator {
         'moderation service not properly configured',
       )
     }
-
     this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined
     this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords)
     if (abyssEndpoint && abyssPassword) {
@@ -157,18 +156,22 @@ export class AutoModerator {
     if (!this.textFlagger) return
     const matches = this.textFlagger.getMatches(text)
     if (matches.length < 1) return
-    if (!this.services.moderation) {
-      log.error(
-        { subject, text, matches },
-        'no moderation service setup to flag record text',
-      )
-      return
-    }
-    await this.services.moderation(this.ctx.db).report({
-      reasonType: REASONOTHER,
-      reason: `Automatically flagged for possible slurs: ${matches.join(', ')}`,
-      subject,
-      reportedBy: this.ctx.cfg.labelerDid,
+    await this.ctx.db.transaction(async (dbTxn) => {
+      if (!this.services.moderation) {
+        log.error(
+          { subject, text, matches },
+          'no moderation service setup to flag record text',
+        )
+        return
+      }
+      return this.services.moderation(dbTxn).report({
+        reasonType: REASONOTHER,
+        reason: `Automatically flagged for possible slurs: ${matches.join(
+          ', ',
+        )}`,
+        subject,
+        reportedBy: this.ctx.cfg.labelerDid,
+      })
     })
   }
 
@@ -244,15 +247,17 @@ export class AutoModerator {
     }
 
     if (this.pushAgent) {
-      await this.pushAgent.com.atproto.admin.takeModerationAction({
-        action: 'com.atproto.admin.defs#takedown',
+      await this.pushAgent.com.atproto.admin.emitModerationEvent({
+        event: {
+          $type: 'com.atproto.admin.defs#modEventTakedown',
+          comment: takedownReason,
+        },
         subject: {
           $type: 'com.atproto.repo.strongRef',
           uri: uri.toString(),
           cid: recordCid.toString(),
         },
         subjectBlobCids: takedownCids.map((c) => c.toString()),
-        reason: takedownReason,
         createdBy: this.ctx.cfg.labelerDid,
       })
     } else {
@@ -261,11 +266,13 @@ export class AutoModerator {
           throw new Error('no mod push agent or uri invalidator setup')
         }
         const modSrvc = this.services.moderation(dbTxn)
-        const action = await modSrvc.logAction({
-          action: 'com.atproto.admin.defs#takedown',
+        const action = await modSrvc.logEvent({
+          event: {
+            $type: 'com.atproto.admin.defs#modEventTakedown',
+            comment: takedownReason,
+          },
           subject: { uri, cid: recordCid },
           subjectBlobCids: takedownCids,
-          reason: takedownReason,
           createdBy: this.ctx.cfg.labelerDid,
         })
         await modSrvc.takedownRecord({

package/src/cache/read-through.ts

@@ -0,0 +1,151 @@
+import { cacheLogger as log } from '../logger'
+import { Redis } from '../redis'
+
+export type CacheItem<T> = {
+  val: T | null // null here is for negative caching
+  updatedAt: number
+}
+
+export type CacheOptions<T> = {
+  staleTTL: number
+  maxTTL: number
+  fetchMethod: (key: string) => Promise<T | null>
+  fetchManyMethod?: (keys: string[]) => Promise<Record<string, T | null>>
+}
+
+export class ReadThroughCache<T> {
+  constructor(public redis: Redis, public opts: CacheOptions<T>) {}
+
+  private async _fetchMany(keys: string[]): Promise<Record<string, T | null>> {
+    let result: Record<string, T | null> = {}
+    if (this.opts.fetchManyMethod) {
+      result = await this.opts.fetchManyMethod(keys)
+    } else {
+      const got = await Promise.all(keys.map((k) => this.opts.fetchMethod(k)))
+      for (let i = 0; i < keys.length; i++) {
+        result[keys[i]] = got[i] ?? null
+      }
+    }
+    // ensure caching negatives
+    for (const key of keys) {
+      result[key] ??= null
+    }
+    return result
+  }
+
+  private async fetchAndCache(key: string): Promise<T | null> {
+    const fetched = await this.opts.fetchMethod(key)
+    this.set(key, fetched).catch((err) =>
+      log.error({ err, key }, 'failed to set cache value'),
+    )
+    return fetched
+  }
+
+  private async fetchAndCacheMany(keys: string[]): Promise<Record<string, T>> {
+    const fetched = await this._fetchMany(keys)
+    this.setMany(fetched).catch((err) =>
+      log.error({ err, keys }, 'failed to set cache values'),
+    )
+    return removeNulls(fetched)
+  }
+
+  async get(key: string, opts?: { revalidate?: boolean }): Promise<T | null> {
+    if (opts?.revalidate) {
+      return this.fetchAndCache(key)
+    }
+    let cached: CacheItem<T> | null
+    try {
+      const got = await this.redis.get(key)
+      cached = got ? JSON.parse(got) : null
+    } catch (err) {
+      cached = null
+      log.warn({ key, err }, 'failed to fetch value from cache')
+    }
+    if (!cached || this.isExpired(cached)) {
+      return this.fetchAndCache(key)
+    }
+    if (this.isStale(cached)) {
+      this.fetchAndCache(key).catch((err) =>
+        log.warn({ key, err }, 'failed to refresh stale cache value'),
+      )
+    }
+    return cached.val
+  }
+
+  async getMany(
+    keys: string[],
+    opts?: { revalidate?: boolean },
+  ): Promise<Record<string, T>> {
+    if (opts?.revalidate) {
+      return this.fetchAndCacheMany(keys)
+    }
+    let cached: Record<string, string>
+    try {
+      cached = await this.redis.getMulti(keys)
+    } catch (err) {
+      cached = {}
+      log.warn({ keys, err }, 'failed to fetch values from cache')
+    }
+
+    const stale: string[] = []
+    const toFetch: string[] = []
+    const results: Record<string, T> = {}
+    for (const key of keys) {
+      const val = cached[key] ? (JSON.parse(cached[key]) as CacheItem<T>) : null
+      if (!val || this.isExpired(val)) {
+        toFetch.push(key)
+        continue
+      }
+      if (this.isStale(val)) {
+        stale.push(key)
+      }
+      if (val.val) {
+        results[key] = val.val
+      }
+    }
+    const fetched = await this.fetchAndCacheMany(toFetch)
+    this.fetchAndCacheMany(stale).catch((err) =>
+      log.warn({ keys, err }, 'failed to refresh stale cache values'),
+    )
+    return {
+      ...results,
+      ...fetched,
+    }
+  }
+
+  async set(key: string, val: T | null) {
+    await this.setMany({ [key]: val })
+  }
+
+  async setMany(vals: Record<string, T | null>) {
+    const items: Record<string, string> = {}
+    for (const key of Object.keys(vals)) {
+      items[key] = JSON.stringify({
+        val: vals[key],
+        updatedAt: Date.now(),
+      })
+    }
+    await this.redis.setMulti(items, this.opts.maxTTL)
+  }
+
+  async clearEntry(key: string) {
+    await this.redis.del(key)
+  }
+
+  isExpired(result: CacheItem<T>) {
+    return Date.now() > result.updatedAt + this.opts.maxTTL
+  }
+
+  isStale(result: CacheItem<T>) {
+    return Date.now() > result.updatedAt + this.opts.staleTTL
+  }
+}
+
+const removeNulls = <T>(obj: Record<string, T | null>): Record<string, T> => {
+  return Object.entries(obj).reduce((acc, [key, val]) => {
+    if (val !== null) {
+      acc[key] = val
+    }
+    return acc
+  }, {} as Record<string, T>)
+}

package/src/config.ts

@@ -1,5 +1,11 @@
 import assert from 'assert'
-import { DAY, HOUR, parseIntWithFallback } from '@atproto/common'
+import {
+  DAY,
+  HOUR,
+  MINUTE,
+  SECOND,
+  parseIntWithFallback,
+} from '@atproto/common'
 
 export interface ServerConfigValues {
   version: string
@@ -12,9 +18,15 @@ export interface ServerConfigValues {
   dbReplicaPostgresUrls?: string[]
   dbReplicaTags?: Record<string, number[]> // E.g. { timeline: [0], thread: [1] }
   dbPostgresSchema?: string
+  redisHost?: string // either set redis host, or both sentinel name and hosts
+  redisSentinelName?: string
+  redisSentinelHosts?: string[]
+  redisPassword?: string
   didPlcUrl: string
   didCacheStaleTTL: number
   didCacheMaxTTL: number
+  labelCacheStaleTTL: number
+  labelCacheMaxTTL: number
   handleResolveNameservers?: string[]
   imgUriEndpoint?: string
   blobCacheLocation?: string
@@ -24,6 +36,9 @@ export interface ServerConfigValues {
   moderatorPassword?: string
   triagePassword?: string
   moderationPushUrl?: string
+  rateLimitsEnabled: boolean
+  rateLimitBypassKey?: string
+  rateLimitBypassIps?: string[]
 }
 
 export class ServerConfig {
@@ -38,6 +53,19 @@ export class ServerConfig {
     const feedGenDid = process.env.FEED_GEN_DID
     const envPort = parseInt(process.env.PORT || '', 10)
     const port = isNaN(envPort) ? 2584 : envPort
+    const redisHost =
+      overrides?.redisHost || process.env.REDIS_HOST || undefined
+    const redisSentinelName =
+      overrides?.redisSentinelName ||
+      process.env.REDIS_SENTINEL_NAME ||
+      undefined
+    const redisSentinelHosts =
+      overrides?.redisSentinelHosts ||
+      (process.env.REDIS_SENTINEL_HOSTS
+        ? process.env.REDIS_SENTINEL_HOSTS.split(',')
+        : [])
+    const redisPassword =
+      overrides?.redisPassword || process.env.REDIS_PASSWORD || undefined
     const didPlcUrl = process.env.DID_PLC_URL || 'http://localhost:2582'
     const didCacheStaleTTL = parseIntWithFallback(
       process.env.DID_CACHE_STALE_TTL,
@@ -47,6 +75,14 @@ export class ServerConfig {
       process.env.DID_CACHE_MAX_TTL,
       DAY,
     )
+    const labelCacheStaleTTL = parseIntWithFallback(
+      process.env.LABEL_CACHE_STALE_TTL,
+      30 * SECOND,
+    )
+    const labelCacheMaxTTL = parseIntWithFallback(
+      process.env.LABEL_CACHE_MAX_TTL,
+      MINUTE,
+    )
     const handleResolveNameservers = process.env.HANDLE_RESOLVE_NAMESERVERS
       ? process.env.HANDLE_RESOLVE_NAMESERVERS.split(',')
       : []
@@ -82,6 +118,14 @@ export class ServerConfig {
       overrides?.moderationPushUrl ||
       process.env.MODERATION_PUSH_URL ||
       undefined
+    const rateLimitsEnabled = process.env.RATE_LIMITS_ENABLED === 'true'
+    const rateLimitBypassKey = process.env.RATE_LIMIT_BYPASS_KEY
+    const rateLimitBypassIps = process.env.RATE_LIMIT_BYPASS_IPS
+      ? process.env.RATE_LIMIT_BYPASS_IPS.split(',').map((ipOrCidr) =>
+          ipOrCidr.split('/')[0]?.trim(),
+        )
+      : undefined
+
     return new ServerConfig({
       version,
       debugMode,
@@ -93,9 +137,15 @@ export class ServerConfig {
       dbReplicaPostgresUrls,
       dbReplicaTags,
       dbPostgresSchema,
+      redisHost,
+      redisSentinelName,
+      redisSentinelHosts,
+      redisPassword,
       didPlcUrl,
       didCacheStaleTTL,
       didCacheMaxTTL,
+      labelCacheStaleTTL,
+      labelCacheMaxTTL,
       handleResolveNameservers,
       imgUriEndpoint,
       blobCacheLocation,
@@ -105,6 +155,9 @@ export class ServerConfig {
       moderatorPassword,
       triagePassword,
       moderationPushUrl,
+      rateLimitsEnabled,
+      rateLimitBypassKey,
+      rateLimitBypassIps,
       ...stripUndefineds(overrides ?? {}),
     })
   }
@@ -162,6 +215,22 @@ export class ServerConfig {
     return this.cfg.dbPostgresSchema
   }
 
+  get redisHost() {
+    return this.cfg.redisHost
+  }
+
+  get redisSentinelName() {
+    return this.cfg.redisSentinelName
+  }
+
+  get redisSentinelHosts() {
+    return this.cfg.redisSentinelHosts
+  }
+
+  get redisPassword() {
+    return this.cfg.redisPassword
+  }
+
   get didCacheStaleTTL() {
     return this.cfg.didCacheStaleTTL
   }
@@ -170,6 +239,14 @@ export class ServerConfig {
     return this.cfg.didCacheMaxTTL
   }
 
+  get labelCacheStaleTTL() {
+    return this.cfg.labelCacheStaleTTL
+  }
+
+  get labelCacheMaxTTL() {
+    return this.cfg.labelCacheMaxTTL
+  }
+
   get handleResolveNameservers() {
     return this.cfg.handleResolveNameservers
   }
@@ -209,6 +286,18 @@ export class ServerConfig {
   get moderationPushUrl() {
     return this.cfg.moderationPushUrl
   }
+
+  get rateLimitsEnabled() {
+    return this.cfg.rateLimitsEnabled
+  }
+
+  get rateLimitBypassKey() {
+    return this.cfg.rateLimitBypassKey
+  }
+
+  get rateLimitBypassIps() {
+    return this.cfg.rateLimitBypassIps
+  }
 }
 
 function getTagIdxs(str?: string): number[] {

package/src/context.ts

@@ -8,11 +8,11 @@ import { ServerConfig } from './config'
 import { ImageUriBuilder } from './image/uri'
 import { Services } from './services'
 import * as auth from './auth'
-import DidSqlCache from './did-cache'
+import DidRedisCache from './did-cache'
 import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
-import { LabelCache } from './label-cache'
 import { NotificationServer } from './notifications'
+import { Redis } from './redis'
 
 export class AppContext {
   public moderationPushAgent: AtpAgent | undefined
@@ -24,8 +24,8 @@ export class AppContext {
       services: Services
       signingKey: Keypair
       idResolver: IdResolver
-      didCache: DidSqlCache
-      labelCache: LabelCache
+      didCache: DidRedisCache
+      redis: Redis
       backgroundQueue: BackgroundQueue
       searchAgent?: AtpAgent
       algos: MountedAlgos
@@ -70,12 +70,12 @@ export class AppContext {
     return this.opts.idResolver
   }
 
-  get didCache(): DidSqlCache {
+  get didCache(): DidRedisCache {
     return this.opts.didCache
   }
 
-  get labelCache(): LabelCache {
-    return this.opts.labelCache
+  get redis(): Redis {
+    return this.opts.redis
   }
 
   get notifServer(): NotificationServer {
@@ -94,6 +94,10 @@ export class AppContext {
     return auth.authVerifier(this.idResolver, { aud: null })
   }
 
+  get authOptionalVerifierAnyAudience() {
+    return auth.authOptionalVerifier(this.idResolver, { aud: null })
+  }
+
   get authOptionalVerifier() {
     return auth.authOptionalVerifier(this.idResolver, {
       aud: this.cfg.serverDid,