@atproto/bsky 0.0.15 → 0.0.17

package/src/db/tables/moderation.ts

@@ -1,76 +1,59 @@
 import { Generated } from 'kysely'
 import {
-  ACKNOWLEDGE,
-  FLAG,
-  TAKEDOWN,
-  ESCALATE,
+  REVIEWCLOSED,
+  REVIEWOPEN,
+  REVIEWESCALATED,
 } from '../../lexicon/types/com/atproto/admin/defs'
-import {
-  REASONOTHER,
-  REASONSPAM,
-  REASONMISLEADING,
-  REASONRUDE,
-  REASONSEXUAL,
-  REASONVIOLATION,
-} from '../../lexicon/types/com/atproto/moderation/defs'
 
-export const actionTableName = 'moderation_action'
-export const actionSubjectBlobTableName = 'moderation_action_subject_blob'
-export const reportTableName = 'moderation_report'
-export const reportResolutionTableName = 'moderation_report_resolution'
+export const eventTableName = 'moderation_event'
+export const subjectStatusTableName = 'moderation_subject_status'
 
-export interface ModerationAction {
+export interface ModerationEvent {
   id: Generated<number>
-  action: typeof TAKEDOWN | typeof FLAG | typeof ACKNOWLEDGE | typeof ESCALATE
+  action:
+    | 'com.atproto.admin.defs#modEventTakedown'
+    | 'com.atproto.admin.defs#modEventAcknowledge'
+    | 'com.atproto.admin.defs#modEventEscalate'
+    | 'com.atproto.admin.defs#modEventComment'
+    | 'com.atproto.admin.defs#modEventLabel'
+    | 'com.atproto.admin.defs#modEventReport'
+    | 'com.atproto.admin.defs#modEventMute'
+    | 'com.atproto.admin.defs#modEventReverseTakedown'
+    | 'com.atproto.admin.defs#modEventEmail'
   subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
   subjectDid: string
   subjectUri: string | null
   subjectCid: string | null
   createLabelVals: string | null
   negateLabelVals: string | null
-  reason: string
+  comment: string | null
   createdAt: string
   createdBy: string
-  reversedAt: string | null
-  reversedBy: string | null
-  reversedReason: string | null
   durationInHours: number | null
   expiresAt: string | null
+  meta: Record<string, string | boolean> | null
+  legacyRefId: number | null
 }
 
-export interface ModerationActionSubjectBlob {
-  actionId: number
-  cid: string
-}
-
-export interface ModerationReport {
+export interface ModerationSubjectStatus {
   id: Generated<number>
-  subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
-  subjectDid: string
-  subjectUri: string | null
-  subjectCid: string | null
-  reasonType:
-    | typeof REASONSPAM
-    | typeof REASONOTHER
-    | typeof REASONMISLEADING
-    | typeof REASONRUDE
-    | typeof REASONSEXUAL
-    | typeof REASONVIOLATION
-  reason: string | null
-  reportedByDid: string
+  did: string
+  recordPath: string
+  recordCid: string | null
+  blobCids: string[] | null
+  reviewState: typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED
   createdAt: string
-}
-
-export interface ModerationReportResolution {
-  reportId: number
-  actionId: number
-  createdAt: string
-  createdBy: string
+  updatedAt: string
+  lastReviewedBy: string | null
+  lastReviewedAt: string | null
+  lastReportedAt: string | null
+  muteUntil: string | null
+  suspendUntil: string | null
+  takendown: boolean
+  comment: string | null
 }
 
 export type PartialDB = {
-  [actionTableName]: ModerationAction
-  [actionSubjectBlobTableName]: ModerationActionSubjectBlob
-  [reportTableName]: ModerationReport
-  [reportResolutionTableName]: ModerationReportResolution
+  [eventTableName]: ModerationEvent
+  [subjectStatusTableName]: ModerationSubjectStatus
 }

package/src/did-cache.ts

@@ -1,81 +1,61 @@
 import PQueue from 'p-queue'
 import { CacheResult, DidCache, DidDocument } from '@atproto/identity'
-import { PrimaryDatabase } from './db'
-import { excluded } from './db/util'
-import { dbLogger } from './logger'
+import { cacheLogger as log } from './logger'
+import { Redis } from './redis'
 
-export class DidSqlCache implements DidCache {
-  public pQueue: PQueue | null //null during teardown
+type CacheOptions = {
+  staleTTL: number
+  maxTTL: number
+}
+
+export class DidRedisCache implements DidCache {
+  public pQueue: PQueue | null // null during teardown
 
-  constructor(
-    // @TODO perhaps could use both primary and non-primary. not high enough
-    // throughput to matter right now. also may just move this over to redis before long!
-    public db: PrimaryDatabase,
-    public staleTTL: number,
-    public maxTTL: number,
-  ) {
+  constructor(public redis: Redis, public opts: CacheOptions) {
     this.pQueue = new PQueue()
   }
 
-  async cacheDid(
-    did: string,
-    doc: DidDocument,
-    prevResult?: CacheResult,
-  ): Promise<void> {
-    if (prevResult) {
-      await this.db.db
-        .updateTable('did_cache')
-        .set({ doc, updatedAt: Date.now() })
-        .where('did', '=', did)
-        .where('updatedAt', '=', prevResult.updatedAt)
-        .execute()
-    } else {
-      await this.db.db
-        .insertInto('did_cache')
-        .values({ did, doc, updatedAt: Date.now() })
-        .onConflict((oc) =>
-          oc.column('did').doUpdateSet({
-            doc: excluded(this.db.db, 'doc'),
-            updatedAt: excluded(this.db.db, 'updatedAt'),
-          }),
-        )
-        .executeTakeFirst()
-    }
+  async cacheDid(did: string, doc: DidDocument): Promise<void> {
+    const item = JSON.stringify({
+      doc,
+      updatedAt: Date.now(),
+    })
+    await this.redis.set(did, item, this.opts.maxTTL)
   }
 
   async refreshCache(
     did: string,
     getDoc: () => Promise<DidDocument | null>,
-    prevResult?: CacheResult,
   ): Promise<void> {
     this.pQueue?.add(async () => {
       try {
         const doc = await getDoc()
         if (doc) {
-          await this.cacheDid(did, doc, prevResult)
+          await this.cacheDid(did, doc)
         } else {
           await this.clearEntry(did)
         }
       } catch (err) {
-        dbLogger.error({ did, err }, 'refreshing did cache failed')
+        log.error({ did, err }, 'refreshing did cache failed')
       }
     })
   }
 
   async checkCache(did: string): Promise<CacheResult | null> {
-    const res = await this.db.db
-      .selectFrom('did_cache')
-      .where('did', '=', did)
-      .selectAll()
-      .executeTakeFirst()
-    if (!res) return null
-
+    let got: string | null
+    try {
+      got = await this.redis.get(did)
+    } catch (err) {
+      got = null
+      log.error({ did, err }, 'error fetching did from cache')
+    }
+    if (!got) return null
+    const { doc, updatedAt } = JSON.parse(got) as CacheResult
     const now = Date.now()
-    const updatedAt = new Date(res.updatedAt).getTime()
-    const expired = now > updatedAt + this.maxTTL
-    const stale = now > updatedAt + this.staleTTL
+    const expired = now > updatedAt + this.opts.maxTTL
+    const stale = now > updatedAt + this.opts.staleTTL
     return {
-      doc: res.doc,
+      doc,
       updatedAt,
       did,
       stale,
@@ -84,14 +64,11 @@ export class DidSqlCache implements DidCache {
   }
 
   async clearEntry(did: string): Promise<void> {
-    await this.db.db
-      .deleteFrom('did_cache')
-      .where('did', '=', did)
-      .executeTakeFirst()
+    await this.redis.del(did)
   }
 
   async clear(): Promise<void> {
-    await this.db.db.deleteFrom('did_cache').execute()
+    throw new Error('Not implemented for redis cache')
   }
 
   async processAll() {
@@ -107,4 +84,4 @@ export class DidSqlCache implements DidCache {
   }
 }
 
-export default DidSqlCache
+export default DidRedisCache

package/src/feed-gen/bsky-team.ts

@@ -14,7 +14,7 @@ const BSKY_TEAM: NotEmptyArray<string> = [
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  _viewer: string,
+  _viewer: string | null,
 ): Promise<AlgoResponse> => {
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')

package/src/feed-gen/hot-classic.ts

@@ -11,7 +11,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = ['!no-promote']
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  _viewer: string,
+  _viewer: string | null,
 ): Promise<AlgoResponse> => {
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')

package/src/feed-gen/index.ts

@@ -1,9 +1,7 @@
 import { AtUri } from '@atproto/syntax'
 import { ids } from '../lexicon/lexicons'
-import withFriends from './with-friends'
 import bskyTeam from './bsky-team'
 import hotClassic from './hot-classic'
-import bestOfFollows from './best-of-follows'
 import mutuals from './mutuals'
 import { MountedAlgos } from './types'
 
@@ -13,9 +11,7 @@ const feedgenUri = (did, name) =>
 // These are custom algorithms that will be mounted directly onto an AppView
 // Feel free to remove, update to your own, or serve the following logic at a record that you control
 export const makeAlgos = (did: string): MountedAlgos => ({
-  [feedgenUri(did, 'with-friends')]: withFriends,
   [feedgenUri(did, 'bsky-team')]: bskyTeam,
   [feedgenUri(did, 'hot-classic')]: hotClassic,
-  [feedgenUri(did, 'best-of-follows')]: bestOfFollows,
   [feedgenUri(did, 'mutuals')]: mutuals,
 })

package/src/feed-gen/mutuals.ts

@@ -3,16 +3,20 @@ import AppContext from '../context'
 import { paginate } from '../db/pagination'
 import { AlgoHandler, AlgoResponse } from './types'
 import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed'
+import { AuthRequiredError } from '@atproto/xrpc-server'
 
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  viewer: string,
+  viewer: string | null,
 ): Promise<AlgoResponse> => {
+  if (!viewer) {
+    throw new AuthRequiredError('This feed requires being logged-in')
+  }
+
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')
   const feedService = ctx.services.feed(db)
-
   const { ref } = db.db.dynamic
 
   const mutualsSubquery = db.db

package/src/feed-gen/types.ts

@@ -11,7 +11,7 @@ export type AlgoResponse = {
 export type AlgoHandler = (
   ctx: AppContext,
   params: SkeletonParams,
-  requester: string,
+  viewer: string | null,
 ) => Promise<AlgoResponse>
 
 export type MountedAlgos = Record<string, AlgoHandler>

package/src/index.ts

@@ -6,6 +6,12 @@ import { createHttpTerminator, HttpTerminator } from 'http-terminator'
 import cors from 'cors'
 import compression from 'compression'
 import { IdResolver } from '@atproto/identity'
+import {
+  RateLimiter,
+  RateLimiterOpts,
+  Options as XrpcServerOptions,
+} from '@atproto/xrpc-server'
+import { MINUTE } from '@atproto/common'
 import API, { health, wellKnown, blobResolver } from './api'
 import { DatabaseCoordinator } from './db'
 import * as error from './error'
@@ -16,29 +22,31 @@ import { ImageUriBuilder } from './image/uri'
 import { BlobDiskCache, ImageProcessingServer } from './image/server'
 import { createServices } from './services'
 import AppContext from './context'
-import DidSqlCache from './did-cache'
+import DidRedisCache from './did-cache'
 import {
   ImageInvalidator,
   ImageProcessingServerInvalidator,
 } from './image/invalidator'
 import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
-import { LabelCache } from './label-cache'
 import { NotificationServer } from './notifications'
 import { AtpAgent } from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
+import { Redis } from './redis'
 
 export type { ServerConfigValues } from './config'
 export type { MountedAlgos } from './feed-gen/types'
 export { ServerConfig } from './config'
 export { Database, PrimaryDatabase, DatabaseCoordinator } from './db'
-export { PeriodicModerationActionReversal } from './db/periodic-moderation-action-reversal'
+export { PeriodicModerationEventReversal } from './db/periodic-moderation-event-reversal'
 export { Redis } from './redis'
 export { ViewMaintainer } from './db/views'
 export { AppContext } from './context'
 export { makeAlgos } from './feed-gen'
+export * from './daemon'
 export * from './indexer'
 export * from './ingester'
+export { MigrateModerationData } from './migrate-moderation-data'
 
 export class BskyAppView {
   public ctx: AppContext
@@ -54,23 +62,25 @@ export class BskyAppView {
 
   static create(opts: {
     db: DatabaseCoordinator
+    redis: Redis
     config: ServerConfig
     signingKey: Keypair
     imgInvalidator?: ImageInvalidator
     algos?: MountedAlgos
   }): BskyAppView {
-    const { db, config, signingKey, algos = {} } = opts
+    const { db, redis, config, signingKey, algos = {} } = opts
     let maybeImgInvalidator = opts.imgInvalidator
     const app = express()
+    app.set('trust proxy', true)
     app.use(cors())
     app.use(loggerMiddleware)
     app.use(compression())
 
-    const didCache = new DidSqlCache(
-      db.getPrimary(),
-      config.didCacheStaleTTL,
-      config.didCacheMaxTTL,
-    )
+    const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
+      staleTTL: config.didCacheStaleTTL,
+      maxTTL: config.didCacheMaxTTL,
+    })
+
     const idResolver = new IdResolver({
       plcUrl: config.didPlcUrl,
       didCache,
@@ -101,7 +111,7 @@ export class BskyAppView {
     }
 
     const backgroundQueue = new BackgroundQueue(db.getPrimary())
-    const labelCache = new LabelCache(db.getPrimary())
+
     const notifServer = new NotificationServer(db.getPrimary())
     const searchAgent = config.searchEndpoint
       ? new AtpAgent({ service: config.searchEndpoint })
@@ -110,7 +120,11 @@ export class BskyAppView {
     const services = createServices({
       imgUriBuilder,
       imgInvalidator,
-      labelCache,
+      labelCacheOpts: {
+        redis: redis.withNamespace('label'),
+        staleTTL: config.labelCacheStaleTTL,
+        maxTTL: config.labelCacheMaxTTL,
+      },
     })
 
     const ctx = new AppContext({
@@ -121,21 +135,48 @@ export class BskyAppView {
       signingKey,
       idResolver,
       didCache,
-      labelCache,
+      redis,
       backgroundQueue,
       searchAgent,
       algos,
       notifServer,
     })
 
-    let server = createServer({
+    const xrpcOpts: XrpcServerOptions = {
       validateResponse: config.debugMode,
       payload: {
         jsonLimit: 100 * 1024, // 100kb
         textLimit: 100 * 1024, // 100kb
         blobLimit: 5 * 1024 * 1024, // 5mb
       },
-    })
+    }
+    if (config.rateLimitsEnabled) {
+      const rlCreator = (opts: RateLimiterOpts) =>
+        RateLimiter.redis(redis.driver, {
+          bypassSecret: config.rateLimitBypassKey,
+          bypassIps: config.rateLimitBypassIps,
+          ...opts,
+        })
+      xrpcOpts['rateLimits'] = {
+        creator: rlCreator,
+        global: [
+          {
+            name: 'global-unauthed-ip',
+            durationMs: 5 * MINUTE,
+            points: 3000,
+            calcKey: (ctx) => (ctx.auth ? null : ctx.req.ip),
+          },
+          {
+            name: 'global-authed-did',
+            durationMs: 5 * MINUTE,
+            points: 3000,
+            calcKey: (ctx) => ctx.auth?.credentials?.did ?? null,
+          },
+        ],
+      }
+    }
+
+    let server = createServer(xrpcOpts)
 
     server = API(server, ctx)
 
@@ -184,7 +225,6 @@ export class BskyAppView {
         'background queue stats',
       )
     }, 10000)
-    this.ctx.labelCache.start()
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
@@ -195,11 +235,11 @@ export class BskyAppView {
     return server
   }
 
-  async destroy(opts?: { skipDb: boolean }): Promise<void> {
-    this.ctx.labelCache.stop()
+  async destroy(opts?: { skipDb: boolean; skipRedis: boolean }): Promise<void> {
     await this.ctx.didCache.destroy()
     await this.terminator?.terminate()
     await this.ctx.backgroundQueue.destroy()
+    if (!opts?.skipRedis) await this.ctx.redis.destroy()
     if (!opts?.skipDb) await this.ctx.db.close()
     clearInterval(this.dbStatsInterval)
   }

package/src/indexer/context.ts

@@ -12,6 +12,7 @@ export class IndexerContext {
     private opts: {
       db: PrimaryDatabase
       redis: Redis
+      redisCache: Redis
       cfg: IndexerConfig
       services: Services
       idResolver: IdResolver
@@ -29,6 +30,10 @@ export class IndexerContext {
     return this.opts.redis
   }
 
+  get redisCache(): Redis {
+    return this.opts.redisCache
+  }
+
   get cfg(): IndexerConfig {
     return this.opts.cfg
   }

package/src/indexer/index.ts

@@ -2,7 +2,7 @@ import express from 'express'
 import { IdResolver } from '@atproto/identity'
 import { BackgroundQueue } from '../background'
 import { PrimaryDatabase } from '../db'
-import DidSqlCache from '../did-cache'
+import DidRedisCache from '../did-cache'
 import log from './logger'
 import { dbLogger } from '../logger'
 import { IndexerConfig } from './config'
@@ -40,15 +40,15 @@ export class BskyIndexer {
   static create(opts: {
     db: PrimaryDatabase
     redis: Redis
+    redisCache: Redis
     cfg: IndexerConfig
     imgInvalidator?: ImageInvalidator
   }): BskyIndexer {
-    const { db, redis, cfg } = opts
-    const didCache = new DidSqlCache(
-      db,
-      cfg.didCacheStaleTTL,
-      cfg.didCacheMaxTTL,
-    )
+    const { db, redis, redisCache, cfg } = opts
+    const didCache = new DidRedisCache(redisCache.withNamespace('did-doc'), {
+      staleTTL: cfg.didCacheStaleTTL,
+      maxTTL: cfg.didCacheMaxTTL,
+    })
     const idResolver = new IdResolver({
       plcUrl: cfg.didPlcUrl,
       didCache,
@@ -81,6 +81,7 @@ export class BskyIndexer {
     const ctx = new IndexerContext({
       db,
       redis,
+      redisCache,
       cfg,
       services,
       idResolver,
@@ -139,7 +140,9 @@ export class BskyIndexer {
     if (this.closeServer) await this.closeServer()
     await this.sub.destroy()
     clearInterval(this.subStatsInterval)
+    await this.ctx.didCache.destroy()
     if (!opts?.skipRedis) await this.ctx.redis.destroy()
+    if (!opts?.skipRedis) await this.ctx.redisCache.destroy()
     if (!opts?.skipDb) await this.ctx.db.close()
     clearInterval(this.dbStatsInterval)
   }