@atproto/bsky 0.0.16 → 0.0.17

package/src/daemon/notifications.ts

@@ -0,0 +1,54 @@
+import { tidyNotifications } from '../services/util/notification'
+import DaemonContext from './context'
+import logger from './logger'
+
+export class NotificationsDaemon {
+  ac = new AbortController()
+  running: Promise<void> | undefined
+  count = 0
+  lastDid: string | null = null
+
+  constructor(private ctx: DaemonContext) {}
+
+  run(opts?: RunOptions) {
+    if (this.running) return
+    this.count = 0
+    this.lastDid = null
+    this.ac = new AbortController()
+    this.running = this.tidyNotifications({
+      ...opts,
+      forever: opts?.forever !== false, // run forever by default
+    })
+      .catch((err) => {
+        // allow this to cause an unhandled rejection, let deployment handle the crash.
+        logger.error({ err }, 'notifications daemon crashed')
+        throw err
+      })
+      .finally(() => (this.running = undefined))
+  }
+
+  private async tidyNotifications(opts: RunOptions) {
+    const actorService = this.ctx.services.actor(this.ctx.db)
+    for await (const { did } of actorService.all(opts)) {
+      if (this.ac.signal.aborted) return
+      try {
+        await tidyNotifications(this.ctx.db, did)
+        this.count++
+        this.lastDid = did
+      } catch (err) {
+        logger.warn({ err, did }, 'failed to tidy notifications for actor')
+      }
+    }
+  }
+
+  async destroy() {
+    this.ac.abort()
+    await this.running
+  }
+}
+
+type RunOptions = {
+  forever?: boolean
+  batchSize?: number
+  startFromDid?: string
+}

package/src/daemon/services.ts

@@ -0,0 +1,22 @@
+import { PrimaryDatabase } from '../db'
+import { ActorService } from '../services/actor'
+import { ImageUriBuilder } from '../image/uri'
+import { GraphService } from '../services/graph'
+import { LabelService } from '../services/label'
+
+export function createServices(resources: {
+  imgUriBuilder: ImageUriBuilder
+}): Services {
+  const { imgUriBuilder } = resources
+  const graph = GraphService.creator(imgUriBuilder)
+  const label = LabelService.creator(null)
+  return {
+    actor: ActorService.creator(imgUriBuilder, graph, label),
+  }
+}
+
+export type Services = {
+  actor: FromDbPrimary<ActorService>
+}
+
+type FromDbPrimary<T> = (db: PrimaryDatabase) => T

package/src/db/database-schema.ts

@@ -24,7 +24,6 @@ import * as actorSync from './tables/actor-sync'
 import * as record from './tables/record'
 import * as notification from './tables/notification'
 import * as notificationPushToken from './tables/notification-push-token'
-import * as didCache from './tables/did-cache'
 import * as moderation from './tables/moderation'
 import * as label from './tables/label'
 import * as algo from './tables/algo'
@@ -57,7 +56,6 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   record.PartialDB &
   notification.PartialDB &
   notificationPushToken.PartialDB &
-  didCache.PartialDB &
   moderation.PartialDB &
   label.PartialDB &
   algo.PartialDB &

package/src/db/migrations/20231205T000257238Z-remove-did-cache.ts

@@ -0,0 +1,14 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('did_cache').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('did_cache')
+    .addColumn('did', 'varchar', (col) => col.primaryKey())
+    .addColumn('doc', 'jsonb', (col) => col.notNull())
+    .addColumn('updatedAt', 'bigint', (col) => col.notNull())
+    .execute()
+}

package/src/db/migrations/index.ts

@@ -31,3 +31,4 @@ export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
 export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
 export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'
 export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
+export * as _20231205T000257238Z from './20231205T000257238Z-remove-did-cache'

package/src/did-cache.ts

@@ -1,81 +1,61 @@
 import PQueue from 'p-queue'
 import { CacheResult, DidCache, DidDocument } from '@atproto/identity'
-import { PrimaryDatabase } from './db'
-import { excluded } from './db/util'
-import { dbLogger } from './logger'
+import { cacheLogger as log } from './logger'
+import { Redis } from './redis'
 
-export class DidSqlCache implements DidCache {
-  public pQueue: PQueue | null //null during teardown
+type CacheOptions = {
+  staleTTL: number
+  maxTTL: number
+}
+
+export class DidRedisCache implements DidCache {
+  public pQueue: PQueue | null // null during teardown
 
-  constructor(
-    // @TODO perhaps could use both primary and non-primary. not high enough
-    // throughput to matter right now. also may just move this over to redis before long!
-    public db: PrimaryDatabase,
-    public staleTTL: number,
-    public maxTTL: number,
-  ) {
+  constructor(public redis: Redis, public opts: CacheOptions) {
     this.pQueue = new PQueue()
   }
 
-  async cacheDid(
-    did: string,
-    doc: DidDocument,
-    prevResult?: CacheResult,
-  ): Promise<void> {
-    if (prevResult) {
-      await this.db.db
-        .updateTable('did_cache')
-        .set({ doc, updatedAt: Date.now() })
-        .where('did', '=', did)
-        .where('updatedAt', '=', prevResult.updatedAt)
-        .execute()
-    } else {
-      await this.db.db
-        .insertInto('did_cache')
-        .values({ did, doc, updatedAt: Date.now() })
-        .onConflict((oc) =>
-          oc.column('did').doUpdateSet({
-            doc: excluded(this.db.db, 'doc'),
-            updatedAt: excluded(this.db.db, 'updatedAt'),
-          }),
-        )
-        .executeTakeFirst()
-    }
+  async cacheDid(did: string, doc: DidDocument): Promise<void> {
+    const item = JSON.stringify({
+      doc,
+      updatedAt: Date.now(),
+    })
+    await this.redis.set(did, item, this.opts.maxTTL)
   }
 
   async refreshCache(
     did: string,
     getDoc: () => Promise<DidDocument | null>,
-    prevResult?: CacheResult,
   ): Promise<void> {
     this.pQueue?.add(async () => {
       try {
         const doc = await getDoc()
         if (doc) {
-          await this.cacheDid(did, doc, prevResult)
+          await this.cacheDid(did, doc)
         } else {
           await this.clearEntry(did)
         }
       } catch (err) {
-        dbLogger.error({ did, err }, 'refreshing did cache failed')
+        log.error({ did, err }, 'refreshing did cache failed')
       }
     })
   }
 
   async checkCache(did: string): Promise<CacheResult | null> {
-    const res = await this.db.db
-      .selectFrom('did_cache')
-      .where('did', '=', did)
-      .selectAll()
-      .executeTakeFirst()
-    if (!res) return null
-
+    let got: string | null
+    try {
+      got = await this.redis.get(did)
+    } catch (err) {
+      got = null
+      log.error({ did, err }, 'error fetching did from cache')
+    }
+    if (!got) return null
+    const { doc, updatedAt } = JSON.parse(got) as CacheResult
     const now = Date.now()
-    const updatedAt = new Date(res.updatedAt).getTime()
-    const expired = now > updatedAt + this.maxTTL
-    const stale = now > updatedAt + this.staleTTL
+    const expired = now > updatedAt + this.opts.maxTTL
+    const stale = now > updatedAt + this.opts.staleTTL
     return {
-      doc: res.doc,
+      doc,
       updatedAt,
       did,
       stale,
@@ -84,14 +64,11 @@ export class DidSqlCache implements DidCache {
   }
 
   async clearEntry(did: string): Promise<void> {
-    await this.db.db
-      .deleteFrom('did_cache')
-      .where('did', '=', did)
-      .executeTakeFirst()
+    await this.redis.del(did)
   }
 
   async clear(): Promise<void> {
-    await this.db.db.deleteFrom('did_cache').execute()
+    throw new Error('Not implemented for redis cache')
   }
 
   async processAll() {
@@ -107,4 +84,4 @@ export class DidSqlCache implements DidCache {
   }
 }
 
-export default DidSqlCache
+export default DidRedisCache

package/src/feed-gen/index.ts

@@ -1,9 +1,7 @@
 import { AtUri } from '@atproto/syntax'
 import { ids } from '../lexicon/lexicons'
-import withFriends from './with-friends'
 import bskyTeam from './bsky-team'
 import hotClassic from './hot-classic'
-import bestOfFollows from './best-of-follows'
 import mutuals from './mutuals'
 import { MountedAlgos } from './types'
 
@@ -13,9 +11,7 @@ const feedgenUri = (did, name) =>
 // These are custom algorithms that will be mounted directly onto an AppView
 // Feel free to remove, update to your own, or serve the following logic at a record that you control
 export const makeAlgos = (did: string): MountedAlgos => ({
-  [feedgenUri(did, 'with-friends')]: withFriends,
   [feedgenUri(did, 'bsky-team')]: bskyTeam,
   [feedgenUri(did, 'hot-classic')]: hotClassic,
-  [feedgenUri(did, 'best-of-follows')]: bestOfFollows,
   [feedgenUri(did, 'mutuals')]: mutuals,
 })

package/src/index.ts

@@ -6,6 +6,12 @@ import { createHttpTerminator, HttpTerminator } from 'http-terminator'
 import cors from 'cors'
 import compression from 'compression'
 import { IdResolver } from '@atproto/identity'
+import {
+  RateLimiter,
+  RateLimiterOpts,
+  Options as XrpcServerOptions,
+} from '@atproto/xrpc-server'
+import { MINUTE } from '@atproto/common'
 import API, { health, wellKnown, blobResolver } from './api'
 import { DatabaseCoordinator } from './db'
 import * as error from './error'
@@ -16,17 +22,17 @@ import { ImageUriBuilder } from './image/uri'
 import { BlobDiskCache, ImageProcessingServer } from './image/server'
 import { createServices } from './services'
 import AppContext from './context'
-import DidSqlCache from './did-cache'
+import DidRedisCache from './did-cache'
 import {
   ImageInvalidator,
   ImageProcessingServerInvalidator,
 } from './image/invalidator'
 import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
-import { LabelCache } from './label-cache'
 import { NotificationServer } from './notifications'
 import { AtpAgent } from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
+import { Redis } from './redis'
 
 export type { ServerConfigValues } from './config'
 export type { MountedAlgos } from './feed-gen/types'
@@ -37,6 +43,7 @@ export { Redis } from './redis'
 export { ViewMaintainer } from './db/views'
 export { AppContext } from './context'
 export { makeAlgos } from './feed-gen'
+export * from './daemon'
 export * from './indexer'
 export * from './ingester'
 export { MigrateModerationData } from './migrate-moderation-data'
@@ -55,23 +62,25 @@ export class BskyAppView {
 
   static create(opts: {
     db: DatabaseCoordinator
+    redis: Redis
     config: ServerConfig
     signingKey: Keypair
     imgInvalidator?: ImageInvalidator
     algos?: MountedAlgos
   }): BskyAppView {
-    const { db, config, signingKey, algos = {} } = opts
+    const { db, redis, config, signingKey, algos = {} } = opts
     let maybeImgInvalidator = opts.imgInvalidator
     const app = express()
+    app.set('trust proxy', true)
     app.use(cors())
     app.use(loggerMiddleware)
     app.use(compression())
 
-    const didCache = new DidSqlCache(
-      db.getPrimary(),
-      config.didCacheStaleTTL,
-      config.didCacheMaxTTL,
-    )
+    const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
+      staleTTL: config.didCacheStaleTTL,
+      maxTTL: config.didCacheMaxTTL,
+    })
+
     const idResolver = new IdResolver({
       plcUrl: config.didPlcUrl,
       didCache,
@@ -102,7 +111,7 @@ export class BskyAppView {
     }
 
     const backgroundQueue = new BackgroundQueue(db.getPrimary())
-    const labelCache = new LabelCache(db.getPrimary())
+
     const notifServer = new NotificationServer(db.getPrimary())
     const searchAgent = config.searchEndpoint
       ? new AtpAgent({ service: config.searchEndpoint })
@@ -111,7 +120,11 @@ export class BskyAppView {
     const services = createServices({
       imgUriBuilder,
       imgInvalidator,
-      labelCache,
+      labelCacheOpts: {
+        redis: redis.withNamespace('label'),
+        staleTTL: config.labelCacheStaleTTL,
+        maxTTL: config.labelCacheMaxTTL,
+      },
     })
 
     const ctx = new AppContext({
@@ -122,21 +135,48 @@ export class BskyAppView {
       signingKey,
       idResolver,
       didCache,
-      labelCache,
+      redis,
       backgroundQueue,
       searchAgent,
       algos,
       notifServer,
     })
 
-    let server = createServer({
+    const xrpcOpts: XrpcServerOptions = {
       validateResponse: config.debugMode,
       payload: {
         jsonLimit: 100 * 1024, // 100kb
         textLimit: 100 * 1024, // 100kb
         blobLimit: 5 * 1024 * 1024, // 5mb
       },
-    })
+    }
+    if (config.rateLimitsEnabled) {
+      const rlCreator = (opts: RateLimiterOpts) =>
+        RateLimiter.redis(redis.driver, {
+          bypassSecret: config.rateLimitBypassKey,
+          bypassIps: config.rateLimitBypassIps,
+          ...opts,
+        })
+      xrpcOpts['rateLimits'] = {
+        creator: rlCreator,
+        global: [
+          {
+            name: 'global-unauthed-ip',
+            durationMs: 5 * MINUTE,
+            points: 3000,
+            calcKey: (ctx) => (ctx.auth ? null : ctx.req.ip),
+          },
+          {
+            name: 'global-authed-did',
+            durationMs: 5 * MINUTE,
+            points: 3000,
+            calcKey: (ctx) => ctx.auth?.credentials?.did ?? null,
+          },
+        ],
+      }
+    }
+
+    let server = createServer(xrpcOpts)
 
     server = API(server, ctx)
 
@@ -185,7 +225,6 @@ export class BskyAppView {
         'background queue stats',
       )
     }, 10000)
-    this.ctx.labelCache.start()
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
@@ -196,11 +235,11 @@ export class BskyAppView {
     return server
   }
 
-  async destroy(opts?: { skipDb: boolean }): Promise<void> {
-    this.ctx.labelCache.stop()
+  async destroy(opts?: { skipDb: boolean; skipRedis: boolean }): Promise<void> {
     await this.ctx.didCache.destroy()
     await this.terminator?.terminate()
     await this.ctx.backgroundQueue.destroy()
+    if (!opts?.skipRedis) await this.ctx.redis.destroy()
     if (!opts?.skipDb) await this.ctx.db.close()
     clearInterval(this.dbStatsInterval)
   }

package/src/indexer/context.ts

@@ -12,6 +12,7 @@ export class IndexerContext {
     private opts: {
       db: PrimaryDatabase
       redis: Redis
+      redisCache: Redis
       cfg: IndexerConfig
       services: Services
       idResolver: IdResolver
@@ -29,6 +30,10 @@ export class IndexerContext {
     return this.opts.redis
   }
 
+  get redisCache(): Redis {
+    return this.opts.redisCache
+  }
+
   get cfg(): IndexerConfig {
     return this.opts.cfg
   }

package/src/indexer/index.ts

@@ -2,7 +2,7 @@ import express from 'express'
 import { IdResolver } from '@atproto/identity'
 import { BackgroundQueue } from '../background'
 import { PrimaryDatabase } from '../db'
-import DidSqlCache from '../did-cache'
+import DidRedisCache from '../did-cache'
 import log from './logger'
 import { dbLogger } from '../logger'
 import { IndexerConfig } from './config'
@@ -40,15 +40,15 @@ export class BskyIndexer {
   static create(opts: {
     db: PrimaryDatabase
     redis: Redis
+    redisCache: Redis
     cfg: IndexerConfig
     imgInvalidator?: ImageInvalidator
   }): BskyIndexer {
-    const { db, redis, cfg } = opts
-    const didCache = new DidSqlCache(
-      db,
-      cfg.didCacheStaleTTL,
-      cfg.didCacheMaxTTL,
-    )
+    const { db, redis, redisCache, cfg } = opts
+    const didCache = new DidRedisCache(redisCache.withNamespace('did-doc'), {
+      staleTTL: cfg.didCacheStaleTTL,
+      maxTTL: cfg.didCacheMaxTTL,
+    })
     const idResolver = new IdResolver({
       plcUrl: cfg.didPlcUrl,
       didCache,
@@ -81,6 +81,7 @@ export class BskyIndexer {
     const ctx = new IndexerContext({
       db,
       redis,
+      redisCache,
       cfg,
       services,
       idResolver,
@@ -139,7 +140,9 @@ export class BskyIndexer {
     if (this.closeServer) await this.closeServer()
     await this.sub.destroy()
     clearInterval(this.subStatsInterval)
+    await this.ctx.didCache.destroy()
     if (!opts?.skipRedis) await this.ctx.redis.destroy()
+    if (!opts?.skipRedis) await this.ctx.redisCache.destroy()
     if (!opts?.skipDb) await this.ctx.db.close()
     clearInterval(this.dbStatsInterval)
   }

package/src/lexicon/index.ts

@@ -9,6 +9,7 @@ import {
   StreamAuthVerifier,
 } from '@atproto/xrpc-server'
 import { schemas } from './lexicons'
+import * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount'
 import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites'
 import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes'
 import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent'
@@ -73,6 +74,9 @@ import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf
 import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl'
 import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos'
 import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels'
+import * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo'
+import * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob'
+import * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount'
 import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences'
 import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile'
 import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles'
@@ -194,6 +198,17 @@ export class AdminNS {
     this._server = server
   }
 
+  deleteAccount<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoAdminDeleteAccount.Handler<ExtractAuth<AV>>,
+      ComAtprotoAdminDeleteAccount.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.admin.deleteAccount' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
   disableAccountInvites<AV extends AuthVerifier>(
     cfg: ConfigOf<
       AV,
@@ -953,6 +968,39 @@ export class TempNS {
     const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore
     return this._server.xrpc.method(nsid, cfg)
   }
+
+  importRepo<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoTempImportRepo.Handler<ExtractAuth<AV>>,
+      ComAtprotoTempImportRepo.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.temp.importRepo' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
+  pushBlob<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoTempPushBlob.Handler<ExtractAuth<AV>>,
+      ComAtprotoTempPushBlob.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.temp.pushBlob' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
+  transferAccount<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoTempTransferAccount.Handler<ExtractAuth<AV>>,
+      ComAtprotoTempTransferAccount.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.temp.transferAccount' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
 }
 
 export class AppNS {
@@ -1549,11 +1597,13 @@ type RouteRateLimitOpts<T> = {
   calcKey?: (ctx: T) => string
   calcPoints?: (ctx: T) => number
 }
+type HandlerOpts = { blobLimit?: number }
 type HandlerRateLimitOpts<T> = SharedRateLimitOpts<T> | RouteRateLimitOpts<T>
 type ConfigOf<Auth, Handler, ReqCtx> =
   | Handler
   | {
       auth?: Auth
+      opts?: HandlerOpts
       rateLimit?: HandlerRateLimitOpts<ReqCtx> | HandlerRateLimitOpts<ReqCtx>[]
       handler: Handler
     }