@atproto/bsky 0.0.16 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/cache/read-through.d.ts +30 -0
  3. package/dist/config.d.ts +18 -0
  4. package/dist/context.d.ts +6 -6
  5. package/dist/daemon/config.d.ts +15 -0
  6. package/dist/daemon/context.d.ts +15 -0
  7. package/dist/daemon/index.d.ts +23 -0
  8. package/dist/daemon/logger.d.ts +3 -0
  9. package/dist/daemon/notifications.d.ts +18 -0
  10. package/dist/daemon/services.d.ts +11 -0
  11. package/dist/db/database-schema.d.ts +1 -2
  12. package/dist/db/index.js +16 -1
  13. package/dist/db/index.js.map +3 -3
  14. package/dist/db/migrations/20231205T000257238Z-remove-did-cache.d.ts +3 -0
  15. package/dist/db/migrations/index.d.ts +1 -0
  16. package/dist/did-cache.d.ts +10 -7
  17. package/dist/index.d.ts +4 -0
  18. package/dist/index.js +1921 -938
  19. package/dist/index.js.map +3 -3
  20. package/dist/indexer/context.d.ts +2 -0
  21. package/dist/indexer/index.d.ts +1 -0
  22. package/dist/lexicon/index.d.ts +12 -0
  23. package/dist/lexicon/lexicons.d.ts +134 -0
  24. package/dist/lexicon/types/com/atproto/admin/deleteAccount.d.ts +25 -0
  25. package/dist/lexicon/types/com/atproto/temp/importRepo.d.ts +32 -0
  26. package/dist/lexicon/types/com/atproto/temp/pushBlob.d.ts +25 -0
  27. package/dist/lexicon/types/com/atproto/temp/transferAccount.d.ts +42 -0
  28. package/dist/logger.d.ts +1 -0
  29. package/dist/redis.d.ts +10 -1
  30. package/dist/services/actor/index.d.ts +18 -4
  31. package/dist/services/actor/views.d.ts +5 -7
  32. package/dist/services/feed/index.d.ts +6 -4
  33. package/dist/services/feed/views.d.ts +5 -4
  34. package/dist/services/index.d.ts +3 -7
  35. package/dist/services/label/index.d.ts +10 -4
  36. package/dist/services/moderation/index.d.ts +0 -1
  37. package/dist/services/types.d.ts +3 -0
  38. package/dist/services/util/notification.d.ts +5 -0
  39. package/dist/services/util/post.d.ts +6 -6
  40. package/dist/util/retry.d.ts +1 -6
  41. package/package.json +6 -6
  42. package/src/api/app/bsky/actor/searchActorsTypeahead.ts +1 -1
  43. package/src/cache/read-through.ts +151 -0
  44. package/src/config.ts +90 -1
  45. package/src/context.ts +7 -7
  46. package/src/daemon/config.ts +60 -0
  47. package/src/daemon/context.ts +27 -0
  48. package/src/daemon/index.ts +78 -0
  49. package/src/daemon/logger.ts +6 -0
  50. package/src/daemon/notifications.ts +54 -0
  51. package/src/daemon/services.ts +22 -0
  52. package/src/db/database-schema.ts +0 -2
  53. package/src/db/migrations/20231205T000257238Z-remove-did-cache.ts +14 -0
  54. package/src/db/migrations/index.ts +1 -0
  55. package/src/did-cache.ts +33 -56
  56. package/src/feed-gen/index.ts +0 -4
  57. package/src/index.ts +55 -16
  58. package/src/indexer/context.ts +5 -0
  59. package/src/indexer/index.ts +10 -7
  60. package/src/lexicon/index.ts +50 -0
  61. package/src/lexicon/lexicons.ts +156 -0
  62. package/src/lexicon/types/com/atproto/admin/deleteAccount.ts +38 -0
  63. package/src/lexicon/types/com/atproto/temp/importRepo.ts +45 -0
  64. package/src/lexicon/types/com/atproto/temp/pushBlob.ts +39 -0
  65. package/src/lexicon/types/com/atproto/temp/transferAccount.ts +62 -0
  66. package/src/logger.ts +2 -0
  67. package/src/redis.ts +43 -3
  68. package/src/services/actor/index.ts +55 -7
  69. package/src/services/actor/views.ts +16 -13
  70. package/src/services/feed/index.ts +27 -13
  71. package/src/services/feed/views.ts +20 -10
  72. package/src/services/index.ts +14 -14
  73. package/src/services/indexing/index.ts +7 -10
  74. package/src/services/indexing/plugins/post.ts +13 -0
  75. package/src/services/label/index.ts +66 -22
  76. package/src/services/moderation/index.ts +1 -1
  77. package/src/services/moderation/status.ts +1 -4
  78. package/src/services/types.ts +4 -0
  79. package/src/services/util/notification.ts +70 -0
  80. package/src/util/retry.ts +1 -44
  81. package/tests/admin/get-repo.test.ts +5 -3
  82. package/tests/admin/moderation.test.ts +2 -2
  83. package/tests/admin/repo-search.test.ts +1 -0
  84. package/tests/algos/hot-classic.test.ts +1 -2
  85. package/tests/auth.test.ts +1 -1
  86. package/tests/auto-moderator/labeler.test.ts +19 -20
  87. package/tests/auto-moderator/takedowns.test.ts +16 -10
  88. package/tests/blob-resolver.test.ts +4 -2
  89. package/tests/daemon.test.ts +191 -0
  90. package/tests/did-cache.test.ts +20 -5
  91. package/tests/handle-invalidation.test.ts +1 -5
  92. package/tests/indexing.test.ts +20 -13
  93. package/tests/redis-cache.test.ts +231 -0
  94. package/tests/seeds/basic.ts +3 -0
  95. package/tests/subscription/repo.test.ts +4 -7
  96. package/tests/views/profile.test.ts +0 -1
  97. package/tests/views/thread.test.ts +73 -78
  98. package/tests/views/threadgating.test.ts +38 -0
  99. package/dist/db/tables/did-cache.d.ts +0 -10
  100. package/dist/feed-gen/best-of-follows.d.ts +0 -29
  101. package/dist/feed-gen/whats-hot.d.ts +0 -29
  102. package/dist/feed-gen/with-friends.d.ts +0 -3
  103. package/dist/label-cache.d.ts +0 -19
  104. package/src/db/tables/did-cache.ts +0 -13
  105. package/src/feed-gen/best-of-follows.ts +0 -77
  106. package/src/feed-gen/whats-hot.ts +0 -101
  107. package/src/feed-gen/with-friends.ts +0 -43
  108. package/src/label-cache.ts +0 -90
  109. package/tests/algos/whats-hot.test.ts +0 -118
  110. package/tests/algos/with-friends.test.ts +0 -145
@@ -0,0 +1,78 @@
1
+ import { PrimaryDatabase } from '../db'
2
+ import { dbLogger } from '../logger'
3
+ import { DaemonConfig } from './config'
4
+ import { DaemonContext } from './context'
5
+ import { createServices } from './services'
6
+ import { ImageUriBuilder } from '../image/uri'
7
+ import { NotificationsDaemon } from './notifications'
8
+ import logger from './logger'
9
+
10
+ export { DaemonConfig } from './config'
11
+ export type { DaemonConfigValues } from './config'
12
+
13
+ export class BskyDaemon {
14
+ public ctx: DaemonContext
15
+ public notifications: NotificationsDaemon
16
+ private dbStatsInterval: NodeJS.Timer
17
+ private notifStatsInterval: NodeJS.Timer
18
+
19
+ constructor(opts: {
20
+ ctx: DaemonContext
21
+ notifications: NotificationsDaemon
22
+ }) {
23
+ this.ctx = opts.ctx
24
+ this.notifications = opts.notifications
25
+ }
26
+
27
+ static create(opts: { db: PrimaryDatabase; cfg: DaemonConfig }): BskyDaemon {
28
+ const { db, cfg } = opts
29
+ const imgUriBuilder = new ImageUriBuilder('https://daemon.invalid') // will not be used by daemon
30
+ const services = createServices({
31
+ imgUriBuilder,
32
+ })
33
+ const ctx = new DaemonContext({
34
+ db,
35
+ cfg,
36
+ services,
37
+ })
38
+ const notifications = new NotificationsDaemon(ctx)
39
+ return new BskyDaemon({ ctx, notifications })
40
+ }
41
+
42
+ async start() {
43
+ const { db, cfg } = this.ctx
44
+ const pool = db.pool
45
+ this.notifications.run({
46
+ startFromDid: cfg.notificationsDaemonFromDid,
47
+ })
48
+ this.dbStatsInterval = setInterval(() => {
49
+ dbLogger.info(
50
+ {
51
+ idleCount: pool.idleCount,
52
+ totalCount: pool.totalCount,
53
+ waitingCount: pool.waitingCount,
54
+ },
55
+ 'db pool stats',
56
+ )
57
+ }, 10000)
58
+ this.notifStatsInterval = setInterval(() => {
59
+ logger.info(
60
+ {
61
+ count: this.notifications.count,
62
+ lastDid: this.notifications.lastDid,
63
+ },
64
+ 'notifications daemon stats',
65
+ )
66
+ }, 10000)
67
+ return this
68
+ }
69
+
70
+ async destroy(): Promise<void> {
71
+ await this.notifications.destroy()
72
+ await this.ctx.db.close()
73
+ clearInterval(this.dbStatsInterval)
74
+ clearInterval(this.notifStatsInterval)
75
+ }
76
+ }
77
+
78
+ export default BskyDaemon
@@ -0,0 +1,6 @@
1
+ import { subsystemLogger } from '@atproto/common'
2
+
3
+ const logger: ReturnType<typeof subsystemLogger> =
4
+ subsystemLogger('bsky:daemon')
5
+
6
+ export default logger
@@ -0,0 +1,54 @@
1
+ import { tidyNotifications } from '../services/util/notification'
2
+ import DaemonContext from './context'
3
+ import logger from './logger'
4
+
5
+ export class NotificationsDaemon {
6
+ ac = new AbortController()
7
+ running: Promise<void> | undefined
8
+ count = 0
9
+ lastDid: string | null = null
10
+
11
+ constructor(private ctx: DaemonContext) {}
12
+
13
+ run(opts?: RunOptions) {
14
+ if (this.running) return
15
+ this.count = 0
16
+ this.lastDid = null
17
+ this.ac = new AbortController()
18
+ this.running = this.tidyNotifications({
19
+ ...opts,
20
+ forever: opts?.forever !== false, // run forever by default
21
+ })
22
+ .catch((err) => {
23
+ // allow this to cause an unhandled rejection, let deployment handle the crash.
24
+ logger.error({ err }, 'notifications daemon crashed')
25
+ throw err
26
+ })
27
+ .finally(() => (this.running = undefined))
28
+ }
29
+
30
+ private async tidyNotifications(opts: RunOptions) {
31
+ const actorService = this.ctx.services.actor(this.ctx.db)
32
+ for await (const { did } of actorService.all(opts)) {
33
+ if (this.ac.signal.aborted) return
34
+ try {
35
+ await tidyNotifications(this.ctx.db, did)
36
+ this.count++
37
+ this.lastDid = did
38
+ } catch (err) {
39
+ logger.warn({ err, did }, 'failed to tidy notifications for actor')
40
+ }
41
+ }
42
+ }
43
+
44
+ async destroy() {
45
+ this.ac.abort()
46
+ await this.running
47
+ }
48
+ }
49
+
50
+ type RunOptions = {
51
+ forever?: boolean
52
+ batchSize?: number
53
+ startFromDid?: string
54
+ }
@@ -0,0 +1,22 @@
1
+ import { PrimaryDatabase } from '../db'
2
+ import { ActorService } from '../services/actor'
3
+ import { ImageUriBuilder } from '../image/uri'
4
+ import { GraphService } from '../services/graph'
5
+ import { LabelService } from '../services/label'
6
+
7
+ export function createServices(resources: {
8
+ imgUriBuilder: ImageUriBuilder
9
+ }): Services {
10
+ const { imgUriBuilder } = resources
11
+ const graph = GraphService.creator(imgUriBuilder)
12
+ const label = LabelService.creator(null)
13
+ return {
14
+ actor: ActorService.creator(imgUriBuilder, graph, label),
15
+ }
16
+ }
17
+
18
+ export type Services = {
19
+ actor: FromDbPrimary<ActorService>
20
+ }
21
+
22
+ type FromDbPrimary<T> = (db: PrimaryDatabase) => T
@@ -24,7 +24,6 @@ import * as actorSync from './tables/actor-sync'
24
24
  import * as record from './tables/record'
25
25
  import * as notification from './tables/notification'
26
26
  import * as notificationPushToken from './tables/notification-push-token'
27
- import * as didCache from './tables/did-cache'
28
27
  import * as moderation from './tables/moderation'
29
28
  import * as label from './tables/label'
30
29
  import * as algo from './tables/algo'
@@ -57,7 +56,6 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
57
56
  record.PartialDB &
58
57
  notification.PartialDB &
59
58
  notificationPushToken.PartialDB &
60
- didCache.PartialDB &
61
59
  moderation.PartialDB &
62
60
  label.PartialDB &
63
61
  algo.PartialDB &
@@ -0,0 +1,14 @@
1
+ import { Kysely } from 'kysely'
2
+
3
+ export async function up(db: Kysely<unknown>): Promise<void> {
4
+ await db.schema.dropTable('did_cache').execute()
5
+ }
6
+
7
+ export async function down(db: Kysely<unknown>): Promise<void> {
8
+ await db.schema
9
+ .createTable('did_cache')
10
+ .addColumn('did', 'varchar', (col) => col.primaryKey())
11
+ .addColumn('doc', 'jsonb', (col) => col.notNull())
12
+ .addColumn('updatedAt', 'bigint', (col) => col.notNull())
13
+ .execute()
14
+ }
@@ -31,3 +31,4 @@ export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
31
31
  export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
32
32
  export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'
33
33
  export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
34
+ export * as _20231205T000257238Z from './20231205T000257238Z-remove-did-cache'
package/src/did-cache.ts CHANGED
@@ -1,81 +1,61 @@
1
1
  import PQueue from 'p-queue'
2
2
  import { CacheResult, DidCache, DidDocument } from '@atproto/identity'
3
- import { PrimaryDatabase } from './db'
4
- import { excluded } from './db/util'
5
- import { dbLogger } from './logger'
3
+ import { cacheLogger as log } from './logger'
4
+ import { Redis } from './redis'
6
5
 
7
- export class DidSqlCache implements DidCache {
8
- public pQueue: PQueue | null //null during teardown
6
+ type CacheOptions = {
7
+ staleTTL: number
8
+ maxTTL: number
9
+ }
10
+
11
+ export class DidRedisCache implements DidCache {
12
+ public pQueue: PQueue | null // null during teardown
9
13
 
10
- constructor(
11
- // @TODO perhaps could use both primary and non-primary. not high enough
12
- // throughput to matter right now. also may just move this over to redis before long!
13
- public db: PrimaryDatabase,
14
- public staleTTL: number,
15
- public maxTTL: number,
16
- ) {
14
+ constructor(public redis: Redis, public opts: CacheOptions) {
17
15
  this.pQueue = new PQueue()
18
16
  }
19
17
 
20
- async cacheDid(
21
- did: string,
22
- doc: DidDocument,
23
- prevResult?: CacheResult,
24
- ): Promise<void> {
25
- if (prevResult) {
26
- await this.db.db
27
- .updateTable('did_cache')
28
- .set({ doc, updatedAt: Date.now() })
29
- .where('did', '=', did)
30
- .where('updatedAt', '=', prevResult.updatedAt)
31
- .execute()
32
- } else {
33
- await this.db.db
34
- .insertInto('did_cache')
35
- .values({ did, doc, updatedAt: Date.now() })
36
- .onConflict((oc) =>
37
- oc.column('did').doUpdateSet({
38
- doc: excluded(this.db.db, 'doc'),
39
- updatedAt: excluded(this.db.db, 'updatedAt'),
40
- }),
41
- )
42
- .executeTakeFirst()
43
- }
18
+ async cacheDid(did: string, doc: DidDocument): Promise<void> {
19
+ const item = JSON.stringify({
20
+ doc,
21
+ updatedAt: Date.now(),
22
+ })
23
+ await this.redis.set(did, item, this.opts.maxTTL)
44
24
  }
45
25
 
46
26
  async refreshCache(
47
27
  did: string,
48
28
  getDoc: () => Promise<DidDocument | null>,
49
- prevResult?: CacheResult,
50
29
  ): Promise<void> {
51
30
  this.pQueue?.add(async () => {
52
31
  try {
53
32
  const doc = await getDoc()
54
33
  if (doc) {
55
- await this.cacheDid(did, doc, prevResult)
34
+ await this.cacheDid(did, doc)
56
35
  } else {
57
36
  await this.clearEntry(did)
58
37
  }
59
38
  } catch (err) {
60
- dbLogger.error({ did, err }, 'refreshing did cache failed')
39
+ log.error({ did, err }, 'refreshing did cache failed')
61
40
  }
62
41
  })
63
42
  }
64
43
 
65
44
  async checkCache(did: string): Promise<CacheResult | null> {
66
- const res = await this.db.db
67
- .selectFrom('did_cache')
68
- .where('did', '=', did)
69
- .selectAll()
70
- .executeTakeFirst()
71
- if (!res) return null
72
-
45
+ let got: string | null
46
+ try {
47
+ got = await this.redis.get(did)
48
+ } catch (err) {
49
+ got = null
50
+ log.error({ did, err }, 'error fetching did from cache')
51
+ }
52
+ if (!got) return null
53
+ const { doc, updatedAt } = JSON.parse(got) as CacheResult
73
54
  const now = Date.now()
74
- const updatedAt = new Date(res.updatedAt).getTime()
75
- const expired = now > updatedAt + this.maxTTL
76
- const stale = now > updatedAt + this.staleTTL
55
+ const expired = now > updatedAt + this.opts.maxTTL
56
+ const stale = now > updatedAt + this.opts.staleTTL
77
57
  return {
78
- doc: res.doc,
58
+ doc,
79
59
  updatedAt,
80
60
  did,
81
61
  stale,
@@ -84,14 +64,11 @@ export class DidSqlCache implements DidCache {
84
64
  }
85
65
 
86
66
  async clearEntry(did: string): Promise<void> {
87
- await this.db.db
88
- .deleteFrom('did_cache')
89
- .where('did', '=', did)
90
- .executeTakeFirst()
67
+ await this.redis.del(did)
91
68
  }
92
69
 
93
70
  async clear(): Promise<void> {
94
- await this.db.db.deleteFrom('did_cache').execute()
71
+ throw new Error('Not implemented for redis cache')
95
72
  }
96
73
 
97
74
  async processAll() {
@@ -107,4 +84,4 @@ export class DidSqlCache implements DidCache {
107
84
  }
108
85
  }
109
86
 
110
- export default DidSqlCache
87
+ export default DidRedisCache
@@ -1,9 +1,7 @@
1
1
  import { AtUri } from '@atproto/syntax'
2
2
  import { ids } from '../lexicon/lexicons'
3
- import withFriends from './with-friends'
4
3
  import bskyTeam from './bsky-team'
5
4
  import hotClassic from './hot-classic'
6
- import bestOfFollows from './best-of-follows'
7
5
  import mutuals from './mutuals'
8
6
  import { MountedAlgos } from './types'
9
7
 
@@ -13,9 +11,7 @@ const feedgenUri = (did, name) =>
13
11
  // These are custom algorithms that will be mounted directly onto an AppView
14
12
  // Feel free to remove, update to your own, or serve the following logic at a record that you control
15
13
  export const makeAlgos = (did: string): MountedAlgos => ({
16
- [feedgenUri(did, 'with-friends')]: withFriends,
17
14
  [feedgenUri(did, 'bsky-team')]: bskyTeam,
18
15
  [feedgenUri(did, 'hot-classic')]: hotClassic,
19
- [feedgenUri(did, 'best-of-follows')]: bestOfFollows,
20
16
  [feedgenUri(did, 'mutuals')]: mutuals,
21
17
  })
package/src/index.ts CHANGED
@@ -6,6 +6,12 @@ import { createHttpTerminator, HttpTerminator } from 'http-terminator'
6
6
  import cors from 'cors'
7
7
  import compression from 'compression'
8
8
  import { IdResolver } from '@atproto/identity'
9
+ import {
10
+ RateLimiter,
11
+ RateLimiterOpts,
12
+ Options as XrpcServerOptions,
13
+ } from '@atproto/xrpc-server'
14
+ import { MINUTE } from '@atproto/common'
9
15
  import API, { health, wellKnown, blobResolver } from './api'
10
16
  import { DatabaseCoordinator } from './db'
11
17
  import * as error from './error'
@@ -16,17 +22,17 @@ import { ImageUriBuilder } from './image/uri'
16
22
  import { BlobDiskCache, ImageProcessingServer } from './image/server'
17
23
  import { createServices } from './services'
18
24
  import AppContext from './context'
19
- import DidSqlCache from './did-cache'
25
+ import DidRedisCache from './did-cache'
20
26
  import {
21
27
  ImageInvalidator,
22
28
  ImageProcessingServerInvalidator,
23
29
  } from './image/invalidator'
24
30
  import { BackgroundQueue } from './background'
25
31
  import { MountedAlgos } from './feed-gen/types'
26
- import { LabelCache } from './label-cache'
27
32
  import { NotificationServer } from './notifications'
28
33
  import { AtpAgent } from '@atproto/api'
29
34
  import { Keypair } from '@atproto/crypto'
35
+ import { Redis } from './redis'
30
36
 
31
37
  export type { ServerConfigValues } from './config'
32
38
  export type { MountedAlgos } from './feed-gen/types'
@@ -37,6 +43,7 @@ export { Redis } from './redis'
37
43
  export { ViewMaintainer } from './db/views'
38
44
  export { AppContext } from './context'
39
45
  export { makeAlgos } from './feed-gen'
46
+ export * from './daemon'
40
47
  export * from './indexer'
41
48
  export * from './ingester'
42
49
  export { MigrateModerationData } from './migrate-moderation-data'
@@ -55,23 +62,25 @@ export class BskyAppView {
55
62
 
56
63
  static create(opts: {
57
64
  db: DatabaseCoordinator
65
+ redis: Redis
58
66
  config: ServerConfig
59
67
  signingKey: Keypair
60
68
  imgInvalidator?: ImageInvalidator
61
69
  algos?: MountedAlgos
62
70
  }): BskyAppView {
63
- const { db, config, signingKey, algos = {} } = opts
71
+ const { db, redis, config, signingKey, algos = {} } = opts
64
72
  let maybeImgInvalidator = opts.imgInvalidator
65
73
  const app = express()
74
+ app.set('trust proxy', true)
66
75
  app.use(cors())
67
76
  app.use(loggerMiddleware)
68
77
  app.use(compression())
69
78
 
70
- const didCache = new DidSqlCache(
71
- db.getPrimary(),
72
- config.didCacheStaleTTL,
73
- config.didCacheMaxTTL,
74
- )
79
+ const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
80
+ staleTTL: config.didCacheStaleTTL,
81
+ maxTTL: config.didCacheMaxTTL,
82
+ })
83
+
75
84
  const idResolver = new IdResolver({
76
85
  plcUrl: config.didPlcUrl,
77
86
  didCache,
@@ -102,7 +111,7 @@ export class BskyAppView {
102
111
  }
103
112
 
104
113
  const backgroundQueue = new BackgroundQueue(db.getPrimary())
105
- const labelCache = new LabelCache(db.getPrimary())
114
+
106
115
  const notifServer = new NotificationServer(db.getPrimary())
107
116
  const searchAgent = config.searchEndpoint
108
117
  ? new AtpAgent({ service: config.searchEndpoint })
@@ -111,7 +120,11 @@ export class BskyAppView {
111
120
  const services = createServices({
112
121
  imgUriBuilder,
113
122
  imgInvalidator,
114
- labelCache,
123
+ labelCacheOpts: {
124
+ redis: redis.withNamespace('label'),
125
+ staleTTL: config.labelCacheStaleTTL,
126
+ maxTTL: config.labelCacheMaxTTL,
127
+ },
115
128
  })
116
129
 
117
130
  const ctx = new AppContext({
@@ -122,21 +135,48 @@ export class BskyAppView {
122
135
  signingKey,
123
136
  idResolver,
124
137
  didCache,
125
- labelCache,
138
+ redis,
126
139
  backgroundQueue,
127
140
  searchAgent,
128
141
  algos,
129
142
  notifServer,
130
143
  })
131
144
 
132
- let server = createServer({
145
+ const xrpcOpts: XrpcServerOptions = {
133
146
  validateResponse: config.debugMode,
134
147
  payload: {
135
148
  jsonLimit: 100 * 1024, // 100kb
136
149
  textLimit: 100 * 1024, // 100kb
137
150
  blobLimit: 5 * 1024 * 1024, // 5mb
138
151
  },
139
- })
152
+ }
153
+ if (config.rateLimitsEnabled) {
154
+ const rlCreator = (opts: RateLimiterOpts) =>
155
+ RateLimiter.redis(redis.driver, {
156
+ bypassSecret: config.rateLimitBypassKey,
157
+ bypassIps: config.rateLimitBypassIps,
158
+ ...opts,
159
+ })
160
+ xrpcOpts['rateLimits'] = {
161
+ creator: rlCreator,
162
+ global: [
163
+ {
164
+ name: 'global-unauthed-ip',
165
+ durationMs: 5 * MINUTE,
166
+ points: 3000,
167
+ calcKey: (ctx) => (ctx.auth ? null : ctx.req.ip),
168
+ },
169
+ {
170
+ name: 'global-authed-did',
171
+ durationMs: 5 * MINUTE,
172
+ points: 3000,
173
+ calcKey: (ctx) => ctx.auth?.credentials?.did ?? null,
174
+ },
175
+ ],
176
+ }
177
+ }
178
+
179
+ let server = createServer(xrpcOpts)
140
180
 
141
181
  server = API(server, ctx)
142
182
 
@@ -185,7 +225,6 @@ export class BskyAppView {
185
225
  'background queue stats',
186
226
  )
187
227
  }, 10000)
188
- this.ctx.labelCache.start()
189
228
  const server = this.app.listen(this.ctx.cfg.port)
190
229
  this.server = server
191
230
  server.keepAliveTimeout = 90000
@@ -196,11 +235,11 @@ export class BskyAppView {
196
235
  return server
197
236
  }
198
237
 
199
- async destroy(opts?: { skipDb: boolean }): Promise<void> {
200
- this.ctx.labelCache.stop()
238
+ async destroy(opts?: { skipDb: boolean; skipRedis: boolean }): Promise<void> {
201
239
  await this.ctx.didCache.destroy()
202
240
  await this.terminator?.terminate()
203
241
  await this.ctx.backgroundQueue.destroy()
242
+ if (!opts?.skipRedis) await this.ctx.redis.destroy()
204
243
  if (!opts?.skipDb) await this.ctx.db.close()
205
244
  clearInterval(this.dbStatsInterval)
206
245
  }
@@ -12,6 +12,7 @@ export class IndexerContext {
12
12
  private opts: {
13
13
  db: PrimaryDatabase
14
14
  redis: Redis
15
+ redisCache: Redis
15
16
  cfg: IndexerConfig
16
17
  services: Services
17
18
  idResolver: IdResolver
@@ -29,6 +30,10 @@ export class IndexerContext {
29
30
  return this.opts.redis
30
31
  }
31
32
 
33
+ get redisCache(): Redis {
34
+ return this.opts.redisCache
35
+ }
36
+
32
37
  get cfg(): IndexerConfig {
33
38
  return this.opts.cfg
34
39
  }
@@ -2,7 +2,7 @@ import express from 'express'
2
2
  import { IdResolver } from '@atproto/identity'
3
3
  import { BackgroundQueue } from '../background'
4
4
  import { PrimaryDatabase } from '../db'
5
- import DidSqlCache from '../did-cache'
5
+ import DidRedisCache from '../did-cache'
6
6
  import log from './logger'
7
7
  import { dbLogger } from '../logger'
8
8
  import { IndexerConfig } from './config'
@@ -40,15 +40,15 @@ export class BskyIndexer {
40
40
  static create(opts: {
41
41
  db: PrimaryDatabase
42
42
  redis: Redis
43
+ redisCache: Redis
43
44
  cfg: IndexerConfig
44
45
  imgInvalidator?: ImageInvalidator
45
46
  }): BskyIndexer {
46
- const { db, redis, cfg } = opts
47
- const didCache = new DidSqlCache(
48
- db,
49
- cfg.didCacheStaleTTL,
50
- cfg.didCacheMaxTTL,
51
- )
47
+ const { db, redis, redisCache, cfg } = opts
48
+ const didCache = new DidRedisCache(redisCache.withNamespace('did-doc'), {
49
+ staleTTL: cfg.didCacheStaleTTL,
50
+ maxTTL: cfg.didCacheMaxTTL,
51
+ })
52
52
  const idResolver = new IdResolver({
53
53
  plcUrl: cfg.didPlcUrl,
54
54
  didCache,
@@ -81,6 +81,7 @@ export class BskyIndexer {
81
81
  const ctx = new IndexerContext({
82
82
  db,
83
83
  redis,
84
+ redisCache,
84
85
  cfg,
85
86
  services,
86
87
  idResolver,
@@ -139,7 +140,9 @@ export class BskyIndexer {
139
140
  if (this.closeServer) await this.closeServer()
140
141
  await this.sub.destroy()
141
142
  clearInterval(this.subStatsInterval)
143
+ await this.ctx.didCache.destroy()
142
144
  if (!opts?.skipRedis) await this.ctx.redis.destroy()
145
+ if (!opts?.skipRedis) await this.ctx.redisCache.destroy()
143
146
  if (!opts?.skipDb) await this.ctx.db.close()
144
147
  clearInterval(this.dbStatsInterval)
145
148
  }
@@ -9,6 +9,7 @@ import {
9
9
  StreamAuthVerifier,
10
10
  } from '@atproto/xrpc-server'
11
11
  import { schemas } from './lexicons'
12
+ import * as ComAtprotoAdminDeleteAccount from './types/com/atproto/admin/deleteAccount'
12
13
  import * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites'
13
14
  import * as ComAtprotoAdminDisableInviteCodes from './types/com/atproto/admin/disableInviteCodes'
14
15
  import * as ComAtprotoAdminEmitModerationEvent from './types/com/atproto/admin/emitModerationEvent'
@@ -73,6 +74,9 @@ import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOf
73
74
  import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl'
74
75
  import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos'
75
76
  import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels'
77
+ import * as ComAtprotoTempImportRepo from './types/com/atproto/temp/importRepo'
78
+ import * as ComAtprotoTempPushBlob from './types/com/atproto/temp/pushBlob'
79
+ import * as ComAtprotoTempTransferAccount from './types/com/atproto/temp/transferAccount'
76
80
  import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences'
77
81
  import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile'
78
82
  import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles'
@@ -194,6 +198,17 @@ export class AdminNS {
194
198
  this._server = server
195
199
  }
196
200
 
201
+ deleteAccount<AV extends AuthVerifier>(
202
+ cfg: ConfigOf<
203
+ AV,
204
+ ComAtprotoAdminDeleteAccount.Handler<ExtractAuth<AV>>,
205
+ ComAtprotoAdminDeleteAccount.HandlerReqCtx<ExtractAuth<AV>>
206
+ >,
207
+ ) {
208
+ const nsid = 'com.atproto.admin.deleteAccount' // @ts-ignore
209
+ return this._server.xrpc.method(nsid, cfg)
210
+ }
211
+
197
212
  disableAccountInvites<AV extends AuthVerifier>(
198
213
  cfg: ConfigOf<
199
214
  AV,
@@ -953,6 +968,39 @@ export class TempNS {
953
968
  const nsid = 'com.atproto.temp.fetchLabels' // @ts-ignore
954
969
  return this._server.xrpc.method(nsid, cfg)
955
970
  }
971
+
972
+ importRepo<AV extends AuthVerifier>(
973
+ cfg: ConfigOf<
974
+ AV,
975
+ ComAtprotoTempImportRepo.Handler<ExtractAuth<AV>>,
976
+ ComAtprotoTempImportRepo.HandlerReqCtx<ExtractAuth<AV>>
977
+ >,
978
+ ) {
979
+ const nsid = 'com.atproto.temp.importRepo' // @ts-ignore
980
+ return this._server.xrpc.method(nsid, cfg)
981
+ }
982
+
983
+ pushBlob<AV extends AuthVerifier>(
984
+ cfg: ConfigOf<
985
+ AV,
986
+ ComAtprotoTempPushBlob.Handler<ExtractAuth<AV>>,
987
+ ComAtprotoTempPushBlob.HandlerReqCtx<ExtractAuth<AV>>
988
+ >,
989
+ ) {
990
+ const nsid = 'com.atproto.temp.pushBlob' // @ts-ignore
991
+ return this._server.xrpc.method(nsid, cfg)
992
+ }
993
+
994
+ transferAccount<AV extends AuthVerifier>(
995
+ cfg: ConfigOf<
996
+ AV,
997
+ ComAtprotoTempTransferAccount.Handler<ExtractAuth<AV>>,
998
+ ComAtprotoTempTransferAccount.HandlerReqCtx<ExtractAuth<AV>>
999
+ >,
1000
+ ) {
1001
+ const nsid = 'com.atproto.temp.transferAccount' // @ts-ignore
1002
+ return this._server.xrpc.method(nsid, cfg)
1003
+ }
956
1004
  }
957
1005
 
958
1006
  export class AppNS {
@@ -1549,11 +1597,13 @@ type RouteRateLimitOpts<T> = {
1549
1597
  calcKey?: (ctx: T) => string
1550
1598
  calcPoints?: (ctx: T) => number
1551
1599
  }
1600
+ type HandlerOpts = { blobLimit?: number }
1552
1601
  type HandlerRateLimitOpts<T> = SharedRateLimitOpts<T> | RouteRateLimitOpts<T>
1553
1602
  type ConfigOf<Auth, Handler, ReqCtx> =
1554
1603
  | Handler
1555
1604
  | {
1556
1605
  auth?: Auth
1606
+ opts?: HandlerOpts
1557
1607
  rateLimit?: HandlerRateLimitOpts<ReqCtx> | HandlerRateLimitOpts<ReqCtx>[]
1558
1608
  handler: Handler
1559
1609
  }