@atproto/bsky 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/api/com/atproto/moderation/util.d.ts +4 -3
  3. package/dist/cache/read-through.d.ts +30 -0
  4. package/dist/config.d.ts +18 -0
  5. package/dist/context.d.ts +21 -6
  6. package/dist/daemon/config.d.ts +15 -0
  7. package/dist/daemon/context.d.ts +15 -0
  8. package/dist/daemon/index.d.ts +23 -0
  9. package/dist/daemon/logger.d.ts +3 -0
  10. package/dist/daemon/notifications.d.ts +18 -0
  11. package/dist/daemon/services.d.ts +11 -0
  12. package/dist/db/database-schema.d.ts +1 -2
  13. package/dist/db/index.js +41 -1
  14. package/dist/db/index.js.map +3 -3
  15. package/dist/db/migrations/20231003T202833377Z-create-moderation-subject-status.d.ts +3 -0
  16. package/dist/db/migrations/20231205T000257238Z-remove-did-cache.d.ts +3 -0
  17. package/dist/db/migrations/index.d.ts +2 -0
  18. package/dist/db/pagination.d.ts +2 -1
  19. package/dist/db/{periodic-moderation-action-reversal.d.ts → periodic-moderation-event-reversal.d.ts} +3 -5
  20. package/dist/db/tables/moderation.d.ts +24 -34
  21. package/dist/did-cache.d.ts +10 -7
  22. package/dist/feed-gen/types.d.ts +1 -1
  23. package/dist/index.d.ts +6 -1
  24. package/dist/index.js +4370 -2758
  25. package/dist/index.js.map +3 -3
  26. package/dist/indexer/context.d.ts +2 -0
  27. package/dist/indexer/index.d.ts +1 -0
  28. package/dist/lexicon/index.d.ts +23 -18
  29. package/dist/lexicon/lexicons.d.ts +561 -412
  30. package/dist/lexicon/types/app/bsky/feed/defs.d.ts +1 -7
  31. package/dist/lexicon/types/app/bsky/graph/defs.d.ts +1 -0
  32. package/dist/lexicon/types/com/atproto/admin/defs.d.ts +114 -48
  33. package/dist/lexicon/types/com/atproto/admin/{resolveModerationReports.d.ts → deleteAccount.d.ts} +2 -13
  34. package/dist/lexicon/types/com/atproto/admin/{takeModerationAction.d.ts → emitModerationEvent.d.ts} +5 -6
  35. package/dist/lexicon/types/com/atproto/admin/{getModerationAction.d.ts → getModerationEvent.d.ts} +1 -1
  36. package/dist/lexicon/types/com/atproto/admin/{getModerationActions.d.ts → queryModerationEvents.d.ts} +5 -1
  37. package/dist/lexicon/types/com/atproto/admin/{getModerationReports.d.ts → queryModerationStatuses.d.ts} +12 -6
  38. package/dist/lexicon/types/com/atproto/admin/sendEmail.d.ts +1 -0
  39. package/dist/lexicon/types/com/atproto/{admin/getModerationReport.d.ts → temp/importRepo.d.ts} +10 -7
  40. package/dist/lexicon/types/com/atproto/temp/pushBlob.d.ts +25 -0
  41. package/dist/lexicon/types/com/atproto/{admin/reverseModerationAction.d.ts → temp/transferAccount.d.ts} +11 -5
  42. package/dist/logger.d.ts +1 -0
  43. package/dist/migrate-moderation-data.d.ts +1 -0
  44. package/dist/redis.d.ts +10 -1
  45. package/dist/services/actor/index.d.ts +18 -4
  46. package/dist/services/actor/views.d.ts +6 -8
  47. package/dist/services/feed/index.d.ts +7 -4
  48. package/dist/services/feed/util.d.ts +9 -1
  49. package/dist/services/feed/views.d.ts +11 -21
  50. package/dist/services/graph/index.d.ts +5 -29
  51. package/dist/services/graph/types.d.ts +1 -0
  52. package/dist/services/index.d.ts +3 -7
  53. package/dist/services/label/index.d.ts +10 -4
  54. package/dist/services/moderation/index.d.ts +134 -72
  55. package/dist/services/moderation/pagination.d.ts +36 -0
  56. package/dist/services/moderation/status.d.ts +13 -0
  57. package/dist/services/moderation/types.d.ts +35 -0
  58. package/dist/services/moderation/views.d.ts +18 -14
  59. package/dist/services/types.d.ts +3 -0
  60. package/dist/services/util/notification.d.ts +5 -0
  61. package/dist/services/util/post.d.ts +6 -6
  62. package/dist/util/debug.d.ts +1 -1
  63. package/dist/util/retry.d.ts +1 -6
  64. package/package.json +11 -11
  65. package/src/api/app/bsky/feed/getActorFeeds.ts +2 -1
  66. package/src/api/app/bsky/feed/getActorLikes.ts +1 -3
  67. package/src/api/app/bsky/feed/getAuthorFeed.ts +1 -3
  68. package/src/api/app/bsky/feed/getFeed.ts +9 -9
  69. package/src/api/app/bsky/feed/getFeedGenerator.ts +3 -0
  70. package/src/api/app/bsky/feed/getFeedGenerators.ts +2 -1
  71. package/src/api/app/bsky/feed/getListFeed.ts +1 -3
  72. package/src/api/app/bsky/feed/getPostThread.ts +15 -54
  73. package/src/api/app/bsky/feed/getPosts.ts +21 -18
  74. package/src/api/app/bsky/feed/getSuggestedFeeds.ts +2 -1
  75. package/src/api/app/bsky/feed/getTimeline.ts +1 -3
  76. package/src/api/app/bsky/feed/searchPosts.ts +20 -17
  77. package/src/api/app/bsky/graph/getList.ts +6 -3
  78. package/src/api/app/bsky/graph/getListBlocks.ts +3 -2
  79. package/src/api/app/bsky/graph/getListMutes.ts +2 -1
  80. package/src/api/app/bsky/graph/getLists.ts +2 -1
  81. package/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +3 -1
  82. package/src/api/blob-resolver.ts +6 -11
  83. package/src/api/com/atproto/admin/emitModerationEvent.ts +220 -0
  84. package/src/api/com/atproto/admin/{getModerationActions.ts → getModerationEvent.ts} +5 -11
  85. package/src/api/com/atproto/admin/getRecord.ts +1 -0
  86. package/src/api/com/atproto/admin/{getModerationReports.ts → queryModerationEvents.ts} +13 -16
  87. package/src/api/com/atproto/admin/queryModerationStatuses.ts +55 -0
  88. package/src/api/com/atproto/moderation/createReport.ts +9 -7
  89. package/src/api/com/atproto/moderation/util.ts +38 -20
  90. package/src/api/index.ts +8 -14
  91. package/src/auth.ts +29 -21
  92. package/src/auto-moderator/index.ts +26 -19
  93. package/src/cache/read-through.ts +151 -0
  94. package/src/config.ts +90 -1
  95. package/src/context.ts +11 -7
  96. package/src/daemon/config.ts +60 -0
  97. package/src/daemon/context.ts +27 -0
  98. package/src/daemon/index.ts +78 -0
  99. package/src/daemon/logger.ts +6 -0
  100. package/src/daemon/notifications.ts +54 -0
  101. package/src/daemon/services.ts +22 -0
  102. package/src/db/database-schema.ts +0 -2
  103. package/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts +123 -0
  104. package/src/db/migrations/20231205T000257238Z-remove-did-cache.ts +14 -0
  105. package/src/db/migrations/index.ts +2 -0
  106. package/src/db/pagination.ts +26 -3
  107. package/src/db/{periodic-moderation-action-reversal.ts → periodic-moderation-event-reversal.ts} +50 -46
  108. package/src/db/tables/moderation.ts +35 -52
  109. package/src/did-cache.ts +33 -56
  110. package/src/feed-gen/bsky-team.ts +1 -1
  111. package/src/feed-gen/hot-classic.ts +1 -1
  112. package/src/feed-gen/index.ts +0 -4
  113. package/src/feed-gen/mutuals.ts +6 -2
  114. package/src/feed-gen/types.ts +1 -1
  115. package/src/index.ts +57 -17
  116. package/src/indexer/context.ts +5 -0
  117. package/src/indexer/index.ts +10 -7
  118. package/src/lexicon/index.ts +80 -67
  119. package/src/lexicon/lexicons.ts +698 -507
  120. package/src/lexicon/types/app/bsky/feed/defs.ts +1 -18
  121. package/src/lexicon/types/app/bsky/graph/defs.ts +1 -0
  122. package/src/lexicon/types/com/atproto/admin/defs.ts +276 -84
  123. package/src/lexicon/types/com/atproto/admin/{resolveModerationReports.ts → deleteAccount.ts} +2 -13
  124. package/src/lexicon/types/com/atproto/admin/{takeModerationAction.ts → emitModerationEvent.ts} +13 -11
  125. package/src/lexicon/types/com/atproto/admin/{getModerationReport.ts → getModerationEvent.ts} +1 -1
  126. package/src/lexicon/types/com/atproto/admin/{getModerationActions.ts → queryModerationEvents.ts} +8 -1
  127. package/src/lexicon/types/com/atproto/admin/{getModerationReports.ts → queryModerationStatuses.ts} +21 -14
  128. package/src/lexicon/types/com/atproto/admin/sendEmail.ts +1 -0
  129. package/src/lexicon/types/com/atproto/{admin/getModerationAction.ts → temp/importRepo.ts} +11 -7
  130. package/src/lexicon/types/com/atproto/temp/pushBlob.ts +39 -0
  131. package/src/lexicon/types/com/atproto/{admin/reverseModerationAction.ts → temp/transferAccount.ts} +18 -5
  132. package/src/logger.ts +2 -0
  133. package/src/migrate-moderation-data.ts +414 -0
  134. package/src/redis.ts +43 -3
  135. package/src/services/actor/index.ts +55 -7
  136. package/src/services/actor/views.ts +18 -21
  137. package/src/services/feed/index.ts +52 -19
  138. package/src/services/feed/util.ts +47 -19
  139. package/src/services/feed/views.ts +87 -13
  140. package/src/services/graph/index.ts +21 -3
  141. package/src/services/graph/types.ts +1 -0
  142. package/src/services/index.ts +14 -14
  143. package/src/services/indexing/index.ts +7 -10
  144. package/src/services/indexing/plugins/block.ts +2 -3
  145. package/src/services/indexing/plugins/feed-generator.ts +2 -3
  146. package/src/services/indexing/plugins/follow.ts +2 -3
  147. package/src/services/indexing/plugins/like.ts +2 -3
  148. package/src/services/indexing/plugins/list-block.ts +2 -3
  149. package/src/services/indexing/plugins/list-item.ts +2 -3
  150. package/src/services/indexing/plugins/list.ts +2 -3
  151. package/src/services/indexing/plugins/post.ts +16 -4
  152. package/src/services/indexing/plugins/repost.ts +2 -3
  153. package/src/services/indexing/plugins/thread-gate.ts +2 -3
  154. package/src/services/label/index.ts +68 -25
  155. package/src/services/moderation/index.ts +380 -395
  156. package/src/services/moderation/pagination.ts +96 -0
  157. package/src/services/moderation/status.ts +241 -0
  158. package/src/services/moderation/types.ts +49 -0
  159. package/src/services/moderation/views.ts +278 -329
  160. package/src/services/types.ts +4 -0
  161. package/src/services/util/notification.ts +70 -0
  162. package/src/util/debug.ts +2 -2
  163. package/src/util/retry.ts +1 -44
  164. package/tests/__snapshots__/feed-generation.test.ts.snap +322 -6
  165. package/tests/__snapshots__/indexing.test.ts.snap +0 -6
  166. package/tests/admin/__snapshots__/get-record.test.ts.snap +30 -132
  167. package/tests/admin/__snapshots__/get-repo.test.ts.snap +14 -60
  168. package/tests/admin/__snapshots__/moderation-events.test.ts.snap +146 -0
  169. package/tests/admin/__snapshots__/moderation-statuses.test.ts.snap +64 -0
  170. package/tests/admin/__snapshots__/moderation.test.ts.snap +0 -125
  171. package/tests/admin/get-record.test.ts +5 -9
  172. package/tests/admin/get-repo.test.ts +10 -12
  173. package/tests/admin/moderation-events.test.ts +221 -0
  174. package/tests/admin/moderation-statuses.test.ts +145 -0
  175. package/tests/admin/moderation.test.ts +512 -860
  176. package/tests/admin/repo-search.test.ts +3 -3
  177. package/tests/algos/hot-classic.test.ts +1 -2
  178. package/tests/auth.test.ts +1 -1
  179. package/tests/auto-moderator/fuzzy-matcher.test.ts +2 -1
  180. package/tests/auto-moderator/labeler.test.ts +19 -20
  181. package/tests/auto-moderator/takedowns.test.ts +61 -28
  182. package/tests/blob-resolver.test.ts +4 -2
  183. package/tests/daemon.test.ts +191 -0
  184. package/tests/did-cache.test.ts +20 -5
  185. package/tests/feed-generation.test.ts +57 -9
  186. package/tests/handle-invalidation.test.ts +1 -5
  187. package/tests/indexing.test.ts +20 -13
  188. package/tests/redis-cache.test.ts +231 -0
  189. package/tests/seeds/basic.ts +3 -0
  190. package/tests/subscription/repo.test.ts +4 -7
  191. package/tests/views/__snapshots__/block-lists.test.ts.snap +3 -9
  192. package/tests/views/__snapshots__/blocks.test.ts.snap +0 -9
  193. package/tests/views/__snapshots__/mute-lists.test.ts.snap +5 -5
  194. package/tests/views/__snapshots__/mutes.test.ts.snap +0 -3
  195. package/tests/views/__snapshots__/thread.test.ts.snap +0 -30
  196. package/tests/views/actor-search.test.ts +2 -3
  197. package/tests/views/author-feed.test.ts +42 -36
  198. package/tests/views/follows.test.ts +40 -35
  199. package/tests/views/list-feed.test.ts +17 -9
  200. package/tests/views/notifications.test.ts +13 -9
  201. package/tests/views/profile.test.ts +20 -19
  202. package/tests/views/thread.test.ts +117 -94
  203. package/tests/views/threadgating.test.ts +89 -19
  204. package/tests/views/timeline.test.ts +21 -13
  205. package/dist/api/com/atproto/admin/resolveModerationReports.d.ts +0 -3
  206. package/dist/api/com/atproto/admin/reverseModerationAction.d.ts +0 -3
  207. package/dist/api/com/atproto/admin/takeModerationAction.d.ts +0 -3
  208. package/dist/db/tables/did-cache.d.ts +0 -10
  209. package/dist/feed-gen/best-of-follows.d.ts +0 -29
  210. package/dist/feed-gen/whats-hot.d.ts +0 -29
  211. package/dist/feed-gen/with-friends.d.ts +0 -3
  212. package/dist/label-cache.d.ts +0 -19
  213. package/src/api/com/atproto/admin/getModerationAction.ts +0 -44
  214. package/src/api/com/atproto/admin/getModerationReport.ts +0 -43
  215. package/src/api/com/atproto/admin/resolveModerationReports.ts +0 -24
  216. package/src/api/com/atproto/admin/reverseModerationAction.ts +0 -115
  217. package/src/api/com/atproto/admin/takeModerationAction.ts +0 -156
  218. package/src/db/tables/did-cache.ts +0 -13
  219. package/src/feed-gen/best-of-follows.ts +0 -74
  220. package/src/feed-gen/whats-hot.ts +0 -101
  221. package/src/feed-gen/with-friends.ts +0 -39
  222. package/src/label-cache.ts +0 -90
  223. package/tests/admin/__snapshots__/get-moderation-action.test.ts.snap +0 -172
  224. package/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap +0 -178
  225. package/tests/admin/__snapshots__/get-moderation-report.test.ts.snap +0 -177
  226. package/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap +0 -307
  227. package/tests/admin/get-moderation-action.test.ts +0 -100
  228. package/tests/admin/get-moderation-actions.test.ts +0 -164
  229. package/tests/admin/get-moderation-report.test.ts +0 -100
  230. package/tests/admin/get-moderation-reports.test.ts +0 -332
  231. package/tests/algos/whats-hot.test.ts +0 -118
  232. package/tests/algos/with-friends.test.ts +0 -145
  233. /package/dist/api/com/atproto/admin/{getModerationAction.d.ts → emitModerationEvent.d.ts} +0 -0
  234. /package/dist/api/com/atproto/admin/{getModerationActions.d.ts → getModerationEvent.d.ts} +0 -0
  235. /package/dist/api/com/atproto/admin/{getModerationReport.d.ts → queryModerationEvents.d.ts} +0 -0
  236. /package/dist/api/com/atproto/admin/{getModerationReports.d.ts → queryModerationStatuses.d.ts} +0 -0
@@ -1,6 +1,5 @@
1
1
  import { SeedClient, TestNetwork } from '@atproto/dev-env'
2
2
  import AtpAgent from '@atproto/api'
3
- import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs'
4
3
  import { paginateAll } from '../_util'
5
4
  import usersBulkSeed from '../seeds/users-bulk'
6
5
 
@@ -18,6 +17,7 @@ describe('admin repo search view', () => {
18
17
  sc = network.getSeedClient()
19
18
  await usersBulkSeed(sc)
20
19
  headers = network.pds.adminAuthHeaders()
20
+ await network.processAll()
21
21
  })
22
22
 
23
23
  afterAll(async () => {
@@ -25,8 +25,8 @@ describe('admin repo search view', () => {
25
25
  })
26
26
 
27
27
  beforeAll(async () => {
28
- await sc.takeModerationAction({
29
- action: TAKEDOWN,
28
+ await sc.emitModerationEvent({
29
+ event: { $type: 'com.atproto.admin.defs#modEventTakedown' },
30
30
  subject: {
31
31
  $type: 'com.atproto.admin.defs#repoRef',
32
32
  did: sc.dids['cara-wiegand69.test'],
@@ -31,7 +31,6 @@ describe('algo hot-classic', () => {
31
31
  alice = sc.dids.alice
32
32
  bob = sc.dids.bob
33
33
  await network.processAll()
34
- await network.bsky.processAll()
35
34
  })
36
35
 
37
36
  afterAll(async () => {
@@ -59,7 +58,7 @@ describe('algo hot-classic', () => {
59
58
  await sc.like(sc.dids[name], two.ref)
60
59
  await sc.like(sc.dids[name], three.ref)
61
60
  }
62
- await network.bsky.processAll()
61
+ await network.processAll()
63
62
 
64
63
  const res = await agent.api.app.bsky.feed.getFeed(
65
64
  { feed: feedUri },
@@ -36,7 +36,7 @@ describe('auth', () => {
36
36
  { headers: { authorization: `Bearer ${jwt}` } },
37
37
  )
38
38
  }
39
- const origSigningKey = network.pds.ctx.repoSigningKey
39
+ const origSigningKey = await network.pds.ctx.actorStore.keypair(issuer)
40
40
  const newSigningKey = await Secp256k1Keypair.create({ exportable: true })
41
41
  // confirm original signing key works
42
42
  await expect(attemptWithKey(origSigningKey)).resolves.toBeDefined()
@@ -37,7 +37,8 @@ describe('fuzzy matcher', () => {
37
37
  const getAllReports = () => {
38
38
  return network.bsky.ctx.db
39
39
  .getPrimary()
40
- .db.selectFrom('moderation_report')
40
+ .db.selectFrom('moderation_event')
41
+ .where('action', '=', 'com.atproto.admin.defs#modEventReport')
41
42
  .selectAll()
42
43
  .orderBy('id', 'asc')
43
44
  .execute()
@@ -40,26 +40,25 @@ describe('labeler', () => {
40
40
  await usersSeed(sc)
41
41
  await network.processAll()
42
42
  alice = sc.dids.alice
43
- const repoSvc = pdsCtx.services.repo(pdsCtx.db)
44
- const storeBlob = async (bytes: Uint8Array) => {
45
- const blobRef = await repoSvc.blobs.addUntetheredBlob(
46
- alice,
47
- 'image/jpeg',
48
- Readable.from([bytes], { objectMode: false }),
49
- )
50
- const preparedBlobRef = {
51
- cid: blobRef.ref,
52
- mimeType: 'image/jpeg',
53
- constraints: {},
54
- }
55
- await repoSvc.blobs.verifyBlobAndMakePermanent(alice, preparedBlobRef)
56
- await repoSvc.blobs.associateBlob(
57
- preparedBlobRef,
58
- postUri(),
59
- TID.nextStr(),
60
- alice,
61
- )
62
- return blobRef
43
+ const storeBlob = (bytes: Uint8Array) => {
44
+ return pdsCtx.actorStore.transact(alice, async (store) => {
45
+ const blobRef = await store.repo.blob.addUntetheredBlob(
46
+ 'image/jpeg',
47
+ Readable.from([bytes], { objectMode: false }),
48
+ )
49
+ const preparedBlobRef = {
50
+ cid: blobRef.ref,
51
+ mimeType: 'image/jpeg',
52
+ constraints: {},
53
+ }
54
+ await store.repo.blob.verifyBlobAndMakePermanent(preparedBlobRef)
55
+ await store.repo.blob.associateBlob(
56
+ preparedBlobRef,
57
+ postUri(),
58
+ TID.nextStr(),
59
+ )
60
+ return blobRef
61
+ })
63
62
  }
64
63
  const bytes1 = new Uint8Array([1, 2, 3, 4])
65
64
  const bytes2 = new Uint8Array([5, 6, 7, 8])
@@ -77,28 +77,45 @@ describe('takedowner', () => {
77
77
  const post = await sc.post(alice, 'blah', undefined, [goodBlob, badBlob1])
78
78
  await network.processAll()
79
79
  await autoMod.processAll()
80
- const modAction = await ctx.db.db
81
- .selectFrom('moderation_action')
82
- .where('subjectUri', '=', post.ref.uriStr)
83
- .select(['action', 'id'])
84
- .executeTakeFirst()
85
- if (!modAction) {
80
+ const [modStatus, takedownEvent] = await Promise.all([
81
+ ctx.db.db
82
+ .selectFrom('moderation_subject_status')
83
+ .where('did', '=', alice)
84
+ .where(
85
+ 'recordPath',
86
+ '=',
87
+ `${post.ref.uri.collection}/${post.ref.uri.rkey}`,
88
+ )
89
+ .select(['takendown', 'id'])
90
+ .executeTakeFirst(),
91
+ ctx.db.db
92
+ .selectFrom('moderation_event')
93
+ .where('subjectDid', '=', alice)
94
+ .where('action', '=', 'com.atproto.admin.defs#modEventTakedown')
95
+ .selectAll()
96
+ .executeTakeFirst(),
97
+ ])
98
+ if (!modStatus || !takedownEvent) {
86
99
  throw new Error('expected mod action')
87
100
  }
88
- expect(modAction.action).toEqual('com.atproto.admin.defs#takedown')
101
+ expect(modStatus.takendown).toEqual(true)
89
102
  const record = await ctx.db.db
90
103
  .selectFrom('record')
91
104
  .where('uri', '=', post.ref.uriStr)
92
105
  .select('takedownId')
93
106
  .executeTakeFirst()
94
- expect(record?.takedownId).toEqual(modAction.id)
107
+ expect(record?.takedownId).toBeGreaterThan(0)
95
108
 
96
- const recordPds = await network.pds.ctx.db.db
97
- .selectFrom('record')
98
- .where('uri', '=', post.ref.uriStr)
99
- .select('takedownRef')
100
- .executeTakeFirst()
101
- expect(recordPds?.takedownRef).toEqual(modAction.id.toString())
109
+ const recordPds = await network.pds.ctx.actorStore.read(
110
+ post.ref.uri.hostname,
111
+ (store) =>
112
+ store.db.db
113
+ .selectFrom('record')
114
+ .where('uri', '=', post.ref.uriStr)
115
+ .select('takedownRef')
116
+ .executeTakeFirst(),
117
+ )
118
+ expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString())
102
119
 
103
120
  expect(testInvalidator.invalidated.length).toBe(1)
104
121
  expect(testInvalidator.invalidated[0].subject).toBe(
@@ -119,28 +136,44 @@ describe('takedowner', () => {
119
136
  { headers: sc.getHeaders(alice), encoding: 'application/json' },
120
137
  )
121
138
  await network.processAll()
122
- const modAction = await ctx.db.db
123
- .selectFrom('moderation_action')
124
- .where('subjectUri', '=', res.data.uri)
125
- .select(['action', 'id'])
126
- .executeTakeFirst()
127
- if (!modAction) {
139
+ const [modStatus, takedownEvent] = await Promise.all([
140
+ ctx.db.db
141
+ .selectFrom('moderation_subject_status')
142
+ .where('did', '=', alice)
143
+ .where('recordPath', '=', `${ids.AppBskyActorProfile}/self`)
144
+ .select(['takendown', 'id'])
145
+ .executeTakeFirst(),
146
+ ctx.db.db
147
+ .selectFrom('moderation_event')
148
+ .where('subjectDid', '=', alice)
149
+ .where(
150
+ 'subjectUri',
151
+ '=',
152
+ AtUri.make(alice, ids.AppBskyActorProfile, 'self').toString(),
153
+ )
154
+ .where('action', '=', 'com.atproto.admin.defs#modEventTakedown')
155
+ .selectAll()
156
+ .executeTakeFirst(),
157
+ ])
158
+ if (!modStatus || !takedownEvent) {
128
159
  throw new Error('expected mod action')
129
160
  }
130
- expect(modAction.action).toEqual('com.atproto.admin.defs#takedown')
161
+ expect(modStatus.takendown).toEqual(true)
131
162
  const record = await ctx.db.db
132
163
  .selectFrom('record')
133
164
  .where('uri', '=', res.data.uri)
134
165
  .select('takedownId')
135
166
  .executeTakeFirst()
136
- expect(record?.takedownId).toEqual(modAction.id)
167
+ expect(record?.takedownId).toBeGreaterThan(0)
137
168
 
138
- const recordPds = await network.pds.ctx.db.db
139
- .selectFrom('record')
140
- .where('uri', '=', res.data.uri)
141
- .select('takedownRef')
142
- .executeTakeFirst()
143
- expect(recordPds?.takedownRef).toEqual(modAction.id.toString())
169
+ const recordPds = await network.pds.ctx.actorStore.read(alice, (store) =>
170
+ store.db.db
171
+ .selectFrom('record')
172
+ .where('uri', '=', res.data.uri)
173
+ .select('takedownRef')
174
+ .executeTakeFirst(),
175
+ )
176
+ expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString())
144
177
 
145
178
  expect(testInvalidator.invalidated.length).toBe(2)
146
179
  expect(testInvalidator.invalidated[1].subject).toBe(
@@ -77,8 +77,10 @@ describe('blob resolver', () => {
77
77
  })
78
78
 
79
79
  it('fails on blob with bad signature check.', async () => {
80
- await network.pds.ctx.blobstore.delete(fileCid)
81
- await network.pds.ctx.blobstore.putPermanent(fileCid, randomBytes(100))
80
+ await network.pds.ctx.blobstore(fileDid).delete(fileCid)
81
+ await network.pds.ctx
82
+ .blobstore(fileDid)
83
+ .putPermanent(fileCid, randomBytes(100))
82
84
  const tryGetBlob = client.get(`/blob/${fileDid}/${fileCid.toString()}`)
83
85
  await expect(tryGetBlob).rejects.toThrow(
84
86
  'maxContentLength size of -1 exceeded',
@@ -0,0 +1,191 @@
1
+ import assert from 'assert'
2
+ import { AtUri } from '@atproto/api'
3
+ import { TestNetwork } from '@atproto/dev-env'
4
+ import { BskyDaemon, DaemonConfig, PrimaryDatabase } from '../src'
5
+ import usersSeed from './seeds/users'
6
+ import { countAll, excluded } from '../src/db/util'
7
+ import { NotificationsDaemon } from '../src/daemon/notifications'
8
+ import {
9
+ BEFORE_LAST_SEEN_DAYS,
10
+ BEFORE_LATEST_UNREAD_DAYS,
11
+ UNREAD_KEPT_COUNT,
12
+ } from '../src/services/util/notification'
13
+
14
+ describe('daemon', () => {
15
+ let network: TestNetwork
16
+ let daemon: BskyDaemon
17
+ let db: PrimaryDatabase
18
+ let actors: { did: string }[] = []
19
+
20
+ beforeAll(async () => {
21
+ network = await TestNetwork.create({
22
+ dbPostgresSchema: 'bsky_daemon',
23
+ })
24
+ db = network.bsky.ctx.db.getPrimary()
25
+ daemon = BskyDaemon.create({
26
+ db,
27
+ cfg: new DaemonConfig({
28
+ version: network.bsky.ctx.cfg.version,
29
+ dbPostgresUrl: network.bsky.ctx.cfg.dbPrimaryPostgresUrl,
30
+ dbPostgresSchema: network.bsky.ctx.cfg.dbPostgresSchema,
31
+ }),
32
+ })
33
+ const sc = network.getSeedClient()
34
+ await usersSeed(sc)
35
+ await network.processAll()
36
+ actors = await db.db.selectFrom('actor').selectAll().execute()
37
+ })
38
+
39
+ afterAll(async () => {
40
+ await network.close()
41
+ })
42
+
43
+ describe('notifications daemon', () => {
44
+ it('processes all dids', async () => {
45
+ for (const { did } of actors) {
46
+ await Promise.all([
47
+ setLastSeen(daemon.ctx.db, { did }),
48
+ createNotifications(daemon.ctx.db, {
49
+ did,
50
+ daysAgo: 2 * BEFORE_LAST_SEEN_DAYS,
51
+ count: 1,
52
+ }),
53
+ ])
54
+ }
55
+ await expect(countNotifications(db)).resolves.toBe(actors.length)
56
+ await runNotifsOnce(daemon.notifications)
57
+ await expect(countNotifications(db)).resolves.toBe(0)
58
+ })
59
+
60
+ it('removes read notifications older than threshold.', async () => {
61
+ const { did } = actors[0]
62
+ const lastSeenDaysAgo = 10
63
+ await Promise.all([
64
+ setLastSeen(daemon.ctx.db, { did, daysAgo: lastSeenDaysAgo }),
65
+ // read, delete
66
+ createNotifications(daemon.ctx.db, {
67
+ did,
68
+ daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS + 1,
69
+ count: 2,
70
+ }),
71
+ // read, keep
72
+ createNotifications(daemon.ctx.db, {
73
+ did,
74
+ daysAgo: lastSeenDaysAgo + BEFORE_LAST_SEEN_DAYS - 1,
75
+ count: 3,
76
+ }),
77
+ // unread, keep
78
+ createNotifications(daemon.ctx.db, {
79
+ did,
80
+ daysAgo: lastSeenDaysAgo - 1,
81
+ count: 4,
82
+ }),
83
+ ])
84
+ await expect(countNotifications(db)).resolves.toBe(9)
85
+ await runNotifsOnce(daemon.notifications)
86
+ await expect(countNotifications(db)).resolves.toBe(7)
87
+ await clearNotifications(db)
88
+ })
89
+
90
+ it('removes unread notifications older than threshold.', async () => {
91
+ const { did } = actors[0]
92
+ await Promise.all([
93
+ setLastSeen(daemon.ctx.db, {
94
+ did,
95
+ daysAgo: 2 * BEFORE_LATEST_UNREAD_DAYS, // all are unread
96
+ }),
97
+ createNotifications(daemon.ctx.db, {
98
+ did,
99
+ daysAgo: 0,
100
+ count: 1,
101
+ }),
102
+ createNotifications(daemon.ctx.db, {
103
+ did,
104
+ daysAgo: BEFORE_LATEST_UNREAD_DAYS - 1,
105
+ count: 99,
106
+ }),
107
+ createNotifications(daemon.ctx.db, {
108
+ did,
109
+ daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1,
110
+ count: 400,
111
+ }),
112
+ ])
113
+ await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT)
114
+ await runNotifsOnce(daemon.notifications)
115
+ // none removed when within UNREAD_KEPT_COUNT
116
+ await expect(countNotifications(db)).resolves.toBe(UNREAD_KEPT_COUNT)
117
+ // add one more, tip over UNREAD_KEPT_COUNT
118
+ await createNotifications(daemon.ctx.db, {
119
+ did,
120
+ daysAgo: BEFORE_LATEST_UNREAD_DAYS + 1,
121
+ count: 1,
122
+ })
123
+ await runNotifsOnce(daemon.notifications)
124
+ // removed all older than BEFORE_LATEST_UNREAD_DAYS
125
+ await expect(countNotifications(db)).resolves.toBe(100)
126
+ await clearNotifications(db)
127
+ })
128
+ })
129
+
130
+ const runNotifsOnce = async (notifsDaemon: NotificationsDaemon) => {
131
+ assert(!notifsDaemon.running, 'notifications daemon is already running')
132
+ notifsDaemon.run({ forever: false, batchSize: 2 })
133
+ await notifsDaemon.running
134
+ }
135
+
136
+ const setLastSeen = async (
137
+ db: PrimaryDatabase,
138
+ opts: { did: string; daysAgo?: number },
139
+ ) => {
140
+ const { did, daysAgo = 0 } = opts
141
+ const lastSeenAt = new Date()
142
+ lastSeenAt.setDate(lastSeenAt.getDate() - daysAgo)
143
+ await db.db
144
+ .insertInto('actor_state')
145
+ .values({ did, lastSeenNotifs: lastSeenAt.toISOString() })
146
+ .onConflict((oc) =>
147
+ oc.column('did').doUpdateSet({
148
+ lastSeenNotifs: excluded(db.db, 'lastSeenNotifs'),
149
+ }),
150
+ )
151
+ .execute()
152
+ }
153
+
154
+ const createNotifications = async (
155
+ db: PrimaryDatabase,
156
+ opts: {
157
+ did: string
158
+ count: number
159
+ daysAgo: number
160
+ },
161
+ ) => {
162
+ const { did, count, daysAgo } = opts
163
+ const sortAt = new Date()
164
+ sortAt.setDate(sortAt.getDate() - daysAgo)
165
+ await db.db
166
+ .insertInto('notification')
167
+ .values(
168
+ [...Array(count)].map(() => ({
169
+ did,
170
+ author: did,
171
+ reason: 'none',
172
+ recordCid: 'bafycid',
173
+ recordUri: AtUri.make(did, 'invalid.collection', 'self').toString(),
174
+ sortAt: sortAt.toISOString(),
175
+ })),
176
+ )
177
+ .execute()
178
+ }
179
+
180
+ const clearNotifications = async (db: PrimaryDatabase) => {
181
+ await db.db.deleteFrom('notification').execute()
182
+ }
183
+
184
+ const countNotifications = async (db: PrimaryDatabase) => {
185
+ const { count } = await db.db
186
+ .selectFrom('notification')
187
+ .select(countAll.as('count'))
188
+ .executeTakeFirstOrThrow()
189
+ return count
190
+ }
191
+ })
@@ -1,14 +1,16 @@
1
1
  import { TestNetwork, SeedClient } from '@atproto/dev-env'
2
2
  import userSeed from './seeds/users'
3
3
  import { IdResolver } from '@atproto/identity'
4
- import DidSqlCache from '../src/did-cache'
4
+ import DidRedisCache from '../src/did-cache'
5
5
  import { wait } from '@atproto/common'
6
+ import { Redis } from '../src'
6
7
 
7
8
  describe('did cache', () => {
8
9
  let network: TestNetwork
9
10
  let sc: SeedClient
10
11
  let idResolver: IdResolver
11
- let didCache: DidSqlCache
12
+ let redis: Redis
13
+ let didCache: DidRedisCache
12
14
 
13
15
  let alice: string
14
16
  let bob: string
@@ -20,6 +22,7 @@ describe('did cache', () => {
20
22
  dbPostgresSchema: 'bsky_did_cache',
21
23
  })
22
24
  idResolver = network.bsky.indexer.ctx.idResolver
25
+ redis = network.bsky.indexer.ctx.redis
23
26
  didCache = network.bsky.indexer.ctx.didCache
24
27
  sc = network.getSeedClient()
25
28
  await userSeed(sc)
@@ -50,7 +53,12 @@ describe('did cache', () => {
50
53
  })
51
54
 
52
55
  it('clears cache and repopulates', async () => {
53
- await idResolver.did.cache?.clear()
56
+ await Promise.all([
57
+ idResolver.did.cache?.clearEntry(alice),
58
+ idResolver.did.cache?.clearEntry(bob),
59
+ idResolver.did.cache?.clearEntry(carol),
60
+ idResolver.did.cache?.clearEntry(dan),
61
+ ])
54
62
  const docsCleared = await Promise.all([
55
63
  idResolver.did.cache?.checkCache(alice),
56
64
  idResolver.did.cache?.checkCache(bob),
@@ -81,7 +89,10 @@ describe('did cache', () => {
81
89
  })
82
90
 
83
91
  it('accurately reports expired dids & refreshes the cache', async () => {
84
- const didCache = new DidSqlCache(network.bsky.ctx.db.getPrimary(), 1, 60000)
92
+ const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
93
+ staleTTL: 1,
94
+ maxTTL: 60000,
95
+ })
85
96
  const shortCacheResolver = new IdResolver({
86
97
  plcUrl: network.bsky.ctx.cfg.didPlcUrl,
87
98
  didCache,
@@ -110,7 +121,10 @@ describe('did cache', () => {
110
121
  })
111
122
 
112
123
  it('does not return expired dids & refreshes the cache', async () => {
113
- const didCache = new DidSqlCache(network.bsky.ctx.db.getPrimary(), 0, 1)
124
+ const didCache = new DidRedisCache(redis.withNamespace('did-doc'), {
125
+ staleTTL: 0,
126
+ maxTTL: 1,
127
+ })
114
128
  const shortExpireResolver = new IdResolver({
115
129
  plcUrl: network.bsky.ctx.cfg.didPlcUrl,
116
130
  didCache,
@@ -125,5 +139,6 @@ describe('did cache', () => {
125
139
  // see that the resolver does not return expired value & instead force refreshes
126
140
  const staleGet = await shortExpireResolver.did.resolve(alice)
127
141
  expect(staleGet?.id).toEqual(alice)
142
+ await didCache.destroy()
128
143
  })
129
144
  })
@@ -9,7 +9,6 @@ import {
9
9
  import { Handler as SkeletonHandler } from '../src/lexicon/types/app/bsky/feed/getFeedSkeleton'
10
10
  import { GeneratorView } from '@atproto/api/src/client/types/app/bsky/feed/defs'
11
11
  import { UnknownFeedError } from '@atproto/api/src/client/types/app/bsky/feed/getFeed'
12
- import { TAKEDOWN } from '@atproto/api/src/client/types/com/atproto/admin/defs'
13
12
  import { ids } from '../src/lexicon/lexicons'
14
13
  import {
15
14
  FeedViewPost,
@@ -17,6 +16,9 @@ import {
17
16
  } from '../src/lexicon/types/app/bsky/feed/defs'
18
17
  import basicSeed from './seeds/basic'
19
18
  import { forSnapshot, paginateAll } from './_util'
19
+ import { AuthRequiredError } from '@atproto/xrpc-server'
20
+ import assert from 'assert'
21
+ import { XRPCError } from '@atproto/xrpc'
20
22
 
21
23
  describe('feed generation', () => {
22
24
  let network: TestNetwork
@@ -33,6 +35,7 @@ describe('feed generation', () => {
33
35
  let feedUriBadPagination: string
34
36
  let feedUriPrime: string // Taken-down
35
37
  let feedUriPrimeRef: RecordRef
38
+ let feedUriNeedsAuth: string
36
39
 
37
40
  beforeAll(async () => {
38
41
  network = await TestNetwork.create({
@@ -52,11 +55,17 @@ describe('feed generation', () => {
52
55
  )
53
56
  const evenUri = AtUri.make(alice, 'app.bsky.feed.generator', 'even')
54
57
  const primeUri = AtUri.make(alice, 'app.bsky.feed.generator', 'prime')
58
+ const needsAuthUri = AtUri.make(
59
+ alice,
60
+ 'app.bsky.feed.generator',
61
+ 'needs-auth',
62
+ )
55
63
  gen = await network.createFeedGen({
56
64
  [allUri.toString()]: feedGenHandler('all'),
57
65
  [evenUri.toString()]: feedGenHandler('even'),
58
66
  [feedUriBadPagination.toString()]: feedGenHandler('bad-pagination'),
59
67
  [primeUri.toString()]: feedGenHandler('prime'),
68
+ [needsAuthUri.toString()]: feedGenHandler('needs-auth'),
60
69
  })
61
70
 
62
71
  const feedSuggestions = [
@@ -137,10 +146,20 @@ describe('feed generation', () => {
137
146
  },
138
147
  sc.getHeaders(alice),
139
148
  )
149
+ const needsAuth = await pdsAgent.api.app.bsky.feed.generator.create(
150
+ { repo: alice, rkey: 'needs-auth' },
151
+ {
152
+ did: gen.did,
153
+ displayName: 'Needs Auth',
154
+ description: 'Provides all feed candidates when authed',
155
+ createdAt: new Date().toISOString(),
156
+ },
157
+ sc.getHeaders(alice),
158
+ )
140
159
  await network.processAll()
141
- await agent.api.com.atproto.admin.takeModerationAction(
160
+ await agent.api.com.atproto.admin.emitModerationEvent(
142
161
  {
143
- action: TAKEDOWN,
162
+ event: { $type: 'com.atproto.admin.defs#modEventTakedown' },
144
163
  subject: {
145
164
  $type: 'com.atproto.repo.strongRef',
146
165
  uri: prime.uri,
@@ -161,6 +180,7 @@ describe('feed generation', () => {
161
180
  feedUriBadPagination = badPagination.uri
162
181
  feedUriPrime = prime.uri
163
182
  feedUriPrimeRef = new RecordRef(prime.uri, prime.cid)
183
+ feedUriNeedsAuth = needsAuth.uri
164
184
  })
165
185
 
166
186
  it('feed gen records can be updated', async () => {
@@ -198,11 +218,12 @@ describe('feed generation', () => {
198
218
 
199
219
  const paginatedAll: GeneratorView[] = results(await paginateAll(paginator))
200
220
 
201
- expect(paginatedAll.length).toEqual(4)
221
+ expect(paginatedAll.length).toEqual(5)
202
222
  expect(paginatedAll[0].uri).toEqual(feedUriOdd)
203
- expect(paginatedAll[1].uri).toEqual(feedUriBadPagination)
204
- expect(paginatedAll[2].uri).toEqual(feedUriEven)
205
- expect(paginatedAll[3].uri).toEqual(feedUriAll)
223
+ expect(paginatedAll[1].uri).toEqual(feedUriNeedsAuth)
224
+ expect(paginatedAll[2].uri).toEqual(feedUriBadPagination)
225
+ expect(paginatedAll[3].uri).toEqual(feedUriEven)
226
+ expect(paginatedAll[4].uri).toEqual(feedUriAll)
206
227
  expect(paginatedAll.map((fg) => fg.uri)).not.toContain(feedUriPrime) // taken-down
207
228
  expect(forSnapshot(paginatedAll)).toMatchSnapshot()
208
229
  })
@@ -348,7 +369,9 @@ describe('feed generation', () => {
348
369
  {},
349
370
  { headers: await network.serviceHeaders(sc.dids.bob) },
350
371
  )
351
- expect(resEven.data.feeds.map((f) => f.likeCount)).toEqual([2, 0, 0, 0])
372
+ expect(resEven.data.feeds.map((f) => f.likeCount)).toEqual([
373
+ 2, 0, 0, 0, 0,
374
+ ])
352
375
  expect(resEven.data.feeds.map((f) => f.uri)).not.toContain(feedUriPrime) // taken-down
353
376
  })
354
377
 
@@ -389,6 +412,16 @@ describe('feed generation', () => {
389
412
  expect(forSnapshot(feed.data.feed)).toMatchSnapshot()
390
413
  })
391
414
 
415
+ it('resolves basic feed contents without auth.', async () => {
416
+ const feed = await agent.api.app.bsky.feed.getFeed({ feed: feedUriEven })
417
+ expect(feed.data.feed.map((item) => item.post.uri)).toEqual([
418
+ sc.posts[sc.dids.alice][0].ref.uriStr,
419
+ sc.posts[sc.dids.carol][0].ref.uriStr,
420
+ sc.replies[sc.dids.carol][0].ref.uriStr,
421
+ ])
422
+ expect(forSnapshot(feed.data.feed)).toMatchSnapshot()
423
+ })
424
+
392
425
  it('paginates, handling replies and reposts.', async () => {
393
426
  const results = (results) => results.flatMap((res) => res.feed)
394
427
  const paginator = async (cursor?: string) => {
@@ -461,6 +494,16 @@ describe('feed generation', () => {
461
494
  expect(feed.data['$auth']?.['iss']).toEqual(alice)
462
495
  })
463
496
 
497
+ it('passes through auth error from feed.', async () => {
498
+ const tryGetFeed = agent.api.app.bsky.feed.getFeed({
499
+ feed: feedUriNeedsAuth,
500
+ })
501
+ const err = await tryGetFeed.catch((err) => err)
502
+ assert(err instanceof XRPCError)
503
+ expect(err.status).toBe(401)
504
+ expect(err.message).toBe('This feed requires auth')
505
+ })
506
+
464
507
  it('provides timing info in server-timing header.', async () => {
465
508
  const result = await agent.api.app.bsky.feed.getFeed(
466
509
  { feed: feedUriEven },
@@ -482,8 +525,13 @@ describe('feed generation', () => {
482
525
  })
483
526
 
484
527
  const feedGenHandler =
485
- (feedName: 'even' | 'all' | 'prime' | 'bad-pagination'): SkeletonHandler =>
528
+ (
529
+ feedName: 'even' | 'all' | 'prime' | 'bad-pagination' | 'needs-auth',
530
+ ): SkeletonHandler =>
486
531
  async ({ req, params }) => {
532
+ if (feedName === 'needs-auth' && !req.headers.authorization) {
533
+ throw new AuthRequiredError('This feed requires auth')
534
+ }
487
535
  const { limit, cursor } = params
488
536
  const candidates: SkeletonFeedPost[] = [
489
537
  { post: sc.posts[sc.dids.alice][0].ref.uriStr },
@@ -102,11 +102,7 @@ describe('handle invalidation', () => {
102
102
  it('deals with handle contention', async () => {
103
103
  await backdateIndexedAt(bob)
104
104
  // update alices handle so that the pds will let bob take her old handle
105
- await network.pds.ctx.db.db
106
- .updateTable('did_handle')
107
- .where('did', '=', alice)
108
- .set({ handle: 'not-alice.test' })
109
- .execute()
105
+ await network.pds.ctx.accountManager.updateHandle(alice, 'not-alice.test')
110
106
 
111
107
  await pdsAgent.api.com.atproto.identity.updateHandle(
112
108
  {