@atproto/bsky 0.0.14 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/api/app/bsky/feed/searchPosts.d.ts +3 -0
  3. package/dist/api/com/atproto/moderation/util.d.ts +4 -3
  4. package/dist/config.d.ts +2 -2
  5. package/dist/context.d.ts +16 -1
  6. package/dist/db/index.js +26 -1
  7. package/dist/db/index.js.map +3 -3
  8. package/dist/db/migrations/20231003T202833377Z-create-moderation-subject-status.d.ts +3 -0
  9. package/dist/db/migrations/index.d.ts +1 -0
  10. package/dist/db/pagination.d.ts +2 -1
  11. package/dist/db/{periodic-moderation-action-reversal.d.ts → periodic-moderation-event-reversal.d.ts} +3 -5
  12. package/dist/db/tables/moderation.d.ts +24 -34
  13. package/dist/feed-gen/types.d.ts +1 -1
  14. package/dist/index.d.ts +2 -1
  15. package/dist/index.js +3332 -2430
  16. package/dist/index.js.map +3 -3
  17. package/dist/lexicon/index.d.ts +18 -18
  18. package/dist/lexicon/lexicons.d.ts +460 -385
  19. package/dist/lexicon/types/app/bsky/feed/defs.d.ts +1 -7
  20. package/dist/lexicon/types/app/bsky/graph/defs.d.ts +1 -0
  21. package/dist/lexicon/types/com/atproto/admin/defs.d.ts +116 -48
  22. package/dist/lexicon/types/com/atproto/admin/{takeModerationAction.d.ts → emitModerationEvent.d.ts} +5 -6
  23. package/dist/lexicon/types/com/atproto/admin/{getModerationAction.d.ts → getModerationEvent.d.ts} +1 -1
  24. package/dist/lexicon/types/com/atproto/admin/{getModerationActions.d.ts → queryModerationEvents.d.ts} +5 -1
  25. package/dist/lexicon/types/com/atproto/admin/{getModerationReports.d.ts → queryModerationStatuses.d.ts} +12 -6
  26. package/dist/lexicon/types/com/atproto/admin/sendEmail.d.ts +1 -0
  27. package/dist/lexicon/types/com/atproto/{admin/getModerationReport.d.ts → temp/fetchLabels.d.ts} +7 -3
  28. package/dist/migrate-moderation-data.d.ts +1 -0
  29. package/dist/services/actor/views.d.ts +2 -5
  30. package/dist/services/feed/index.d.ts +1 -0
  31. package/dist/services/feed/util.d.ts +9 -1
  32. package/dist/services/feed/views.d.ts +6 -17
  33. package/dist/services/graph/index.d.ts +5 -29
  34. package/dist/services/graph/types.d.ts +1 -0
  35. package/dist/services/moderation/index.d.ts +135 -72
  36. package/dist/services/moderation/pagination.d.ts +36 -0
  37. package/dist/services/moderation/status.d.ts +13 -0
  38. package/dist/services/moderation/types.d.ts +35 -0
  39. package/dist/services/moderation/views.d.ts +18 -14
  40. package/dist/util/debug.d.ts +1 -1
  41. package/package.json +14 -15
  42. package/src/api/app/bsky/actor/getSuggestions.ts +45 -21
  43. package/src/api/app/bsky/feed/getActorFeeds.ts +2 -1
  44. package/src/api/app/bsky/feed/getActorLikes.ts +1 -3
  45. package/src/api/app/bsky/feed/getAuthorFeed.ts +1 -3
  46. package/src/api/app/bsky/feed/getFeed.ts +9 -9
  47. package/src/api/app/bsky/feed/getFeedGenerator.ts +3 -0
  48. package/src/api/app/bsky/feed/getFeedGenerators.ts +2 -1
  49. package/src/api/app/bsky/feed/getListFeed.ts +1 -3
  50. package/src/api/app/bsky/feed/getPostThread.ts +31 -58
  51. package/src/api/app/bsky/feed/getPosts.ts +21 -18
  52. package/src/api/app/bsky/feed/getSuggestedFeeds.ts +2 -1
  53. package/src/api/app/bsky/feed/getTimeline.ts +1 -3
  54. package/src/api/app/bsky/feed/searchPosts.ts +130 -0
  55. package/src/api/app/bsky/graph/getList.ts +6 -3
  56. package/src/api/app/bsky/graph/getListBlocks.ts +3 -2
  57. package/src/api/app/bsky/graph/getListMutes.ts +2 -1
  58. package/src/api/app/bsky/graph/getLists.ts +2 -1
  59. package/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +3 -1
  60. package/src/api/blob-resolver.ts +6 -11
  61. package/src/api/com/atproto/admin/emitModerationEvent.ts +220 -0
  62. package/src/api/com/atproto/admin/{getModerationActions.ts → getModerationEvent.ts} +5 -11
  63. package/src/api/com/atproto/admin/getRecord.ts +1 -0
  64. package/src/api/com/atproto/admin/{getModerationReports.ts → queryModerationEvents.ts} +13 -16
  65. package/src/api/com/atproto/admin/queryModerationStatuses.ts +55 -0
  66. package/src/api/com/atproto/admin/util.ts +3 -1
  67. package/src/api/com/atproto/moderation/createReport.ts +9 -7
  68. package/src/api/com/atproto/moderation/util.ts +38 -20
  69. package/src/api/com/atproto/temp/fetchLabels.ts +30 -0
  70. package/src/api/index.ts +12 -14
  71. package/src/auth.ts +29 -21
  72. package/src/auto-moderator/index.ts +26 -19
  73. package/src/config.ts +6 -6
  74. package/src/context.ts +15 -9
  75. package/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts +123 -0
  76. package/src/db/migrations/index.ts +1 -0
  77. package/src/db/pagination.ts +26 -3
  78. package/src/db/{periodic-moderation-action-reversal.ts → periodic-moderation-event-reversal.ts} +51 -55
  79. package/src/db/tables/moderation.ts +35 -52
  80. package/src/feed-gen/best-of-follows.ts +6 -3
  81. package/src/feed-gen/bsky-team.ts +1 -1
  82. package/src/feed-gen/hot-classic.ts +1 -1
  83. package/src/feed-gen/mutuals.ts +6 -2
  84. package/src/feed-gen/types.ts +1 -1
  85. package/src/feed-gen/whats-hot.ts +1 -1
  86. package/src/feed-gen/with-friends.ts +7 -3
  87. package/src/index.ts +2 -1
  88. package/src/lexicon/index.ts +52 -67
  89. package/src/lexicon/lexicons.ts +674 -579
  90. package/src/lexicon/types/app/bsky/actor/defs.ts +2 -2
  91. package/src/lexicon/types/app/bsky/actor/searchActors.ts +2 -2
  92. package/src/lexicon/types/app/bsky/actor/searchActorsTypeahead.ts +2 -2
  93. package/src/lexicon/types/app/bsky/feed/defs.ts +1 -18
  94. package/src/lexicon/types/app/bsky/feed/searchPosts.ts +3 -3
  95. package/src/lexicon/types/app/bsky/graph/defs.ts +3 -2
  96. package/src/lexicon/types/app/bsky/unspecced/searchActorsSkeleton.ts +4 -4
  97. package/src/lexicon/types/app/bsky/unspecced/searchPostsSkeleton.ts +3 -3
  98. package/src/lexicon/types/com/atproto/admin/defs.ts +278 -84
  99. package/src/lexicon/types/com/atproto/admin/disableAccountInvites.ts +1 -1
  100. package/src/lexicon/types/com/atproto/admin/{takeModerationAction.ts → emitModerationEvent.ts} +13 -11
  101. package/src/lexicon/types/com/atproto/admin/enableAccountInvites.ts +1 -1
  102. package/src/lexicon/types/com/atproto/admin/{getModerationReport.ts → getModerationEvent.ts} +1 -1
  103. package/src/lexicon/types/com/atproto/admin/{getModerationReports.ts → queryModerationEvents.ts} +8 -15
  104. package/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts +70 -0
  105. package/src/lexicon/types/com/atproto/admin/sendEmail.ts +1 -0
  106. package/src/lexicon/types/com/atproto/label/defs.ts +9 -9
  107. package/src/lexicon/types/com/atproto/label/queryLabels.ts +2 -2
  108. package/src/lexicon/types/com/atproto/repo/applyWrites.ts +1 -1
  109. package/src/lexicon/types/com/atproto/repo/createRecord.ts +2 -2
  110. package/src/lexicon/types/com/atproto/repo/deleteRecord.ts +2 -2
  111. package/src/lexicon/types/com/atproto/repo/listRecords.ts +1 -1
  112. package/src/lexicon/types/com/atproto/repo/putRecord.ts +3 -3
  113. package/src/lexicon/types/com/atproto/sync/listBlobs.ts +1 -1
  114. package/src/lexicon/types/com/atproto/sync/subscribeRepos.ts +4 -4
  115. package/src/lexicon/types/com/atproto/{admin/getModerationActions.ts → temp/fetchLabels.ts} +3 -5
  116. package/src/migrate-moderation-data.ts +414 -0
  117. package/src/services/actor/views.ts +5 -14
  118. package/src/services/feed/index.ts +26 -7
  119. package/src/services/feed/util.ts +47 -19
  120. package/src/services/feed/views.ts +68 -4
  121. package/src/services/graph/index.ts +21 -3
  122. package/src/services/graph/types.ts +1 -0
  123. package/src/services/indexing/plugins/block.ts +2 -3
  124. package/src/services/indexing/plugins/feed-generator.ts +2 -3
  125. package/src/services/indexing/plugins/follow.ts +2 -3
  126. package/src/services/indexing/plugins/like.ts +2 -3
  127. package/src/services/indexing/plugins/list-block.ts +2 -3
  128. package/src/services/indexing/plugins/list-item.ts +2 -3
  129. package/src/services/indexing/plugins/list.ts +2 -3
  130. package/src/services/indexing/plugins/post.ts +3 -4
  131. package/src/services/indexing/plugins/repost.ts +2 -3
  132. package/src/services/indexing/plugins/thread-gate.ts +2 -3
  133. package/src/services/label/index.ts +2 -3
  134. package/src/services/moderation/index.ts +380 -395
  135. package/src/services/moderation/pagination.ts +96 -0
  136. package/src/services/moderation/status.ts +244 -0
  137. package/src/services/moderation/types.ts +49 -0
  138. package/src/services/moderation/views.ts +278 -329
  139. package/src/util/debug.ts +2 -2
  140. package/tests/__snapshots__/feed-generation.test.ts.snap +322 -6
  141. package/tests/__snapshots__/indexing.test.ts.snap +0 -6
  142. package/tests/admin/__snapshots__/get-record.test.ts.snap +30 -132
  143. package/tests/admin/__snapshots__/get-repo.test.ts.snap +14 -60
  144. package/tests/admin/__snapshots__/moderation-events.test.ts.snap +146 -0
  145. package/tests/admin/__snapshots__/moderation-statuses.test.ts.snap +64 -0
  146. package/tests/admin/__snapshots__/moderation.test.ts.snap +0 -125
  147. package/tests/admin/get-record.test.ts +5 -9
  148. package/tests/admin/get-repo.test.ts +38 -9
  149. package/tests/admin/moderation-events.test.ts +221 -0
  150. package/tests/admin/moderation-statuses.test.ts +145 -0
  151. package/tests/admin/moderation.test.ts +512 -860
  152. package/tests/admin/repo-search.test.ts +2 -3
  153. package/tests/auto-moderator/fuzzy-matcher.test.ts +2 -1
  154. package/tests/auto-moderator/takedowns.test.ts +45 -18
  155. package/tests/feed-generation.test.ts +57 -9
  156. package/tests/views/__snapshots__/block-lists.test.ts.snap +3 -9
  157. package/tests/views/__snapshots__/blocks.test.ts.snap +0 -9
  158. package/tests/views/__snapshots__/mute-lists.test.ts.snap +5 -5
  159. package/tests/views/__snapshots__/mutes.test.ts.snap +0 -3
  160. package/tests/views/__snapshots__/thread.test.ts.snap +0 -30
  161. package/tests/views/actor-search.test.ts +2 -3
  162. package/tests/views/author-feed.test.ts +42 -36
  163. package/tests/views/follows.test.ts +40 -35
  164. package/tests/views/list-feed.test.ts +17 -9
  165. package/tests/views/notifications.test.ts +13 -9
  166. package/tests/views/profile.test.ts +20 -18
  167. package/tests/views/suggestions.test.ts +15 -7
  168. package/tests/views/thread.test.ts +54 -26
  169. package/tests/views/threadgating.test.ts +51 -19
  170. package/tests/views/timeline.test.ts +21 -13
  171. package/dist/api/com/atproto/admin/reverseModerationAction.d.ts +0 -3
  172. package/dist/api/com/atproto/admin/takeModerationAction.d.ts +0 -3
  173. package/dist/lexicon/types/com/atproto/admin/resolveModerationReports.d.ts +0 -36
  174. package/dist/lexicon/types/com/atproto/admin/reverseModerationAction.d.ts +0 -36
  175. package/src/api/com/atproto/admin/getModerationAction.ts +0 -44
  176. package/src/api/com/atproto/admin/getModerationReport.ts +0 -43
  177. package/src/api/com/atproto/admin/resolveModerationReports.ts +0 -24
  178. package/src/api/com/atproto/admin/reverseModerationAction.ts +0 -115
  179. package/src/api/com/atproto/admin/takeModerationAction.ts +0 -156
  180. package/src/lexicon/types/com/atproto/admin/getModerationAction.ts +0 -41
  181. package/src/lexicon/types/com/atproto/admin/resolveModerationReports.ts +0 -49
  182. package/src/lexicon/types/com/atproto/admin/reverseModerationAction.ts +0 -49
  183. package/tests/admin/__snapshots__/get-moderation-action.test.ts.snap +0 -172
  184. package/tests/admin/__snapshots__/get-moderation-actions.test.ts.snap +0 -178
  185. package/tests/admin/__snapshots__/get-moderation-report.test.ts.snap +0 -177
  186. package/tests/admin/__snapshots__/get-moderation-reports.test.ts.snap +0 -307
  187. package/tests/admin/get-moderation-action.test.ts +0 -100
  188. package/tests/admin/get-moderation-actions.test.ts +0 -164
  189. package/tests/admin/get-moderation-report.test.ts +0 -100
  190. package/tests/admin/get-moderation-reports.test.ts +0 -332
  191. /package/dist/api/com/atproto/admin/{getModerationAction.d.ts → emitModerationEvent.d.ts} +0 -0
  192. /package/dist/api/com/atproto/admin/{getModerationActions.d.ts → getModerationEvent.d.ts} +0 -0
  193. /package/dist/api/com/atproto/admin/{getModerationReport.d.ts → queryModerationEvents.d.ts} +0 -0
  194. /package/dist/api/com/atproto/admin/{getModerationReports.d.ts → queryModerationStatuses.d.ts} +0 -0
  195. /package/dist/api/com/atproto/{admin/resolveModerationReports.d.ts → temp/fetchLabels.d.ts} +0 -0
@@ -61,7 +61,6 @@ export class AutoModerator {
61
61
  'moderation service not properly configured',
62
62
  )
63
63
  }
64
-
65
64
  this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined
66
65
  this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords)
67
66
  if (abyssEndpoint && abyssPassword) {
@@ -157,18 +156,22 @@ export class AutoModerator {
157
156
  if (!this.textFlagger) return
158
157
  const matches = this.textFlagger.getMatches(text)
159
158
  if (matches.length < 1) return
160
- if (!this.services.moderation) {
161
- log.error(
162
- { subject, text, matches },
163
- 'no moderation service setup to flag record text',
164
- )
165
- return
166
- }
167
- await this.services.moderation(this.ctx.db).report({
168
- reasonType: REASONOTHER,
169
- reason: `Automatically flagged for possible slurs: ${matches.join(', ')}`,
170
- subject,
171
- reportedBy: this.ctx.cfg.labelerDid,
159
+ await this.ctx.db.transaction(async (dbTxn) => {
160
+ if (!this.services.moderation) {
161
+ log.error(
162
+ { subject, text, matches },
163
+ 'no moderation service setup to flag record text',
164
+ )
165
+ return
166
+ }
167
+ return this.services.moderation(dbTxn).report({
168
+ reasonType: REASONOTHER,
169
+ reason: `Automatically flagged for possible slurs: ${matches.join(
170
+ ', ',
171
+ )}`,
172
+ subject,
173
+ reportedBy: this.ctx.cfg.labelerDid,
174
+ })
172
175
  })
173
176
  }
174
177
 
@@ -244,15 +247,17 @@ export class AutoModerator {
244
247
  }
245
248
 
246
249
  if (this.pushAgent) {
247
- await this.pushAgent.com.atproto.admin.takeModerationAction({
248
- action: 'com.atproto.admin.defs#takedown',
250
+ await this.pushAgent.com.atproto.admin.emitModerationEvent({
251
+ event: {
252
+ $type: 'com.atproto.admin.defs#modEventTakedown',
253
+ comment: takedownReason,
254
+ },
249
255
  subject: {
250
256
  $type: 'com.atproto.repo.strongRef',
251
257
  uri: uri.toString(),
252
258
  cid: recordCid.toString(),
253
259
  },
254
260
  subjectBlobCids: takedownCids.map((c) => c.toString()),
255
- reason: takedownReason,
256
261
  createdBy: this.ctx.cfg.labelerDid,
257
262
  })
258
263
  } else {
@@ -261,11 +266,13 @@ export class AutoModerator {
261
266
  throw new Error('no mod push agent or uri invalidator setup')
262
267
  }
263
268
  const modSrvc = this.services.moderation(dbTxn)
264
- const action = await modSrvc.logAction({
265
- action: 'com.atproto.admin.defs#takedown',
269
+ const action = await modSrvc.logEvent({
270
+ event: {
271
+ $type: 'com.atproto.admin.defs#modEventTakedown',
272
+ comment: takedownReason,
273
+ },
266
274
  subject: { uri, cid: recordCid },
267
275
  subjectBlobCids: takedownCids,
268
- reason: takedownReason,
269
276
  createdBy: this.ctx.cfg.labelerDid,
270
277
  })
271
278
  await modSrvc.takedownRecord({
package/src/config.ts CHANGED
@@ -23,7 +23,7 @@ export interface ServerConfigValues {
23
23
  adminPassword: string
24
24
  moderatorPassword?: string
25
25
  triagePassword?: string
26
- moderationActionReverseUrl?: string
26
+ moderationPushUrl?: string
27
27
  }
28
28
 
29
29
  export class ServerConfig {
@@ -78,8 +78,8 @@ export class ServerConfig {
78
78
  const moderatorPassword = process.env.MODERATOR_PASSWORD || undefined
79
79
  const triagePassword = process.env.TRIAGE_PASSWORD || undefined
80
80
  const labelerDid = process.env.LABELER_DID || 'did:example:labeler'
81
- const moderationActionReverseUrl =
82
- overrides?.moderationActionReverseUrl ||
81
+ const moderationPushUrl =
82
+ overrides?.moderationPushUrl ||
83
83
  process.env.MODERATION_PUSH_URL ||
84
84
  undefined
85
85
  return new ServerConfig({
@@ -104,7 +104,7 @@ export class ServerConfig {
104
104
  adminPassword,
105
105
  moderatorPassword,
106
106
  triagePassword,
107
- moderationActionReverseUrl,
107
+ moderationPushUrl,
108
108
  ...stripUndefineds(overrides ?? {}),
109
109
  })
110
110
  }
@@ -206,8 +206,8 @@ export class ServerConfig {
206
206
  return this.cfg.triagePassword
207
207
  }
208
208
 
209
- get moderationActionReverseUrl() {
210
- return this.cfg.moderationActionReverseUrl
209
+ get moderationPushUrl() {
210
+ return this.cfg.moderationPushUrl
211
211
  }
212
212
  }
213
213
 
package/src/context.ts CHANGED
@@ -15,6 +15,7 @@ import { LabelCache } from './label-cache'
15
15
  import { NotificationServer } from './notifications'
16
16
 
17
17
  export class AppContext {
18
+ public moderationPushAgent: AtpAgent | undefined
18
19
  constructor(
19
20
  private opts: {
20
21
  db: DatabaseCoordinator
@@ -30,7 +31,16 @@ export class AppContext {
30
31
  algos: MountedAlgos
31
32
  notifServer: NotificationServer
32
33
  },
33
- ) {}
34
+ ) {
35
+ if (opts.cfg.moderationPushUrl) {
36
+ const url = new URL(opts.cfg.moderationPushUrl)
37
+ this.moderationPushAgent = new AtpAgent({ service: url.origin })
38
+ this.moderationPushAgent.api.setHeader(
39
+ 'authorization',
40
+ auth.buildBasicAuth(url.username, url.password),
41
+ )
42
+ }
43
+ }
34
44
 
35
45
  get db(): DatabaseCoordinator {
36
46
  return this.opts.db
@@ -84,6 +94,10 @@ export class AppContext {
84
94
  return auth.authVerifier(this.idResolver, { aud: null })
85
95
  }
86
96
 
97
+ get authOptionalVerifierAnyAudience() {
98
+ return auth.authOptionalVerifier(this.idResolver, { aud: null })
99
+ }
100
+
87
101
  get authOptionalVerifier() {
88
102
  return auth.authOptionalVerifier(this.idResolver, {
89
103
  aud: this.cfg.serverDid,
@@ -107,14 +121,6 @@ export class AppContext {
107
121
  })
108
122
  }
109
123
 
110
- async pdsAdminAgent(did: string): Promise<AtpAgent> {
111
- const data = await this.idResolver.did.resolveAtprotoData(did)
112
- const agent = new AtpAgent({ service: data.pds })
113
- const jwt = await this.serviceAuthJwt(did)
114
- agent.api.setHeader('authorization', `Bearer ${jwt}`)
115
- return agent
116
- }
117
-
118
124
  get backgroundQueue(): BackgroundQueue {
119
125
  return this.opts.backgroundQueue
120
126
  }
@@ -0,0 +1,123 @@
1
+ import { Kysely } from 'kysely'
2
+
3
+ export async function up(db: Kysely<unknown>): Promise<void> {
4
+ await db.schema
5
+ .createTable('moderation_event')
6
+ .addColumn('id', 'serial', (col) => col.primaryKey())
7
+ .addColumn('action', 'varchar', (col) => col.notNull())
8
+ .addColumn('subjectType', 'varchar', (col) => col.notNull())
9
+ .addColumn('subjectDid', 'varchar', (col) => col.notNull())
10
+ .addColumn('subjectUri', 'varchar')
11
+ .addColumn('subjectCid', 'varchar')
12
+ .addColumn('comment', 'text')
13
+ .addColumn('meta', 'jsonb')
14
+ .addColumn('createdAt', 'varchar', (col) => col.notNull())
15
+ .addColumn('createdBy', 'varchar', (col) => col.notNull())
16
+ .addColumn('reversedAt', 'varchar')
17
+ .addColumn('reversedBy', 'varchar')
18
+ .addColumn('durationInHours', 'integer')
19
+ .addColumn('expiresAt', 'varchar')
20
+ .addColumn('reversedReason', 'text')
21
+ .addColumn('createLabelVals', 'varchar')
22
+ .addColumn('negateLabelVals', 'varchar')
23
+ .addColumn('legacyRefId', 'integer')
24
+ .execute()
25
+ await db.schema
26
+ .createTable('moderation_subject_status')
27
+ .addColumn('id', 'serial', (col) => col.primaryKey())
28
+
29
+ // Identifiers
30
+ .addColumn('did', 'varchar', (col) => col.notNull())
31
+ // Default to '' so that we can apply unique constraints on did and recordPath columns
32
+ .addColumn('recordPath', 'varchar', (col) => col.notNull().defaultTo(''))
33
+ .addColumn('blobCids', 'jsonb')
34
+ .addColumn('recordCid', 'varchar')
35
+
36
+ // human review team state
37
+ .addColumn('reviewState', 'varchar', (col) => col.notNull())
38
+ .addColumn('comment', 'varchar')
39
+ .addColumn('muteUntil', 'varchar')
40
+ .addColumn('lastReviewedAt', 'varchar')
41
+ .addColumn('lastReviewedBy', 'varchar')
42
+
43
+ // report state
44
+ .addColumn('lastReportedAt', 'varchar')
45
+
46
+ // visibility/intervention state
47
+ .addColumn('takendown', 'boolean', (col) => col.defaultTo(false).notNull())
48
+ .addColumn('suspendUntil', 'varchar')
49
+
50
+ // timestamps
51
+ .addColumn('createdAt', 'varchar', (col) => col.notNull())
52
+ .addColumn('updatedAt', 'varchar', (col) => col.notNull())
53
+ .addUniqueConstraint('moderation_status_unique_idx', ['did', 'recordPath'])
54
+ .execute()
55
+
56
+ await db.schema
57
+ .createIndex('moderation_subject_status_blob_cids_idx')
58
+ .on('moderation_subject_status')
59
+ .using('gin')
60
+ .column('blobCids')
61
+ .execute()
62
+
63
+ // Move foreign keys from moderation_action to moderation_event
64
+ await db.schema
65
+ .alterTable('record')
66
+ .dropConstraint('record_takedown_id_fkey')
67
+ .execute()
68
+ await db.schema
69
+ .alterTable('actor')
70
+ .dropConstraint('actor_takedown_id_fkey')
71
+ .execute()
72
+ await db.schema
73
+ .alterTable('actor')
74
+ .addForeignKeyConstraint(
75
+ 'actor_takedown_id_fkey',
76
+ ['takedownId'],
77
+ 'moderation_event',
78
+ ['id'],
79
+ )
80
+ .execute()
81
+ await db.schema
82
+ .alterTable('record')
83
+ .addForeignKeyConstraint(
84
+ 'record_takedown_id_fkey',
85
+ ['takedownId'],
86
+ 'moderation_event',
87
+ ['id'],
88
+ )
89
+ .execute()
90
+ }
91
+
92
+ export async function down(db: Kysely<unknown>): Promise<void> {
93
+ await db.schema.dropTable('moderation_event').execute()
94
+ await db.schema.dropTable('moderation_subject_status').execute()
95
+
96
+ // Revert foreign key constraints
97
+ await db.schema
98
+ .alterTable('record')
99
+ .dropConstraint('record_takedown_id_fkey')
100
+ .execute()
101
+ await db.schema
102
+ .alterTable('actor')
103
+ .dropConstraint('actor_takedown_id_fkey')
104
+ .execute()
105
+ await db.schema
106
+ .alterTable('actor')
107
+ .addForeignKeyConstraint(
108
+ 'actor_takedown_id_fkey',
109
+ ['takedownId'],
110
+ 'moderation_action',
111
+ ['id'],
112
+ )
113
+ .execute()
114
+ await db.schema
115
+ .alterTable('record')
116
+ .addForeignKeyConstraint(
117
+ 'record_takedown_id_fkey',
118
+ ['takedownId'],
119
+ 'moderation_action',
120
+ ['id'],
121
+ )
122
+ .execute()
123
+ }
@@ -30,3 +30,4 @@ export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
30
30
  export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
31
31
  export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
32
32
  export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'
33
+ export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
@@ -117,13 +117,36 @@ export const paginate = <
117
117
  direction?: 'asc' | 'desc'
118
118
  keyset: K
119
119
  tryIndex?: boolean
120
+ // By default, pg does nullsFirst
121
+ nullsLast?: boolean
120
122
  },
121
123
  ): QB => {
122
- const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts
124
+ const {
125
+ limit,
126
+ cursor,
127
+ keyset,
128
+ direction = 'desc',
129
+ tryIndex,
130
+ nullsLast,
131
+ } = opts
123
132
  const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex)
124
133
  return qb
125
134
  .if(!!limit, (q) => q.limit(limit as number))
126
- .orderBy(keyset.primary, direction)
127
- .orderBy(keyset.secondary, direction)
135
+ .if(!nullsLast, (q) =>
136
+ q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction),
137
+ )
138
+ .if(!!nullsLast, (q) =>
139
+ q
140
+ .orderBy(
141
+ direction === 'asc'
142
+ ? sql`${keyset.primary} asc nulls last`
143
+ : sql`${keyset.primary} desc nulls last`,
144
+ )
145
+ .orderBy(
146
+ direction === 'asc'
147
+ ? sql`${keyset.secondary} asc nulls last`
148
+ : sql`${keyset.secondary} desc nulls last`,
149
+ ),
150
+ )
128
151
  .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
129
152
  }
@@ -2,14 +2,15 @@ import { wait } from '@atproto/common'
2
2
  import { Leader } from './leader'
3
3
  import { dbLogger } from '../logger'
4
4
  import AppContext from '../context'
5
+ import { AtUri } from '@atproto/api'
6
+ import { ModerationSubjectStatusRow } from '../services/moderation/types'
7
+ import { CID } from 'multiformats/cid'
5
8
  import AtpAgent from '@atproto/api'
6
- import { buildBasicAuth } from '../auth'
7
- import { LabelService } from '../services/label'
8
- import { ModerationActionRow } from '../services/moderation'
9
+ import { retryHttp } from '../util/retry'
9
10
 
10
11
  export const MODERATION_ACTION_REVERSAL_ID = 1011
11
12
 
12
- export class PeriodicModerationActionReversal {
13
+ export class PeriodicModerationEventReversal {
13
14
  leader = new Leader(
14
15
  MODERATION_ACTION_REVERSAL_ID,
15
16
  this.appContext.db.getPrimary(),
@@ -18,58 +19,53 @@ export class PeriodicModerationActionReversal {
18
19
  pushAgent?: AtpAgent
19
20
 
20
21
  constructor(private appContext: AppContext) {
21
- if (appContext.cfg.moderationActionReverseUrl) {
22
- const url = new URL(appContext.cfg.moderationActionReverseUrl)
23
- this.pushAgent = new AtpAgent({ service: url.origin })
24
- this.pushAgent.api.setHeader(
25
- 'authorization',
26
- buildBasicAuth(url.username, url.password),
27
- )
28
- }
29
- }
30
-
31
- // invert label creation & negations
32
- async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) {
33
- let uri: string
34
- let cid: string | null = null
35
-
36
- if (actionRow.subjectUri && actionRow.subjectCid) {
37
- uri = actionRow.subjectUri
38
- cid = actionRow.subjectCid
39
- } else {
40
- uri = actionRow.subjectDid
41
- }
42
-
43
- await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, {
44
- create: actionRow.negateLabelVals
45
- ? actionRow.negateLabelVals.split(' ')
46
- : undefined,
47
- negate: actionRow.createLabelVals
48
- ? actionRow.createLabelVals.split(' ')
49
- : undefined,
50
- })
22
+ this.pushAgent = appContext.moderationPushAgent
51
23
  }
52
24
 
53
- async revertAction(actionRow: ModerationActionRow) {
54
- const reverseAction = {
55
- id: actionRow.id,
56
- createdBy: actionRow.createdBy,
57
- createdAt: new Date(),
58
- reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`,
59
- }
60
-
61
- if (this.pushAgent) {
62
- await this.pushAgent.com.atproto.admin.reverseModerationAction(
63
- reverseAction,
64
- )
65
- return
66
- }
67
-
25
+ async revertState(eventRow: ModerationSubjectStatusRow) {
68
26
  await this.appContext.db.getPrimary().transaction(async (dbTxn) => {
69
27
  const moderationTxn = this.appContext.services.moderation(dbTxn)
70
- await moderationTxn.revertAction(reverseAction)
71
- const labelTxn = this.appContext.services.label(dbTxn)
72
- await this.reverseLabels(labelTxn, actionRow)
28
+ const originalEvent =
29
+ await moderationTxn.getLastReversibleEventForSubject(eventRow)
30
+ if (originalEvent) {
31
+ const { restored } = await moderationTxn.revertState({
32
+ action: originalEvent.action,
33
+ createdBy: originalEvent.createdBy,
34
+ comment:
35
+ '[SCHEDULED_REVERSAL] Reverting action as originally scheduled',
36
+ subject:
37
+ eventRow.recordPath && eventRow.recordCid
38
+ ? {
39
+ uri: AtUri.make(
40
+ eventRow.did,
41
+ ...eventRow.recordPath.split('/'),
42
+ ),
43
+ cid: CID.parse(eventRow.recordCid),
44
+ }
45
+ : { did: eventRow.did },
46
+ createdAt: new Date(),
47
+ })
48
+
49
+ const { pushAgent } = this
50
+ if (
51
+ originalEvent.action === 'com.atproto.admin.defs#modEventTakedown' &&
52
+ restored?.subjects?.length &&
53
+ pushAgent
54
+ ) {
55
+ await Promise.allSettled(
56
+ restored.subjects.map((subject) =>
57
+ retryHttp(() =>
58
+ pushAgent.api.com.atproto.admin.updateSubjectStatus({
59
+ subject,
60
+ takedown: {
61
+ applied: false,
62
+ },
63
+ }),
64
+ ),
65
+ ),
66
+ )
67
+ }
68
+ }
73
69
  })
74
70
  }
75
71
 
@@ -77,12 +73,12 @@ export class PeriodicModerationActionReversal {
77
73
  const moderationService = this.appContext.services.moderation(
78
74
  this.appContext.db.getPrimary(),
79
75
  )
80
- const actionsDueForReversal =
81
- await moderationService.getActionsDueForReversal()
76
+ const subjectsDueForReversal =
77
+ await moderationService.getSubjectsDueForReversal()
82
78
 
83
79
  // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine
84
80
  // Internally, each reversal runs within its own transaction
85
- await Promise.all(actionsDueForReversal.map(this.revertAction.bind(this)))
81
+ await Promise.all(subjectsDueForReversal.map(this.revertState.bind(this)))
86
82
  }
87
83
 
88
84
  async run() {
@@ -1,76 +1,59 @@
1
1
  import { Generated } from 'kysely'
2
2
  import {
3
- ACKNOWLEDGE,
4
- FLAG,
5
- TAKEDOWN,
6
- ESCALATE,
3
+ REVIEWCLOSED,
4
+ REVIEWOPEN,
5
+ REVIEWESCALATED,
7
6
  } from '../../lexicon/types/com/atproto/admin/defs'
8
- import {
9
- REASONOTHER,
10
- REASONSPAM,
11
- REASONMISLEADING,
12
- REASONRUDE,
13
- REASONSEXUAL,
14
- REASONVIOLATION,
15
- } from '../../lexicon/types/com/atproto/moderation/defs'
16
7
 
17
- export const actionTableName = 'moderation_action'
18
- export const actionSubjectBlobTableName = 'moderation_action_subject_blob'
19
- export const reportTableName = 'moderation_report'
20
- export const reportResolutionTableName = 'moderation_report_resolution'
8
+ export const eventTableName = 'moderation_event'
9
+ export const subjectStatusTableName = 'moderation_subject_status'
21
10
 
22
- export interface ModerationAction {
11
+ export interface ModerationEvent {
23
12
  id: Generated<number>
24
- action: typeof TAKEDOWN | typeof FLAG | typeof ACKNOWLEDGE | typeof ESCALATE
13
+ action:
14
+ | 'com.atproto.admin.defs#modEventTakedown'
15
+ | 'com.atproto.admin.defs#modEventAcknowledge'
16
+ | 'com.atproto.admin.defs#modEventEscalate'
17
+ | 'com.atproto.admin.defs#modEventComment'
18
+ | 'com.atproto.admin.defs#modEventLabel'
19
+ | 'com.atproto.admin.defs#modEventReport'
20
+ | 'com.atproto.admin.defs#modEventMute'
21
+ | 'com.atproto.admin.defs#modEventReverseTakedown'
22
+ | 'com.atproto.admin.defs#modEventEmail'
25
23
  subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
26
24
  subjectDid: string
27
25
  subjectUri: string | null
28
26
  subjectCid: string | null
29
27
  createLabelVals: string | null
30
28
  negateLabelVals: string | null
31
- reason: string
29
+ comment: string | null
32
30
  createdAt: string
33
31
  createdBy: string
34
- reversedAt: string | null
35
- reversedBy: string | null
36
- reversedReason: string | null
37
32
  durationInHours: number | null
38
33
  expiresAt: string | null
34
+ meta: Record<string, string | boolean> | null
35
+ legacyRefId: number | null
39
36
  }
40
37
 
41
- export interface ModerationActionSubjectBlob {
42
- actionId: number
43
- cid: string
44
- }
45
-
46
- export interface ModerationReport {
38
+ export interface ModerationSubjectStatus {
47
39
  id: Generated<number>
48
- subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
49
- subjectDid: string
50
- subjectUri: string | null
51
- subjectCid: string | null
52
- reasonType:
53
- | typeof REASONSPAM
54
- | typeof REASONOTHER
55
- | typeof REASONMISLEADING
56
- | typeof REASONRUDE
57
- | typeof REASONSEXUAL
58
- | typeof REASONVIOLATION
59
- reason: string | null
60
- reportedByDid: string
40
+ did: string
41
+ recordPath: string
42
+ recordCid: string | null
43
+ blobCids: string[] | null
44
+ reviewState: typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED
61
45
  createdAt: string
62
- }
63
-
64
- export interface ModerationReportResolution {
65
- reportId: number
66
- actionId: number
67
- createdAt: string
68
- createdBy: string
46
+ updatedAt: string
47
+ lastReviewedBy: string | null
48
+ lastReviewedAt: string | null
49
+ lastReportedAt: string | null
50
+ muteUntil: string | null
51
+ suspendUntil: string | null
52
+ takendown: boolean
53
+ comment: string | null
69
54
  }
70
55
 
71
56
  export type PartialDB = {
72
- [actionTableName]: ModerationAction
73
- [actionSubjectBlobTableName]: ModerationActionSubjectBlob
74
- [reportTableName]: ModerationReport
75
- [reportResolutionTableName]: ModerationReportResolution
57
+ [eventTableName]: ModerationEvent
58
+ [subjectStatusTableName]: ModerationSubjectStatus
76
59
  }
@@ -1,4 +1,4 @@
1
- import { InvalidRequestError } from '@atproto/xrpc-server'
1
+ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
2
2
  import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
3
3
  import { AlgoHandler, AlgoResponse } from './types'
4
4
  import { GenericKeyset, paginate } from '../db/pagination'
@@ -7,12 +7,15 @@ import AppContext from '../context'
7
7
  const handler: AlgoHandler = async (
8
8
  ctx: AppContext,
9
9
  params: SkeletonParams,
10
- viewer: string,
10
+ viewer: string | null,
11
11
  ): Promise<AlgoResponse> => {
12
+ if (!viewer) {
13
+ throw new AuthRequiredError('This feed requires being logged-in')
14
+ }
15
+
12
16
  const { limit, cursor } = params
13
17
  const db = ctx.db.getReplica('feed')
14
18
  const feedService = ctx.services.feed(db)
15
-
16
19
  const { ref } = db.db.dynamic
17
20
 
18
21
  // candidates are ranked within a materialized view by like count, depreciated over time.
@@ -14,7 +14,7 @@ const BSKY_TEAM: NotEmptyArray<string> = [
14
14
  const handler: AlgoHandler = async (
15
15
  ctx: AppContext,
16
16
  params: SkeletonParams,
17
- _viewer: string,
17
+ _viewer: string | null,
18
18
  ): Promise<AlgoResponse> => {
19
19
  const { limit = 50, cursor } = params
20
20
  const db = ctx.db.getReplica('feed')
@@ -11,7 +11,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = ['!no-promote']
11
11
  const handler: AlgoHandler = async (
12
12
  ctx: AppContext,
13
13
  params: SkeletonParams,
14
- _viewer: string,
14
+ _viewer: string | null,
15
15
  ): Promise<AlgoResponse> => {
16
16
  const { limit = 50, cursor } = params
17
17
  const db = ctx.db.getReplica('feed')
@@ -3,16 +3,20 @@ import AppContext from '../context'
3
3
  import { paginate } from '../db/pagination'
4
4
  import { AlgoHandler, AlgoResponse } from './types'
5
5
  import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed'
6
+ import { AuthRequiredError } from '@atproto/xrpc-server'
6
7
 
7
8
  const handler: AlgoHandler = async (
8
9
  ctx: AppContext,
9
10
  params: SkeletonParams,
10
- viewer: string,
11
+ viewer: string | null,
11
12
  ): Promise<AlgoResponse> => {
13
+ if (!viewer) {
14
+ throw new AuthRequiredError('This feed requires being logged-in')
15
+ }
16
+
12
17
  const { limit = 50, cursor } = params
13
18
  const db = ctx.db.getReplica('feed')
14
19
  const feedService = ctx.services.feed(db)
15
-
16
20
  const { ref } = db.db.dynamic
17
21
 
18
22
  const mutualsSubquery = db.db
@@ -11,7 +11,7 @@ export type AlgoResponse = {
11
11
  export type AlgoHandler = (
12
12
  ctx: AppContext,
13
13
  params: SkeletonParams,
14
- requester: string,
14
+ viewer: string | null,
15
15
  ) => Promise<AlgoResponse>
16
16
 
17
17
  export type MountedAlgos = Record<string, AlgoHandler>
@@ -21,7 +21,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = [
21
21
  const handler: AlgoHandler = async (
22
22
  ctx: AppContext,
23
23
  params: SkeletonParams,
24
- _viewer: string,
24
+ _viewer: string | null,
25
25
  ): Promise<AlgoResponse> => {
26
26
  const { limit, cursor } = params
27
27
  const db = ctx.db.getReplica('feed')