@atproto/bsky 0.0.15 → 0.0.16

package/src/auto-moderator/index.ts

@@ -61,7 +61,6 @@ export class AutoModerator {
         'moderation service not properly configured',
       )
     }
-
     this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined
     this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords)
     if (abyssEndpoint && abyssPassword) {
@@ -157,18 +156,22 @@ export class AutoModerator {
     if (!this.textFlagger) return
     const matches = this.textFlagger.getMatches(text)
     if (matches.length < 1) return
-    if (!this.services.moderation) {
-      log.error(
-        { subject, text, matches },
-        'no moderation service setup to flag record text',
-      )
-      return
-    }
-    await this.services.moderation(this.ctx.db).report({
-      reasonType: REASONOTHER,
-      reason: `Automatically flagged for possible slurs: ${matches.join(', ')}`,
-      subject,
-      reportedBy: this.ctx.cfg.labelerDid,
+    await this.ctx.db.transaction(async (dbTxn) => {
+      if (!this.services.moderation) {
+        log.error(
+          { subject, text, matches },
+          'no moderation service setup to flag record text',
+        )
+        return
+      }
+      return this.services.moderation(dbTxn).report({
+        reasonType: REASONOTHER,
+        reason: `Automatically flagged for possible slurs: ${matches.join(
+          ', ',
+        )}`,
+        subject,
+        reportedBy: this.ctx.cfg.labelerDid,
+      })
     })
   }
 
@@ -244,15 +247,17 @@ export class AutoModerator {
     }
 
     if (this.pushAgent) {
-      await this.pushAgent.com.atproto.admin.takeModerationAction({
-        action: 'com.atproto.admin.defs#takedown',
+      await this.pushAgent.com.atproto.admin.emitModerationEvent({
+        event: {
+          $type: 'com.atproto.admin.defs#modEventTakedown',
+          comment: takedownReason,
+        },
         subject: {
           $type: 'com.atproto.repo.strongRef',
           uri: uri.toString(),
           cid: recordCid.toString(),
         },
         subjectBlobCids: takedownCids.map((c) => c.toString()),
-        reason: takedownReason,
         createdBy: this.ctx.cfg.labelerDid,
       })
     } else {
@@ -261,11 +266,13 @@ export class AutoModerator {
           throw new Error('no mod push agent or uri invalidator setup')
         }
         const modSrvc = this.services.moderation(dbTxn)
-        const action = await modSrvc.logAction({
-          action: 'com.atproto.admin.defs#takedown',
+        const action = await modSrvc.logEvent({
+          event: {
+            $type: 'com.atproto.admin.defs#modEventTakedown',
+            comment: takedownReason,
+          },
           subject: { uri, cid: recordCid },
           subjectBlobCids: takedownCids,
-          reason: takedownReason,
           createdBy: this.ctx.cfg.labelerDid,
         })
         await modSrvc.takedownRecord({

package/src/context.ts

@@ -94,6 +94,10 @@ export class AppContext {
     return auth.authVerifier(this.idResolver, { aud: null })
   }
 
+  get authOptionalVerifierAnyAudience() {
+    return auth.authOptionalVerifier(this.idResolver, { aud: null })
+  }
+
   get authOptionalVerifier() {
     return auth.authOptionalVerifier(this.idResolver, {
       aud: this.cfg.serverDid,

package/src/db/migrations/20231003T202833377Z-create-moderation-subject-status.ts

@@ -0,0 +1,123 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('moderation_event')
+    .addColumn('id', 'serial', (col) => col.primaryKey())
+    .addColumn('action', 'varchar', (col) => col.notNull())
+    .addColumn('subjectType', 'varchar', (col) => col.notNull())
+    .addColumn('subjectDid', 'varchar', (col) => col.notNull())
+    .addColumn('subjectUri', 'varchar')
+    .addColumn('subjectCid', 'varchar')
+    .addColumn('comment', 'text')
+    .addColumn('meta', 'jsonb')
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('createdBy', 'varchar', (col) => col.notNull())
+    .addColumn('reversedAt', 'varchar')
+    .addColumn('reversedBy', 'varchar')
+    .addColumn('durationInHours', 'integer')
+    .addColumn('expiresAt', 'varchar')
+    .addColumn('reversedReason', 'text')
+    .addColumn('createLabelVals', 'varchar')
+    .addColumn('negateLabelVals', 'varchar')
+    .addColumn('legacyRefId', 'integer')
+    .execute()
+  await db.schema
+    .createTable('moderation_subject_status')
+    .addColumn('id', 'serial', (col) => col.primaryKey())
+
+    // Identifiers
+    .addColumn('did', 'varchar', (col) => col.notNull())
+    // Default to '' so that we can apply unique constraints on did and recordPath columns
+    .addColumn('recordPath', 'varchar', (col) => col.notNull().defaultTo(''))
+    .addColumn('blobCids', 'jsonb')
+    .addColumn('recordCid', 'varchar')
+
+    // human review team state
+    .addColumn('reviewState', 'varchar', (col) => col.notNull())
+    .addColumn('comment', 'varchar')
+    .addColumn('muteUntil', 'varchar')
+    .addColumn('lastReviewedAt', 'varchar')
+    .addColumn('lastReviewedBy', 'varchar')
+
+    // report state
+    .addColumn('lastReportedAt', 'varchar')
+
+    // visibility/intervention state
+    .addColumn('takendown', 'boolean', (col) => col.defaultTo(false).notNull())
+    .addColumn('suspendUntil', 'varchar')
+
+    // timestamps
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('updatedAt', 'varchar', (col) => col.notNull())
+    .addUniqueConstraint('moderation_status_unique_idx', ['did', 'recordPath'])
+    .execute()
+
+  await db.schema
+    .createIndex('moderation_subject_status_blob_cids_idx')
+    .on('moderation_subject_status')
+    .using('gin')
+    .column('blobCids')
+    .execute()
+
+  // Move foreign keys from moderation_action to moderation_event
+  await db.schema
+    .alterTable('record')
+    .dropConstraint('record_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .dropConstraint('actor_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .addForeignKeyConstraint(
+      'actor_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_event',
+      ['id'],
+    )
+    .execute()
+  await db.schema
+    .alterTable('record')
+    .addForeignKeyConstraint(
+      'record_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_event',
+      ['id'],
+    )
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('moderation_event').execute()
+  await db.schema.dropTable('moderation_subject_status').execute()
+
+  // Revert foreign key constraints
+  await db.schema
+    .alterTable('record')
+    .dropConstraint('record_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .dropConstraint('actor_takedown_id_fkey')
+    .execute()
+  await db.schema
+    .alterTable('actor')
+    .addForeignKeyConstraint(
+      'actor_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_action',
+      ['id'],
+    )
+    .execute()
+  await db.schema
+    .alterTable('record')
+    .addForeignKeyConstraint(
+      'record_takedown_id_fkey',
+      ['takedownId'],
+      'moderation_action',
+      ['id'],
+    )
+    .execute()
+}

package/src/db/migrations/index.ts

@@ -30,3 +30,4 @@ export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
 export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'
 export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-indexes'
+export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'

package/src/db/pagination.ts

@@ -117,13 +117,36 @@ export const paginate = <
     direction?: 'asc' | 'desc'
     keyset: K
     tryIndex?: boolean
+    // By default, pg does nullsFirst
+    nullsLast?: boolean
   },
 ): QB => {
-  const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts
+  const {
+    limit,
+    cursor,
+    keyset,
+    direction = 'desc',
+    tryIndex,
+    nullsLast,
+  } = opts
   const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex)
   return qb
     .if(!!limit, (q) => q.limit(limit as number))
-    .orderBy(keyset.primary, direction)
-    .orderBy(keyset.secondary, direction)
+    .if(!nullsLast, (q) =>
+      q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction),
+    )
+    .if(!!nullsLast, (q) =>
+      q
+        .orderBy(
+          direction === 'asc'
+            ? sql`${keyset.primary} asc nulls last`
+            : sql`${keyset.primary} desc nulls last`,
+        )
+        .orderBy(
+          direction === 'asc'
+            ? sql`${keyset.secondary} asc nulls last`
+            : sql`${keyset.secondary} desc nulls last`,
+        ),
+    )
     .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
 }

package/src/db/{periodic-moderation-action-reversal.ts → periodic-moderation-event-reversal.ts}

@@ -2,13 +2,15 @@ import { wait } from '@atproto/common'
 import { Leader } from './leader'
 import { dbLogger } from '../logger'
 import AppContext from '../context'
+import { AtUri } from '@atproto/api'
+import { ModerationSubjectStatusRow } from '../services/moderation/types'
+import { CID } from 'multiformats/cid'
 import AtpAgent from '@atproto/api'
-import { LabelService } from '../services/label'
-import { ModerationActionRow } from '../services/moderation'
+import { retryHttp } from '../util/retry'
 
 export const MODERATION_ACTION_REVERSAL_ID = 1011
 
-export class PeriodicModerationActionReversal {
+export class PeriodicModerationEventReversal {
   leader = new Leader(
     MODERATION_ACTION_REVERSAL_ID,
     this.appContext.db.getPrimary(),
@@ -20,48 +22,50 @@ export class PeriodicModerationActionReversal {
     this.pushAgent = appContext.moderationPushAgent
   }
 
-  // invert label creation & negations
-  async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) {
-    let uri: string
-    let cid: string | null = null
-
-    if (actionRow.subjectUri && actionRow.subjectCid) {
-      uri = actionRow.subjectUri
-      cid = actionRow.subjectCid
-    } else {
-      uri = actionRow.subjectDid
-    }
-
-    await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, {
-      create: actionRow.negateLabelVals
-        ? actionRow.negateLabelVals.split(' ')
-        : undefined,
-      negate: actionRow.createLabelVals
-        ? actionRow.createLabelVals.split(' ')
-        : undefined,
-    })
-  }
-
-  async revertAction(actionRow: ModerationActionRow) {
-    const reverseAction = {
-      id: actionRow.id,
-      createdBy: actionRow.createdBy,
-      createdAt: new Date(),
-      reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`,
-    }
-
-    if (this.pushAgent) {
-      await this.pushAgent.com.atproto.admin.reverseModerationAction(
-        reverseAction,
-      )
-      return
-    }
-
+  async revertState(eventRow: ModerationSubjectStatusRow) {
     await this.appContext.db.getPrimary().transaction(async (dbTxn) => {
       const moderationTxn = this.appContext.services.moderation(dbTxn)
-      await moderationTxn.revertAction(reverseAction)
-      const labelTxn = this.appContext.services.label(dbTxn)
-      await this.reverseLabels(labelTxn, actionRow)
+      const originalEvent =
+        await moderationTxn.getLastReversibleEventForSubject(eventRow)
+      if (originalEvent) {
+        const { restored } = await moderationTxn.revertState({
+          action: originalEvent.action,
+          createdBy: originalEvent.createdBy,
+          comment:
+            '[SCHEDULED_REVERSAL] Reverting action as originally scheduled',
+          subject:
+            eventRow.recordPath && eventRow.recordCid
+              ? {
+                  uri: AtUri.make(
+                    eventRow.did,
+                    ...eventRow.recordPath.split('/'),
+                  ),
+                  cid: CID.parse(eventRow.recordCid),
+                }
+              : { did: eventRow.did },
+          createdAt: new Date(),
+        })
+
+        const { pushAgent } = this
+        if (
+          originalEvent.action === 'com.atproto.admin.defs#modEventTakedown' &&
+          restored?.subjects?.length &&
+          pushAgent
+        ) {
+          await Promise.allSettled(
+            restored.subjects.map((subject) =>
+              retryHttp(() =>
+                pushAgent.api.com.atproto.admin.updateSubjectStatus({
+                  subject,
+                  takedown: {
+                    applied: false,
+                  },
+                }),
+              ),
+            ),
+          )
+        }
+      }
     })
   }
 
@@ -69,12 +73,12 @@ export class PeriodicModerationActionReversal {
     const moderationService = this.appContext.services.moderation(
       this.appContext.db.getPrimary(),
     )
-    const actionsDueForReversal =
-      await moderationService.getActionsDueForReversal()
+    const subjectsDueForReversal =
+      await moderationService.getSubjectsDueForReversal()
 
     // We shouldn't have too many actions due for reversal at any given time, so running in parallel is probably fine
     // Internally, each reversal runs within its own transaction
-    await Promise.all(actionsDueForReversal.map(this.revertAction.bind(this)))
+    await Promise.all(subjectsDueForReversal.map(this.revertState.bind(this)))
   }
 
   async run() {

package/src/db/tables/moderation.ts

@@ -1,76 +1,59 @@
 import { Generated } from 'kysely'
 import {
-  ACKNOWLEDGE,
-  FLAG,
-  TAKEDOWN,
-  ESCALATE,
+  REVIEWCLOSED,
+  REVIEWOPEN,
+  REVIEWESCALATED,
 } from '../../lexicon/types/com/atproto/admin/defs'
-import {
-  REASONOTHER,
-  REASONSPAM,
-  REASONMISLEADING,
-  REASONRUDE,
-  REASONSEXUAL,
-  REASONVIOLATION,
-} from '../../lexicon/types/com/atproto/moderation/defs'
 
-export const actionTableName = 'moderation_action'
-export const actionSubjectBlobTableName = 'moderation_action_subject_blob'
-export const reportTableName = 'moderation_report'
-export const reportResolutionTableName = 'moderation_report_resolution'
+export const eventTableName = 'moderation_event'
+export const subjectStatusTableName = 'moderation_subject_status'
 
-export interface ModerationAction {
+export interface ModerationEvent {
   id: Generated<number>
-  action: typeof TAKEDOWN | typeof FLAG | typeof ACKNOWLEDGE | typeof ESCALATE
+  action:
+    | 'com.atproto.admin.defs#modEventTakedown'
+    | 'com.atproto.admin.defs#modEventAcknowledge'
+    | 'com.atproto.admin.defs#modEventEscalate'
+    | 'com.atproto.admin.defs#modEventComment'
+    | 'com.atproto.admin.defs#modEventLabel'
+    | 'com.atproto.admin.defs#modEventReport'
+    | 'com.atproto.admin.defs#modEventMute'
+    | 'com.atproto.admin.defs#modEventReverseTakedown'
+    | 'com.atproto.admin.defs#modEventEmail'
   subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
   subjectDid: string
   subjectUri: string | null
   subjectCid: string | null
   createLabelVals: string | null
   negateLabelVals: string | null
-  reason: string
+  comment: string | null
   createdAt: string
   createdBy: string
-  reversedAt: string | null
-  reversedBy: string | null
-  reversedReason: string | null
   durationInHours: number | null
   expiresAt: string | null
+  meta: Record<string, string | boolean> | null
+  legacyRefId: number | null
 }
 
-export interface ModerationActionSubjectBlob {
-  actionId: number
-  cid: string
-}
-
-export interface ModerationReport {
+export interface ModerationSubjectStatus {
   id: Generated<number>
-  subjectType: 'com.atproto.admin.defs#repoRef' | 'com.atproto.repo.strongRef'
-  subjectDid: string
-  subjectUri: string | null
-  subjectCid: string | null
-  reasonType:
-    | typeof REASONSPAM
-    | typeof REASONOTHER
-    | typeof REASONMISLEADING
-    | typeof REASONRUDE
-    | typeof REASONSEXUAL
-    | typeof REASONVIOLATION
-  reason: string | null
-  reportedByDid: string
+  did: string
+  recordPath: string
+  recordCid: string | null
+  blobCids: string[] | null
+  reviewState: typeof REVIEWCLOSED | typeof REVIEWOPEN | typeof REVIEWESCALATED
   createdAt: string
-}
-
-export interface ModerationReportResolution {
-  reportId: number
-  actionId: number
-  createdAt: string
-  createdBy: string
+  updatedAt: string
+  lastReviewedBy: string | null
+  lastReviewedAt: string | null
+  lastReportedAt: string | null
+  muteUntil: string | null
+  suspendUntil: string | null
+  takendown: boolean
+  comment: string | null
 }
 
 export type PartialDB = {
-  [actionTableName]: ModerationAction
-  [actionSubjectBlobTableName]: ModerationActionSubjectBlob
-  [reportTableName]: ModerationReport
-  [reportResolutionTableName]: ModerationReportResolution
+  [eventTableName]: ModerationEvent
+  [subjectStatusTableName]: ModerationSubjectStatus
 }

package/src/feed-gen/best-of-follows.ts

@@ -1,4 +1,4 @@
-import { InvalidRequestError } from '@atproto/xrpc-server'
+import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
 import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
 import { AlgoHandler, AlgoResponse } from './types'
 import { GenericKeyset, paginate } from '../db/pagination'
@@ -7,12 +7,15 @@ import AppContext from '../context'
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  viewer: string,
+  viewer: string | null,
 ): Promise<AlgoResponse> => {
+  if (!viewer) {
+    throw new AuthRequiredError('This feed requires being logged-in')
+  }
+
   const { limit, cursor } = params
   const db = ctx.db.getReplica('feed')
   const feedService = ctx.services.feed(db)
-
   const { ref } = db.db.dynamic
 
   // candidates are ranked within a materialized view by like count, depreciated over time.

package/src/feed-gen/bsky-team.ts

@@ -14,7 +14,7 @@ const BSKY_TEAM: NotEmptyArray<string> = [
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  _viewer: string,
+  _viewer: string | null,
 ): Promise<AlgoResponse> => {
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')

package/src/feed-gen/hot-classic.ts

@@ -11,7 +11,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = ['!no-promote']
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  _viewer: string,
+  _viewer: string | null,
 ): Promise<AlgoResponse> => {
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')

package/src/feed-gen/mutuals.ts

@@ -3,16 +3,20 @@ import AppContext from '../context'
 import { paginate } from '../db/pagination'
 import { AlgoHandler, AlgoResponse } from './types'
 import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed'
+import { AuthRequiredError } from '@atproto/xrpc-server'
 
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  viewer: string,
+  viewer: string | null,
 ): Promise<AlgoResponse> => {
+  if (!viewer) {
+    throw new AuthRequiredError('This feed requires being logged-in')
+  }
+
   const { limit = 50, cursor } = params
   const db = ctx.db.getReplica('feed')
   const feedService = ctx.services.feed(db)
-
   const { ref } = db.db.dynamic
 
   const mutualsSubquery = db.db

package/src/feed-gen/types.ts

@@ -11,7 +11,7 @@ export type AlgoResponse = {
 export type AlgoHandler = (
   ctx: AppContext,
   params: SkeletonParams,
-  requester: string,
+  viewer: string | null,
 ) => Promise<AlgoResponse>
 
 export type MountedAlgos = Record<string, AlgoHandler>

package/src/feed-gen/whats-hot.ts

@@ -21,7 +21,7 @@ const NO_WHATS_HOT_LABELS: NotEmptyArray<string> = [
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  _viewer: string,
+  _viewer: string | null,
 ): Promise<AlgoResponse> => {
   const { limit, cursor } = params
   const db = ctx.db.getReplica('feed')

package/src/feed-gen/with-friends.ts

@@ -3,16 +3,20 @@ import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/ge
 import { paginate } from '../db/pagination'
 import { AlgoHandler, AlgoResponse } from './types'
 import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed'
+import { AuthRequiredError } from '@atproto/xrpc-server'
 
 const handler: AlgoHandler = async (
   ctx: AppContext,
   params: SkeletonParams,
-  requester: string,
+  viewer: string | null,
 ): Promise<AlgoResponse> => {
+  if (!viewer) {
+    throw new AuthRequiredError('This feed requires being logged-in')
+  }
+
   const { cursor, limit = 50 } = params
   const db = ctx.db.getReplica('feed')
   const feedService = ctx.services.feed(db)
-
   const { ref } = db.db.dynamic
 
   const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid'))
@@ -23,7 +27,7 @@ const handler: AlgoHandler = async (
     .innerJoin('follow', 'follow.subjectDid', 'post.creator')
     .innerJoin('post_agg', 'post_agg.uri', 'post.uri')
     .where('post_agg.likeCount', '>=', 5)
-    .where('follow.creator', '=', requester)
+    .where('follow.creator', '=', viewer)
     .where('post.sortAt', '>', getFeedDateThreshold(sortFrom))
 
   postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true })

package/src/index.ts

@@ -32,13 +32,14 @@ export type { ServerConfigValues } from './config'
 export type { MountedAlgos } from './feed-gen/types'
 export { ServerConfig } from './config'
 export { Database, PrimaryDatabase, DatabaseCoordinator } from './db'
-export { PeriodicModerationActionReversal } from './db/periodic-moderation-action-reversal'
+export { PeriodicModerationEventReversal } from './db/periodic-moderation-event-reversal'
 export { Redis } from './redis'
 export { ViewMaintainer } from './db/views'
 export { AppContext } from './context'
 export { makeAlgos } from './feed-gen'
 export * from './indexer'
 export * from './ingester'
+export { MigrateModerationData } from './migrate-moderation-data'
 
 export class BskyAppView {
   public ctx: AppContext