@atproto/ozone 0.1.68 → 0.1.70

package/src/mod-service/index.ts

@@ -1,5 +1,5 @@
 import net from 'node:net'
-import { Insertable, SelectQueryBuilder, sql } from 'kysely'
+import { Insertable, sql } from 'kysely'
 import { CID } from 'multiformats/cid'
 import { AtUri, INVALID_HANDLE } from '@atproto/syntax'
 import { InvalidRequestError } from '@atproto/xrpc-server'
@@ -29,16 +29,19 @@ import { RepoRef, RepoBlobRef } from '../lexicon/types/com/atproto/admin/defs'
 import {
   adjustModerationSubjectStatus,
   getStatusIdentifierFromSubject,
+  moderationSubjectStatusQueryBuilder,
 } from './status'
 import {
   ModEventType,
   ModerationEventRow,
   ModerationSubjectStatusRow,
+  ModerationSubjectStatusRowWithHandle,
   ReversibleModerationEvent,
 } from './types'
 import { ModerationEvent } from '../db/schema/moderation_event'
 import { StatusKeyset, TimeIdKeyset, paginate } from '../db/pagination'
 import { Label } from '../lexicon/types/com/atproto/label/defs'
+import { QueryParams as QueryStatusParams } from '../lexicon/types/tools/ozone/moderation/queryStatuses'
 import {
   ModSubject,
   RecordSubject,
@@ -56,6 +59,7 @@ import { httpLogger as log } from '../logger'
 import { OzoneConfig } from '../config'
 import { LABELER_HEADER_NAME, ParsedLabelers } from '../util'
 import { ids } from '../lexicon/lexicons'
+import { getReviewState } from '../api/util'
 
 export type ModerationServiceCreator = (db: Database) => ModerationService
 
@@ -805,45 +809,6 @@ export class ModerationService {
     return result
   }
 
-  applyTagFilter = (
-    builder: SelectQueryBuilder<any, any, any>,
-    tags: string[],
-  ) => {
-    const { ref } = this.db.db.dynamic
-    // Build an array of conditions
-    const conditions = tags
-      .map((tag) => {
-        if (tag.includes('&&')) {
-          // Split by '&&' for AND logic
-          const subTags = tag
-            .split('&&')
-            // Make sure spaces on either sides of '&&' are trimmed
-            .map((subTag) => subTag.trim())
-            // Remove empty strings after trimming is applied
-            .filter(Boolean)
-
-          if (!subTags.length) return null
-
-          return sql`(${sql.join(
-            subTags.map(
-              (subTag) =>
-                sql`${ref('moderation_subject_status.tags')} ? ${subTag}`,
-            ),
-            sql` AND `,
-          )})`
-        } else {
-          // Single tag condition
-          return sql`${ref('moderation_subject_status.tags')} ? ${tag}`
-        }
-      })
-      .filter(Boolean)
-
-    if (!conditions.length) return builder
-
-    // Combine all conditions with OR
-    return builder.where(sql`(${sql.join(conditions, sql` OR `)})`)
-  }
-
   async getSubjectStatuses({
     queueCount,
     queueIndex,
@@ -858,54 +823,31 @@ export class ModerationService {
     reviewedBefore,
     reportedAfter,
     reportedBefore,
-    includeMuted,
+    includeMuted = false,
     hostingDeletedBefore,
     hostingDeletedAfter,
     hostingUpdatedBefore,
     hostingUpdatedAfter,
     hostingStatuses,
-    onlyMuted,
+    onlyMuted = false,
     ignoreSubjects,
-    sortDirection,
+    sortDirection = 'desc',
     lastReviewedBy,
-    sortField,
+    sortField = 'lastReportedAt',
     subject,
     tags,
     excludeTags,
     collections,
     subjectType,
-  }: {
-    queueCount?: number
-    queueIndex?: number
-    queueSeed?: string
-    includeAllUserRecords?: boolean
+    minAccountSuspendCount,
+    minReportedRecordsCount,
+    minTakendownRecordsCount,
+  }: QueryStatusParams): Promise<{
+    statuses: ModerationSubjectStatusRowWithHandle[]
     cursor?: string
-    limit?: number
-    takendown?: boolean
-    appealed?: boolean
-    reviewedBefore?: string
-    reviewState?: ModerationSubjectStatusRow['reviewState']
-    reviewedAfter?: string
-    reportedAfter?: string
-    reportedBefore?: string
-    includeMuted?: boolean
-    hostingDeletedBefore?: string
-    hostingDeletedAfter?: string
-    hostingUpdatedBefore?: string
-    hostingUpdatedAfter?: string
-    hostingStatuses?: string[]
-    onlyMuted?: boolean
-    subject?: string
-    ignoreSubjects?: string[]
-    sortDirection: 'asc' | 'desc'
-    lastReviewedBy?: string
-    sortField: 'lastReviewedAt' | 'lastReportedAt'
-    tags: string[]
-    excludeTags: string[]
-    collections: string[]
-    subjectType?: string
-  }) {
-    let builder = this.db.db.selectFrom('moderation_subject_status').selectAll()
+  }> {
+    let builder = moderationSubjectStatusQueryBuilder(this.db.db)
+
     const { ref } = this.db.db.dynamic
 
     if (subject) {
@@ -919,14 +861,18 @@ export class ModerationService {
       if (!includeAllUserRecords) {
         builder = builder.where((qb) =>
           subjectInfo.recordPath
-            ? qb.where('recordPath', '=', subjectInfo.recordPath)
-            : qb.where('recordPath', '=', ''),
+            ? qb.where(
+                'moderation_subject_status.recordPath',
+                '=',
+                subjectInfo.recordPath,
+              )
+            : qb.where('moderation_subject_status.recordPath', '=', ''),
         )
       }
     } else if (subjectType === 'account') {
-      builder = builder.where('recordPath', '=', '')
+      builder = builder.where('moderation_subject_status.recordPath', '=', '')
     } else if (subjectType === 'record') {
-      builder = builder.where('recordPath', '!=', '')
+      builder = builder.where('moderation_subject_status.recordPath', '!=', '')
     }
 
     // Only fetch items that belongs to the specified queue when specified
@@ -948,110 +894,216 @@ export class ModerationService {
     }
 
     // If subjectType is set to 'account' let that take priority and ignore collections filter
-    if (collections.length && subjectType !== 'account') {
-      builder = builder.where('recordPath', '!=', '').where((qb) => {
-        collections.forEach((collection) => {
-          qb = qb.orWhere('recordPath', 'like', `${collection}/%`)
+    if (subjectType !== 'account' && collections?.length) {
+      builder = builder
+        .where('moderation_subject_status.recordPath', '!=', '')
+        .where((qb) => {
+          for (const collection of collections) {
+            qb = qb.orWhere(
+              'moderation_subject_status.recordPath',
+              'like',
+              `${collection}/%`,
+            )
+          }
+          return qb
         })
-        return qb
-      })
     }
 
     if (ignoreSubjects?.length) {
       builder = builder
-        .where('did', 'not in', ignoreSubjects)
-        .where('recordPath', 'not in', ignoreSubjects)
+        .where('moderation_subject_status.did', 'not in', ignoreSubjects)
+        .where('moderation_subject_status.recordPath', 'not in', ignoreSubjects)
     }
 
-    if (reviewState) {
-      builder = builder.where('reviewState', '=', reviewState)
+    const reviewStateNormalized = getReviewState(reviewState)
+    if (reviewStateNormalized) {
+      builder = builder.where(
+        'moderation_subject_status.reviewState',
+        '=',
+        reviewStateNormalized,
+      )
     }
 
     if (lastReviewedBy) {
-      builder = builder.where('lastReviewedBy', '=', lastReviewedBy)
+      builder = builder.where(
+        'moderation_subject_status.lastReviewedBy',
+        '=',
+        lastReviewedBy,
+      )
     }
 
     if (reviewedAfter) {
-      builder = builder.where('lastReviewedAt', '>', reviewedAfter)
+      builder = builder.where(
+        'moderation_subject_status.lastReviewedAt',
+        '>',
+        reviewedAfter,
+      )
     }
 
     if (reviewedBefore) {
-      builder = builder.where('lastReviewedAt', '<', reviewedBefore)
+      builder = builder.where(
+        'moderation_subject_status.lastReviewedAt',
+        '<',
+        reviewedBefore,
+      )
     }
 
     if (hostingUpdatedAfter) {
-      builder = builder.where('hostingUpdatedAt', '>', hostingUpdatedAfter)
+      builder = builder.where(
+        'moderation_subject_status.hostingUpdatedAt',
+        '>',
+        hostingUpdatedAfter,
+      )
     }
 
     if (hostingUpdatedBefore) {
-      builder = builder.where('hostingUpdatedAt', '<', hostingUpdatedBefore)
+      builder = builder.where(
+        'moderation_subject_status.hostingUpdatedAt',
+        '<',
+        hostingUpdatedBefore,
+      )
     }
 
     if (hostingDeletedAfter) {
-      builder = builder.where('hostingDeletedAt', '>', hostingDeletedAfter)
+      builder = builder.where(
+        'moderation_subject_status.hostingDeletedAt',
+        '>',
+        hostingDeletedAfter,
+      )
     }
 
     if (hostingDeletedBefore) {
-      builder = builder.where('hostingDeletedAt', '<', hostingDeletedBefore)
+      builder = builder.where(
+        'moderation_subject_status.hostingDeletedAt',
+        '<',
+        hostingDeletedBefore,
+      )
     }
 
     if (hostingStatuses?.length) {
-      builder = builder.where('hostingStatus', 'in', hostingStatuses)
+      builder = builder.where(
+        'moderation_subject_status.hostingStatus',
+        'in',
+        hostingStatuses,
+      )
     }
 
     if (reportedAfter) {
-      builder = builder.where('lastReviewedAt', '>', reportedAfter)
+      builder = builder.where(
+        'moderation_subject_status.lastReviewedAt',
+        '>',
+        reportedAfter,
+      )
     }
 
     if (reportedBefore) {
-      builder = builder.where('lastReportedAt', '<', reportedBefore)
+      builder = builder.where(
+        'moderation_subject_status.lastReportedAt',
+        '<',
+        reportedBefore,
+      )
     }
 
     if (takendown) {
-      builder = builder.where('takendown', '=', true)
+      builder = builder.where('moderation_subject_status.takendown', '=', true)
     }
 
     if (appealed !== undefined) {
       builder =
         appealed === false
-          ? builder.where('appealed', 'is', null)
-          : builder.where('appealed', '=', appealed)
+          ? builder.where('moderation_subject_status.appealed', 'is', null)
+          : builder.where('moderation_subject_status.appealed', '=', appealed)
     }
 
     if (!includeMuted) {
       builder = builder.where((qb) =>
         qb
-          .where('muteUntil', '<', new Date().toISOString())
-          .orWhere('muteUntil', 'is', null),
+          .where(
+            'moderation_subject_status.muteUntil',
+            '<',
+            new Date().toISOString(),
+          )
+          .orWhere('moderation_subject_status.muteUntil', 'is', null),
       )
     }
 
     if (onlyMuted) {
       builder = builder.where((qb) =>
         qb
-          .where('muteUntil', '>', new Date().toISOString())
-          .orWhere('muteReportingUntil', '>', new Date().toISOString()),
+          .where(
+            'moderation_subject_status.muteUntil',
+            '>',
+            new Date().toISOString(),
+          )
+          .orWhere(
+            'moderation_subject_status.muteReportingUntil',
+            '>',
+            new Date().toISOString(),
+          ),
       )
     }
 
-    if (tags.length) {
-      builder = this.applyTagFilter(builder, tags)
+    // ["tag1", "tag2 && tag3", "tag4"] => [["tag1"], ["tag2", "tag3"], ["tag4"]]
+    const conditions = parseTags(tags)
+    if (conditions?.length) {
+      // [["tag1"], ["tag2", "tag3"], ["tag4"]] => (tags ? 'tag1') OR (tags ? 'tag2' AND tags ? 'tag3') OR (tags ? 'tag4')
+      builder = builder.where((qb) => {
+        for (const subTags of conditions) {
+          // OR between every conditions items (subTags)
+          qb = qb.orWhere((qb) => {
+            // AND between every subTags items (subTag)
+            for (const subTag of subTags) {
+              qb = qb.where(
+                sql`${ref('moderation_subject_status.tags')} ? ${subTag}`,
+              )
+            }
+            return qb
+          })
+        }
+        return qb
+      })
     }
 
-    if (excludeTags.length) {
+    if (excludeTags?.length) {
       builder = builder.where((qb) =>
         qb
           .where(
-            sql`NOT(${ref(
-              'moderation_subject_status.tags',
-            )} ?| array[${sql.join(excludeTags)}]::TEXT[])`,
+            sql`NOT(${ref('moderation_subject_status.tags')} ?| array[${sql.join(excludeTags)}]::TEXT[])`,
           )
           .orWhere('tags', 'is', null),
       )
     }
 
+    if (minAccountSuspendCount != null && minAccountSuspendCount > 0) {
+      builder = builder.where(
+        'account_events_stats.suspendCount',
+        '>=',
+        minAccountSuspendCount,
+      )
+    }
+
+    if (minTakendownRecordsCount != null && minTakendownRecordsCount > 0) {
+      builder = builder.where(
+        'account_record_status_stats.takendownCount',
+        '>=',
+        minTakendownRecordsCount,
+      )
+    }
+
+    if (minReportedRecordsCount != null && minReportedRecordsCount > 0) {
+      builder = builder.where(
+        'account_record_events_stats.reportedCount',
+        '>=',
+        minReportedRecordsCount,
+      )
+    }
+
     const keyset = new StatusKeyset(
-      ref(`moderation_subject_status.${sortField}`),
+      sortField === 'reportedRecordsCount'
+        ? ref(`account_record_events_stats.reportedCount`)
+        : sortField === 'takendownRecordsCount'
+          ? ref(`account_record_status_stats.takendownCount`)
+          : ref(`moderation_subject_status.${sortField}`),
       ref('moderation_subject_status.id'),
     )
     const paginatedBuilder = paginate(builder, {
@@ -1067,13 +1119,12 @@ export class ModerationService {
     const infos = await this.views.getAccoutInfosByDid(
       results.map((r) => r.did),
     )
-    const resultsWithHandles = results.map((r) => ({
-      ...r,
-      handle: infos.get(r.did)?.handle ?? INVALID_HANDLE,
-    }))
 
     return {
-      statuses: resultsWithHandles,
+      statuses: results.map((r) => ({
+        ...r,
+        handle: infos.get(r.did)?.handle ?? INVALID_HANDLE,
+      })),
       cursor: keyset.packFromResult(results),
     }
   }
@@ -1195,6 +1246,18 @@ export class ModerationService {
   }
 }
 
+const parseTags = (tags?: string[]) =>
+  tags
+    ?.map((tag) =>
+      tag
+        .split(/\s*&&\s*/g)
+        .map((subTag) => subTag.trim())
+        // Ignore invalid syntax ("", "tag1 &&", "&& tag2", "tag1 && && tag2", etc.)
+        .filter(Boolean),
+    )
+    // Ignore invalid items
+    .filter((subTags): subTags is [string, ...string[]] => subTags.length > 0)
+
 const isSafeUrl = (url: URL) => {
   if (url.protocol !== 'https:') return false
   if (!url.hostname || url.hostname === 'localhost') return false

package/src/mod-service/status.ts

@@ -1,18 +1,18 @@
 // This may require better organization but for now, just dumping functions here containing DB queries for moderation status
 
+import { HOUR } from '@atproto/common'
 import { AtUri } from '@atproto/syntax'
 import { Database } from '../db'
-import { ModerationSubjectStatus } from '../db/schema/moderation_subject_status'
+import DatabaseSchema from '../db/schema'
+import { jsonb } from '../db/types'
+import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs'
 import {
-  REVIEWOPEN,
   REVIEWCLOSED,
   REVIEWESCALATED,
   REVIEWNONE,
+  REVIEWOPEN,
 } from '../lexicon/types/tools/ozone/moderation/defs'
 import { ModerationEventRow, ModerationSubjectStatusRow } from './types'
-import { HOUR } from '@atproto/common'
-import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs'
-import { jsonb } from '../db/types'
 
 const getSubjectStatusForModerationEvent = ({
   currentStatus,
@@ -203,6 +203,56 @@ const getSubjectStatusForRecordEvent = ({
   return {}
 }
 
+export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => {
+  // @NOTE: Using select() instead of selectAll() below because the materialized
+  // views might be incomplete, and we don't want the null `did` columns to
+  // interfere with the (never null) `did` column from the
+  // `moderation_subject_status` table in the results
+  return db
+    .selectFrom('moderation_subject_status')
+    .selectAll('moderation_subject_status')
+    .leftJoin('account_events_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_events_stats.subjectDid',
+      ),
+    )
+    .select([
+      'account_events_stats.takedownCount',
+      'account_events_stats.suspendCount',
+      'account_events_stats.escalateCount',
+      'account_events_stats.reportCount',
+      'account_events_stats.appealCount',
+    ])
+    .leftJoin('account_record_events_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_record_events_stats.subjectDid',
+      ),
+    )
+    .select([
+      'account_record_events_stats.totalReports',
+      'account_record_events_stats.reportedCount',
+      'account_record_events_stats.escalatedCount',
+      'account_record_events_stats.appealedCount',
+    ])
+    .leftJoin('account_record_status_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_record_status_stats.did',
+      ),
+    )
+    .select([
+      'account_record_status_stats.subjectCount',
+      'account_record_status_stats.pendingCount',
+      'account_record_status_stats.processedCount',
+      'account_record_status_stats.takendownCount',
+    ])
+}
+
 // Based on a given moderation action event, this function will update the moderation status of the subject
 // If there's no existing status, it will create one
 // If the action event does not affect the status, it will do nothing
@@ -393,29 +443,6 @@ export const adjustModerationSubjectStatus = async (
   return status || null
 }
 
-type ModerationSubjectStatusFilter =
-  | Pick<ModerationSubjectStatus, 'did'>
-  | Pick<ModerationSubjectStatus, 'did' | 'recordPath'>
-  | Pick<ModerationSubjectStatus, 'did' | 'recordPath' | 'recordCid'>
-export const getModerationSubjectStatus = async (
-  db: Database,
-  filters: ModerationSubjectStatusFilter,
-) => {
-  let builder = db.db
-    .selectFrom('moderation_subject_status')
-    // DID will always be passed at the very least
-    .where('did', '=', filters.did)
-    .where('recordPath', '=', 'recordPath' in filters ? filters.recordPath : '')
-
-  if ('recordCid' in filters) {
-    builder = builder.where('recordCid', '=', filters.recordCid)
-  } else {
-    builder = builder.where('recordCid', 'is', null)
-  }
-
-  return builder.executeTakeFirst()
-}
-
 export const getStatusIdentifierFromSubject = (
   subject: string | AtUri,
 ): { did: string; recordPath: string } => {

package/src/mod-service/types.ts

@@ -18,8 +18,29 @@ export type ModerationEventRowWithHandle = ModerationEventRow & {
   creatorHandle?: string | null
 }
 export type ModerationSubjectStatusRow = Selectable<ModerationSubjectStatus>
+export type ModerationSubjectStatusRowWithStats = ModerationSubjectStatusRow & {
+  // account_events_stats
+  takedownCount: number | null
+  suspendCount: number | null
+  escalateCount: number | null
+  reportCount: number | null
+  appealCount: number | null
+
+  // account_record_events_stats
+  totalReports: number | null
+  reportedCount: number | null
+  escalatedCount: number | null
+  appealedCount: number | null
+
+  // account_record_status_stats
+  subjectCount: number | null
+  pendingCount: number | null
+  processedCount: number | null
+  takendownCount: number | null
+}
+
 export type ModerationSubjectStatusRowWithHandle =
-  ModerationSubjectStatusRow & { handle: string | null }
+  ModerationSubjectStatusRowWithStats & { handle: string | null }
 
 export type ModEventType =
   | ToolsOzoneModerationDefs.ModEventTakedown