@atproto/ozone 0.1.69 → 0.1.71

package/src/mod-service/status.ts

@@ -1,18 +1,18 @@
 // This may require better organization but for now, just dumping functions here containing DB queries for moderation status
 
+import { HOUR } from '@atproto/common'
 import { AtUri } from '@atproto/syntax'
 import { Database } from '../db'
-import { ModerationSubjectStatus } from '../db/schema/moderation_subject_status'
+import DatabaseSchema from '../db/schema'
+import { jsonb } from '../db/types'
+import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs'
 import {
-  REVIEWOPEN,
   REVIEWCLOSED,
   REVIEWESCALATED,
   REVIEWNONE,
+  REVIEWOPEN,
 } from '../lexicon/types/tools/ozone/moderation/defs'
 import { ModerationEventRow, ModerationSubjectStatusRow } from './types'
-import { HOUR } from '@atproto/common'
-import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs'
-import { jsonb } from '../db/types'
 
 const getSubjectStatusForModerationEvent = ({
   currentStatus,
@@ -203,6 +203,56 @@ const getSubjectStatusForRecordEvent = ({
   return {}
 }
 
+export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => {
+  // @NOTE: Using select() instead of selectAll() below because the materialized
+  // views might be incomplete, and we don't want the null `did` columns to
+  // interfere with the (never null) `did` column from the
+  // `moderation_subject_status` table in the results
+  return db
+    .selectFrom('moderation_subject_status')
+    .selectAll('moderation_subject_status')
+    .leftJoin('account_events_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_events_stats.subjectDid',
+      ),
+    )
+    .select([
+      'account_events_stats.takedownCount',
+      'account_events_stats.suspendCount',
+      'account_events_stats.escalateCount',
+      'account_events_stats.reportCount',
+      'account_events_stats.appealCount',
+    ])
+    .leftJoin('account_record_events_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_record_events_stats.subjectDid',
+      ),
+    )
+    .select([
+      'account_record_events_stats.totalReports',
+      'account_record_events_stats.reportedCount',
+      'account_record_events_stats.escalatedCount',
+      'account_record_events_stats.appealedCount',
+    ])
+    .leftJoin('account_record_status_stats', (join) =>
+      join.onRef(
+        'moderation_subject_status.did',
+        '=',
+        'account_record_status_stats.did',
+      ),
+    )
+    .select([
+      'account_record_status_stats.subjectCount',
+      'account_record_status_stats.pendingCount',
+      'account_record_status_stats.processedCount',
+      'account_record_status_stats.takendownCount',
+    ])
+}
+
 // Based on a given moderation action event, this function will update the moderation status of the subject
 // If there's no existing status, it will create one
 // If the action event does not affect the status, it will do nothing
@@ -393,29 +443,6 @@ export const adjustModerationSubjectStatus = async (
   return status || null
 }
 
-type ModerationSubjectStatusFilter =
-  | Pick<ModerationSubjectStatus, 'did'>
-  | Pick<ModerationSubjectStatus, 'did' | 'recordPath'>
-  | Pick<ModerationSubjectStatus, 'did' | 'recordPath' | 'recordCid'>
-export const getModerationSubjectStatus = async (
-  db: Database,
-  filters: ModerationSubjectStatusFilter,
-) => {
-  let builder = db.db
-    .selectFrom('moderation_subject_status')
-    // DID will always be passed at the very least
-    .where('did', '=', filters.did)
-    .where('recordPath', '=', 'recordPath' in filters ? filters.recordPath : '')
-
-  if ('recordCid' in filters) {
-    builder = builder.where('recordCid', '=', filters.recordCid)
-  } else {
-    builder = builder.where('recordCid', 'is', null)
-  }
-
-  return builder.executeTakeFirst()
-}
-
 export const getStatusIdentifierFromSubject = (
   subject: string | AtUri,
 ): { did: string; recordPath: string } => {

package/src/mod-service/types.ts

@@ -18,8 +18,29 @@ export type ModerationEventRowWithHandle = ModerationEventRow & {
   creatorHandle?: string | null
 }
 export type ModerationSubjectStatusRow = Selectable<ModerationSubjectStatus>
+export type ModerationSubjectStatusRowWithStats = ModerationSubjectStatusRow & {
+  // account_events_stats
+  takedownCount: number | null
+  suspendCount: number | null
+  escalateCount: number | null
+  reportCount: number | null
+  appealCount: number | null
+
+  // account_record_events_stats
+  totalReports: number | null
+  reportedCount: number | null
+  escalatedCount: number | null
+  appealedCount: number | null
+
+  // account_record_status_stats
+  subjectCount: number | null
+  pendingCount: number | null
+  processedCount: number | null
+  takendownCount: number | null
+}
+
 export type ModerationSubjectStatusRowWithHandle =
-  ModerationSubjectStatusRow & { handle: string | null }
+  ModerationSubjectStatusRowWithStats & { handle: string | null }
 
 export type ModEventType =
   | ToolsOzoneModerationDefs.ModEventTakedown

package/src/mod-service/views.ts

@@ -1,10 +1,6 @@
 import { sql } from 'kysely'
 import { AtUri, INVALID_HANDLE, normalizeDatetimeAlways } from '@atproto/syntax'
-import {
-  AtpAgent,
-  AppBskyFeedDefs,
-  ToolsOzoneModerationDefs,
-} from '@atproto/api'
+import { AtpAgent, AppBskyFeedDefs } from '@atproto/api'
 import { dedupeStrs } from '@atproto/common'
 import { BlobRef } from '@atproto/lexicon'
 import { Keypair } from '@atproto/crypto'
@@ -12,7 +8,6 @@ import { Database } from '../db'
 import {
   ModEventView,
   RepoView,
-  RepoViewDetail,
   RecordView,
   RecordViewDetail,
   BlobView,
@@ -34,6 +29,7 @@ import { dbLogger } from '../logger'
 import { httpLogger } from '../logger'
 import { ParsedLabelers } from '../util'
 import { ids } from '../lexicon/lexicons'
+import { moderationSubjectStatusQueryBuilder } from './status'
 
 export type AuthHeaders = {
   headers: {
@@ -481,8 +477,9 @@ export class ModerationViews {
   async blob(blobs: BlobRef[]): Promise<BlobView[]> {
     if (!blobs.length) return []
     const { ref } = this.db.db.dynamic
-    const modStatusResults = await this.db.db
-      .selectFrom('moderation_subject_status')
+    const modStatusResults = await moderationSubjectStatusQueryBuilder(
+      this.db.db,
+    )
       .where(
         sql<string>`${ref(
           'moderation_subject_status.blobCids',
@@ -529,10 +526,10 @@ export class ModerationViews {
     await Promise.all(
       res.map(async (labelRow) => {
         const signedLabel = await this.formatLabelAndEnsureSig(labelRow)
-        if (!labels.has(labelRow.uri)) {
-          labels.set(labelRow.uri, [])
-        }
-        labels.get(labelRow.uri)?.push(signedLabel)
+
+        const current = labels.get(labelRow.uri)
+        if (current) current.push(signedLabel)
+        else labels.set(labelRow.uri, [signedLabel])
       }),
     )
     return labels
@@ -559,51 +556,39 @@ export class ModerationViews {
   async getSubjectStatus(
     subjects: string[],
   ): Promise<Map<string, ModerationSubjectStatusRowWithHandle>> {
-    const parsedSubjects = subjects.map((subject) => parseSubjectId(subject))
-    const filterForSubject = (did: string, recordPath?: string) => {
-      return (clause: any) => {
-        clause = clause
-          .where('moderation_subject_status.did', '=', did)
-          .where('moderation_subject_status.recordPath', '=', recordPath || '')
-        return clause
-      }
-      // TODO: Fix the typing here?
-    }
-
-    const builder = this.db.db
-      .selectFrom('moderation_subject_status')
-      .where((clause) => {
-        parsedSubjects.forEach((subject, i) => {
-          const applySubjectFilter = filterForSubject(
-            subject.did,
-            subject.recordPath,
+    if (!subjects.length) return new Map()
+
+    const parsedSubjects = subjects.map(parseSubjectId)
+
+    const builder = moderationSubjectStatusQueryBuilder(this.db.db)
+      //
+      .where((qb) => {
+        for (const sub of parsedSubjects) {
+          qb = qb.orWhere((qb) =>
+            qb
+              .where('moderation_subject_status.did', '=', sub.did)
+              .where(
+                'moderation_subject_status.recordPath',
+                '=',
+                sub.recordPath ?? '',
+              ),
           )
-          if (i === 0) {
-            clause = clause.where(applySubjectFilter)
-          } else {
-            clause = clause.orWhere(applySubjectFilter)
-          }
-        })
-
-        return clause
+        }
+        return qb
       })
-      .selectAll()
 
     const [statusRes, accountsByDid] = await Promise.all([
       builder.execute(),
       this.getAccoutInfosByDid(parsedSubjects.map((s) => s.did)),
     ])
 
-    return statusRes.reduce((acc, cur) => {
-      const subject = cur.recordPath
-        ? formatSubjectId(cur.did, cur.recordPath)
-        : cur.did
-      const handle = accountsByDid.get(cur.did)?.handle
-      return acc.set(subject, {
-        ...cur,
-        handle: handle ?? INVALID_HANDLE,
-      })
-    }, new Map<string, ModerationSubjectStatusRowWithHandle>())
+    return new Map(
+      statusRes.map((row): [string, ModerationSubjectStatusRowWithHandle] => {
+        const subjectId = formatSubjectId(row.did, row.recordPath)
+        const handle = accountsByDid.get(row.did)?.handle ?? INVALID_HANDLE
+        return [subjectId, { ...row, handle }]
+      }),
+    )
   }
 
   formatSubjectStatus(
@@ -628,6 +613,35 @@ export class ModerationViews {
       subjectBlobCids: status.blobCids || [],
       tags: status.tags || [],
       subject: subjectFromStatusRow(status).lex(),
+
+      accountStats: {
+        // Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots)
+        $type: 'tools.ozone.moderation.defs#accountStats',
+
+        // account_events_stats
+        reportCount: status.reportCount ?? undefined,
+        appealCount: status.appealCount ?? undefined,
+        suspendCount: status.suspendCount ?? undefined,
+        takedownCount: status.takedownCount ?? undefined,
+        escalateCount: status.escalateCount ?? undefined,
+      },
+
+      recordsStats: {
+        // Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots)
+        $type: 'tools.ozone.moderation.defs#recordStats',
+
+        // account_record_events_stats
+        totalReports: status.totalReports ?? undefined,
+        reportedCount: status.reportedCount ?? undefined,
+        escalatedCount: status.escalatedCount ?? undefined,
+        appealedCount: status.appealedCount ?? undefined,
+
+        // account_record_status_stats
+        subjectCount: status.subjectCount ?? undefined,
+        pendingCount: status.pendingCount ?? undefined,
+        processedCount: status.processedCount ?? undefined,
+        takendownCount: status.takendownCount ?? undefined,
+      },
     }
 
     if (status.recordPath !== '') {
@@ -677,7 +691,7 @@ type RecordInfo = {
   indexedAt: string
 }
 
-function parseSubjectId(subject: string) {
+function parseSubjectId(subject: string): { did: string; recordPath?: string } {
   if (subject.startsWith('did:')) {
     return { did: subject }
   }

package/src/util.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import { createRetryable } from '@atproto/common'
 import { ResponseType, XRPCError } from '@atproto/xrpc'
 import { parseList } from 'structured-headers'
@@ -83,3 +84,147 @@ export const formatLabelerHeader = (parsed: ParsedLabelers): string => {
   )
   return parts.join(',')
 }
+
+/**
+ * Utility function similar to `setInterval()`. The main difference is that the
+ * execution is controlled through a signal and that the function will wait for
+ * `interval` milliseconds *between* the end of the previous execution and the
+ * start of the next one (instead of starting the execution every `interval`
+ * milliseconds), ensuring that the function is not running concurrently.
+ *
+ * @param fn The function to execute. That function must not throw any error
+ * other than {@link signal}'s {@link AbortSignal.reason} or an {@link Error}
+ * that has the {@link signal}'s {@link AbortSignal.reason} as its
+ * {@link Error.cause}.
+ *
+ * @returns A promise that resolves when the signal is aborted, and the last
+ * execution is done.
+ *
+ * @throws {AbortSignal['reason']} if the {@link signal} is already aborted.
+ * @throws {TypeError} if {@link fn} throws an unexpected error (with the
+ * unexpected error as the {@link Error.cause}).
+ */
+export async function startInterval(
+  fn: (signal: AbortSignal) => void | Promise<void>,
+  interval: number,
+  signal: AbortSignal,
+  runImmediately = false,
+) {
+  signal.throwIfAborted()
+
+  // Renaming for clarity
+  const inputSignal = signal
+
+  const intervalController = new AbortController()
+  const intervalSignal = intervalController.signal
+
+  return new Promise<void>((resolve, reject) => {
+    let timer: NodeJS.Timeout | undefined
+
+    const run = async () => {
+      // Cloning the signal for this particular run to prevent memory leaks
+      const runController = boundAbortController(intervalSignal)
+      const runSignal = runController.signal
+
+      try {
+        await fn(runSignal)
+      } catch (err) {
+        if (err != null && isCausedBySignal(err, runSignal)) {
+          // Silently ignore the error if it is caused by the signal. At this
+          // point, the interval controller was aborted, which will cause the
+          // promise to resolve in the "finally" block bellow.
+        } else {
+          // Invalid behavior: stop the interval and reject the promise.
+          const error = new TypeError('Unexpected error', { cause: err })
+
+          // Rejecting here will make `resolve()` in the "finally" block to be a
+          // no-op. Rejecting before aborting the controller to ensure the
+          // promise does not get resolved by the `abort` event listeners.
+          reject(error)
+
+          // Using `error` as abort reason to avoid creating an AbortError.
+          intervalController.abort(error)
+        }
+      } finally {
+        // Cleanup the listeners added by `boundAbortController`
+        runController.abort()
+
+        if (intervalSignal.aborted) resolve()
+        else schedule()
+      }
+    }
+
+    const schedule = () => {
+      assert(timer === undefined, 'unexpected state')
+      timer = setTimeout(() => {
+        timer = undefined // "running" state
+        void run()
+      }, interval)
+    }
+
+    inputSignal.addEventListener(
+      'abort',
+      // This function will only be called if the `inputSignal` is aborted
+      // before the interval controller is aborted.
+      () => {
+        // Stop the interval, using the input signal's reason
+        intervalController.abort(inputSignal.reason)
+
+        if (timer === undefined) {
+          // `fn` is currently running; `run`'s finally block will resolve the
+          // promise.
+        } else {
+          // The execution was scheduled but not started yet. Clear the timer
+          // and resolve the promise.
+          clearTimeout(timer)
+          resolve()
+        }
+      },
+      // Remove the listener whenever the interval is aborted.
+      { signal: intervalSignal },
+    )
+
+    if (runImmediately) void run()
+    else schedule()
+  })
+}
+
+/**
+ * Determines whether the cause of an error is a signal's reason
+ */
+export function isCausedBySignal(err: unknown, signal: AbortSignal) {
+  if (!signal.aborted) return false
+  if (signal.reason == null) return false // Ignore nullish reasons
+  return (
+    err === signal.reason ||
+    (err instanceof Error && err.cause === signal.reason)
+  )
+}
+
+/**
+ * Creates an AbortController that will be aborted when any of the given signals
+ * is aborted.
+ *
+ * @note Make sure to call `abortController.abort()` when you are done with
+ * the controller to avoid memory leaks.
+ *
+ * @throws if any of the input signals is already aborted.
+ */
+export function boundAbortController(
+  ...signals: readonly (AbortSignal | undefined | null)[]
+): AbortController {
+  for (const signal of signals) {
+    signal?.throwIfAborted()
+  }
+
+  const abortController = new AbortController()
+  const abort = function (event: Event) {
+    abortController.abort((event.target as AbortSignal)?.reason)
+  }
+
+  for (const signal of signals) {
+    signal?.addEventListener('abort', abort, { signal: abortController.signal })
+  }
+
+  return abortController
+}

package/tests/__snapshots__/get-record.test.ts.snap

@@ -28,6 +28,9 @@ Object {
   ],
   "moderation": Object {
     "subjectStatus": Object {
+      "accountStats": Object {
+        "$type": "tools.ozone.moderation.defs#accountStats",
+      },
       "createdAt": "1970-01-01T00:00:00.000Z",
       "hosting": Object {
         "$type": "tools.ozone.moderation.defs#recordHosting",
@@ -37,6 +40,17 @@ Object {
       "lastReportedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedBy": "user(1)",
+      "recordsStats": Object {
+        "$type": "tools.ozone.moderation.defs#recordStats",
+        "appealedCount": 0,
+        "escalatedCount": 0,
+        "pendingCount": 0,
+        "processedCount": 1,
+        "reportedCount": 1,
+        "subjectCount": 1,
+        "takendownCount": 1,
+        "totalReports": 2,
+      },
       "reviewState": "tools.ozone.moderation.defs#reviewClosed",
       "subject": Object {
         "$type": "com.atproto.repo.strongRef",
@@ -134,6 +148,9 @@ Object {
   ],
   "moderation": Object {
     "subjectStatus": Object {
+      "accountStats": Object {
+        "$type": "tools.ozone.moderation.defs#accountStats",
+      },
       "createdAt": "1970-01-01T00:00:00.000Z",
       "hosting": Object {
         "$type": "tools.ozone.moderation.defs#recordHosting",
@@ -143,6 +160,17 @@ Object {
       "lastReportedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedBy": "user(1)",
+      "recordsStats": Object {
+        "$type": "tools.ozone.moderation.defs#recordStats",
+        "appealedCount": 0,
+        "escalatedCount": 0,
+        "pendingCount": 0,
+        "processedCount": 1,
+        "reportedCount": 1,
+        "subjectCount": 1,
+        "takendownCount": 1,
+        "totalReports": 2,
+      },
       "reviewState": "tools.ozone.moderation.defs#reviewClosed",
       "subject": Object {
         "$type": "com.atproto.repo.strongRef",

package/tests/__snapshots__/get-records.test.ts.snap

@@ -31,6 +31,9 @@ Object {
       ],
       "moderation": Object {
         "subjectStatus": Object {
+          "accountStats": Object {
+            "$type": "tools.ozone.moderation.defs#accountStats",
+          },
           "createdAt": "1970-01-01T00:00:00.000Z",
           "hosting": Object {
             "$type": "tools.ozone.moderation.defs#recordHosting",
@@ -40,6 +43,17 @@ Object {
           "lastReportedAt": "1970-01-01T00:00:00.000Z",
           "lastReviewedAt": "1970-01-01T00:00:00.000Z",
           "lastReviewedBy": "user(1)",
+          "recordsStats": Object {
+            "$type": "tools.ozone.moderation.defs#recordStats",
+            "appealedCount": 0,
+            "escalatedCount": 0,
+            "pendingCount": 0,
+            "processedCount": 1,
+            "reportedCount": 1,
+            "subjectCount": 1,
+            "takendownCount": 1,
+            "totalReports": 2,
+          },
           "reviewState": "tools.ozone.moderation.defs#reviewClosed",
           "subject": Object {
             "$type": "com.atproto.repo.strongRef",

package/tests/__snapshots__/get-repo.test.ts.snap

@@ -22,6 +22,14 @@ Object {
   ],
   "moderation": Object {
     "subjectStatus": Object {
+      "accountStats": Object {
+        "$type": "tools.ozone.moderation.defs#accountStats",
+        "appealCount": 0,
+        "escalateCount": 0,
+        "reportCount": 2,
+        "suspendCount": 0,
+        "takedownCount": 1,
+      },
       "createdAt": "1970-01-01T00:00:00.000Z",
       "hosting": Object {
         "$type": "tools.ozone.moderation.defs#accountHosting",
@@ -31,6 +39,9 @@ Object {
       "lastReportedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedAt": "1970-01-01T00:00:00.000Z",
       "lastReviewedBy": "user(1)",
+      "recordsStats": Object {
+        "$type": "tools.ozone.moderation.defs#recordStats",
+      },
       "reviewState": "tools.ozone.moderation.defs#reviewClosed",
       "subject": Object {
         "$type": "com.atproto.admin.defs#repoRef",

package/tests/__snapshots__/get-repos.test.ts.snap

@@ -25,6 +25,14 @@ Object {
       ],
       "moderation": Object {
         "subjectStatus": Object {
+          "accountStats": Object {
+            "$type": "tools.ozone.moderation.defs#accountStats",
+            "appealCount": 0,
+            "escalateCount": 0,
+            "reportCount": 2,
+            "suspendCount": 0,
+            "takedownCount": 1,
+          },
           "createdAt": "1970-01-01T00:00:00.000Z",
           "hosting": Object {
             "$type": "tools.ozone.moderation.defs#accountHosting",
@@ -34,6 +42,9 @@ Object {
           "lastReportedAt": "1970-01-01T00:00:00.000Z",
           "lastReviewedAt": "1970-01-01T00:00:00.000Z",
           "lastReviewedBy": "user(1)",
+          "recordsStats": Object {
+            "$type": "tools.ozone.moderation.defs#recordStats",
+          },
           "reviewState": "tools.ozone.moderation.defs#reviewClosed",
           "subject": Object {
             "$type": "com.atproto.admin.defs#repoRef",

package/tests/__snapshots__/moderation-events.test.ts.snap

@@ -18,6 +18,14 @@ Object {
     "indexedAt": "1970-01-01T00:00:00.000Z",
     "moderation": Object {
       "subjectStatus": Object {
+        "accountStats": Object {
+          "$type": "tools.ozone.moderation.defs#accountStats",
+          "appealCount": 0,
+          "escalateCount": 1,
+          "reportCount": 4,
+          "suspendCount": 0,
+          "takedownCount": 0,
+        },
         "createdAt": "1970-01-01T00:00:00.000Z",
         "hosting": Object {
           "$type": "tools.ozone.moderation.defs#accountHosting",
@@ -27,6 +35,17 @@ Object {
         "lastReportedAt": "1970-01-01T00:00:00.000Z",
         "lastReviewedAt": "1970-01-01T00:00:00.000Z",
         "lastReviewedBy": "user(1)",
+        "recordsStats": Object {
+          "$type": "tools.ozone.moderation.defs#recordStats",
+          "appealedCount": 0,
+          "escalatedCount": 0,
+          "pendingCount": 2,
+          "processedCount": 0,
+          "reportedCount": 2,
+          "subjectCount": 2,
+          "takendownCount": 0,
+          "totalReports": 3,
+        },
         "reviewState": "tools.ozone.moderation.defs#reviewEscalated",
         "subject": Object {
           "$type": "com.atproto.admin.defs#repoRef",