@atproto/ozone 0.0.9 → 0.0.10

package/src/lexicon/types/com/atproto/admin/queryModerationEvents.ts

@@ -31,6 +31,10 @@ export interface QueryParams {
   addedLabels?: string[]
   /** If specified, only events where all of these labels were removed are returned */
   removedLabels?: string[]
+  /** If specified, only events where all of these tags were added are returned */
+  addedTags?: string[]
+  /** If specified, only events where all of these tags were removed are returned */
+  removedTags?: string[]
   reportTypes?: string[]
   cursor?: string
 }

package/src/lexicon/types/com/atproto/admin/queryModerationStatuses.ts

@@ -35,6 +35,8 @@ export interface QueryParams {
   /** Get subjects in unresolved appealed status */
   appealed?: boolean
   limit: number
+  tags?: string[]
+  excludeTags?: string[]
   cursor?: string
 }
 

package/src/logger.ts

@@ -5,6 +5,8 @@ export const dbLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('ozone:db')
 export const httpLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('ozone')
+export const langLogger: ReturnType<typeof subsystemLogger> =
+  subsystemLogger('ozone:lang')
 
 export const loggerMiddleware = pinoHttp({
   logger: httpLogger,

package/src/mod-service/index.ts

@@ -12,6 +12,7 @@ import {
   isModEventReport,
   isModEventTakedown,
   isModEventEmail,
+  isModEventTag,
   RepoRef,
   RepoBlobRef,
 } from '../lexicon/types/com/atproto/admin/defs'
@@ -103,6 +104,8 @@ export class ModerationService {
     createdBefore?: string
     addedLabels: string[]
     removedLabels: string[]
+    addedTags: string[]
+    removedTags: string[]
     reportTypes?: string[]
   }): Promise<{ cursor?: string; events: ModerationEventRow[] }> {
     const {
@@ -119,8 +122,11 @@ export class ModerationService {
       createdBefore,
       addedLabels,
       removedLabels,
+      addedTags,
+      removedTags,
       reportTypes,
     } = opts
+    const { ref } = this.db.db.dynamic
     let builder = this.db.db.selectFrom('moderation_event').selectAll()
     if (subject) {
       builder = builder.where((qb) => {
@@ -178,11 +184,18 @@ export class ModerationService {
         builder = builder.where('negateLabelVals', 'ilike', `%${label}%`)
       })
     }
+    if (addedTags.length) {
+      builder = builder.where(sql`${ref('addedTags')} @> ${jsonb(addedTags)}`)
+    }
+    if (removedTags.length) {
+      builder = builder.where(
+        sql`${ref('removedTags')} @> ${jsonb(removedTags)}`,
+      )
+    }
     if (reportTypes?.length) {
       builder = builder.where(sql`meta->>'reportType'`, 'in', reportTypes)
     }
 
-    const { ref } = this.db.db.dynamic
     const keyset = new TimeIdKeyset(
       ref(`moderation_event.createdAt`),
       ref('moderation_event.id'),
@@ -238,7 +251,10 @@ export class ModerationService {
     subject: ModSubject
     createdBy: string
     createdAt?: Date
-  }): Promise<ModerationEventRow> {
+  }): Promise<{
+    event: ModerationEventRow
+    subjectStatus: ModerationSubjectStatusRow | null
+  }> {
     this.db.assertTransaction()
     const { event, subject, createdBy, createdAt = new Date() } = info
 
@@ -253,6 +269,9 @@ export class ModerationService {
 
     const meta: Record<string, string | boolean> = {}
 
+    const addedTags = isModEventTag(event) ? jsonb(event.add) : null
+    const removedTags = isModEventTag(event) ? jsonb(event.remove) : null
+
     if (isModEventReport(event)) {
       meta.reportType = event.reportType
     }
@@ -276,6 +295,8 @@ export class ModerationService {
         createdBy,
         createLabelVals,
         negateLabelVals,
+        addedTags,
+        removedTags,
         durationInHours: event.durationInHours
           ? Number(event.durationInHours)
           : null,
@@ -294,9 +315,13 @@ export class ModerationService {
       .returningAll()
       .executeTakeFirstOrThrow()
 
-    await adjustModerationSubjectStatus(this.db, modEvent, subject.blobCids)
+    const subjectStatus = await adjustModerationSubjectStatus(
+      this.db,
+      modEvent,
+      subject.blobCids,
+    )
 
-    return modEvent
+    return { event: modEvent, subjectStatus }
   }
 
   async getLastReversibleEventForSubject(subject: ReversalSubject) {
@@ -376,7 +401,7 @@ export class ModerationService {
     const isRevertingTakedown =
       action === 'com.atproto.admin.defs#modEventTakedown'
     this.db.assertTransaction()
-    const result = await this.logEvent({
+    const { event } = await this.logEvent({
       event: {
         $type: isRevertingTakedown
           ? 'com.atproto.admin.defs#modEventReverseTakedown'
@@ -396,7 +421,7 @@ export class ModerationService {
       }
     }
 
-    return result
+    return event
   }
 
   async takedownRepo(
@@ -613,7 +638,10 @@ export class ModerationService {
     subject: ModSubject
     reportedBy: string
     createdAt?: Date
-  }): Promise<ModerationEventRow> {
+  }): Promise<{
+    event: ModerationEventRow
+    subjectStatus: ModerationSubjectStatusRow | null
+  }> {
     const {
       reasonType,
       reason,
@@ -622,7 +650,7 @@ export class ModerationService {
       subject,
     } = info
 
-    const event = await this.logEvent({
+    const result = await this.logEvent({
       event: {
         $type: 'com.atproto.admin.defs#modEventReport',
         reportType: reasonType,
@@ -633,7 +661,7 @@ export class ModerationService {
       createdAt,
     })
 
-    return event
+    return result
   }
 
   async getSubjectStatuses({
@@ -652,6 +680,8 @@ export class ModerationService {
     lastReviewedBy,
     sortField,
     subject,
+    tags,
+    excludeTags,
   }: {
     cursor?: string
     limit?: number
@@ -668,8 +698,11 @@ export class ModerationService {
     sortDirection: 'asc' | 'desc'
     lastReviewedBy?: string
     sortField: 'lastReviewedAt' | 'lastReportedAt'
+    tags: string[]
+    excludeTags: string[]
   }) {
     let builder = this.db.db.selectFrom('moderation_subject_status').selectAll()
+    const { ref } = this.db.db.dynamic
 
     if (subject) {
       const subjectInfo = getStatusIdentifierFromSubject(subject)
@@ -731,7 +764,24 @@ export class ModerationService {
       )
     }
 
-    const { ref } = this.db.db.dynamic
+    if (tags.length) {
+      builder = builder.where(
+        sql`${ref('moderation_subject_status.tags')} @> ${jsonb(tags)}`,
+      )
+    }
+
+    if (excludeTags.length) {
+      builder = builder.where((qb) =>
+        qb
+          .where(
+            sql`NOT(${ref('moderation_subject_status.tags')} @> ${jsonb(
+              excludeTags,
+            )})`,
+          )
+          .orWhere('tags', 'is', null),
+      )
+    }
+
     const keyset = new StatusKeyset(
       ref(`moderation_subject_status.${sortField}`),
       ref('moderation_subject_status.id'),

package/src/mod-service/lang.ts

@@ -0,0 +1,82 @@
+import { ModerationService } from '.'
+import { ModSubject } from './subject'
+import { ModerationSubjectStatusRow } from './types'
+import { langLogger as log } from '../logger'
+
+export class ModerationLangService {
+  constructor(private moderationService: ModerationService) {}
+
+  async tagSubjectWithLang({
+    subject,
+    subjectStatus,
+    createdBy,
+  }: {
+    subject: ModSubject
+    createdBy: string
+    subjectStatus: ModerationSubjectStatusRow | null
+  }) {
+    if (
+      subjectStatus &&
+      !subjectStatus.tags?.find((tag) => tag.includes('lang:'))
+    ) {
+      try {
+        const recordLangs = await this.getRecordLang({
+          subject,
+        })
+        await this.moderationService.logEvent({
+          event: {
+            $type: 'com.atproto.admin.defs#modEventTag',
+            add: recordLangs
+              ? recordLangs.map((lang) => `lang:${lang}`)
+              : ['lang:und'],
+            remove: [],
+          },
+          subject,
+          createdBy,
+        })
+      } catch (err) {
+        log.error({ subject, err }, 'Error getting record langs')
+      }
+    }
+  }
+
+  async getRecordLang({
+    subject,
+  }: {
+    subject: ModSubject
+  }): Promise<string[] | null> {
+    const isRecord = subject.isRecord()
+    const langs = new Set<string>()
+
+    if (
+      subject.isRepo() ||
+      (isRecord && subject.uri.endsWith('/app.bsky.actor.profile/self'))
+    ) {
+      const feed = await this.moderationService.views.fetchAuthorFeed(
+        subject.did,
+      )
+      feed.forEach((item) => {
+        const itemLangs = item.post.record['langs'] as string[] | null
+        if (itemLangs?.length) {
+          // Pick the first fragment of the lang code so that instead of `en-US` and `en-GB` we get `en`
+          itemLangs.forEach((lang) => langs.add(lang.split('-')[0]))
+        }
+      })
+    }
+
+    if (isRecord) {
+      const recordByUri = await this.moderationService.views.fetchRecords([
+        subject,
+      ])
+      const record = recordByUri.get(subject.uri)
+      const recordLang = record?.value.langs as string[] | null
+      if (recordLang?.length) {
+        recordLang
+          .map((lang) => lang.split('-')[0])
+          .forEach((lang) => langs.add(lang))
+      }
+    }
+
+    return langs.size > 0 ? Array.from(langs) : null
+  }
+}

package/src/mod-service/status.ts

@@ -10,7 +10,6 @@ import {
 } from '../lexicon/types/com/atproto/admin/defs'
 import { ModerationEventRow, ModerationSubjectStatusRow } from './types'
 import { HOUR } from '@atproto/common'
-import { sql } from 'kysely'
 import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs'
 import { jsonb } from '../db/types'
 
@@ -83,6 +82,8 @@ const getSubjectStatusForModerationEvent = ({
         lastReviewedBy: createdBy,
         lastReviewedAt: createdAt,
       }
+    case 'com.atproto.admin.defs#modEventTag':
+      return { tags: [] }
     case 'com.atproto.admin.defs#modEventResolveAppeal':
       return {
         appealed: false,
@@ -107,6 +108,8 @@ export const adjustModerationSubjectStatus = async (
     subjectCid,
     createdBy,
     meta,
+    addedTags,
+    removedTags,
     comment,
     createdAt,
   } = moderationEvent
@@ -191,6 +194,18 @@ export const adjustModerationSubjectStatus = async (
     subjectStatus.comment = comment
   }
 
+  if (action === 'com.atproto.admin.defs#modEventTag') {
+    let tags = currentStatus?.tags || []
+    if (addedTags?.length) {
+      tags = tags.concat(addedTags)
+    }
+    if (removedTags?.length) {
+      tags = tags.filter((tag) => !removedTags.includes(tag))
+    }
+    newStatus.tags = jsonb([...new Set(tags)]) as unknown as string[]
+    subjectStatus.tags = newStatus.tags
+  }
+
   if (blobCids?.length) {
     const newBlobCids = jsonb(
       blobCids,
@@ -206,7 +221,6 @@ export const adjustModerationSubjectStatus = async (
       ...newStatus,
       createdAt: now,
       updatedAt: now,
-      // TODO: Need to get the types right here.
     } as ModerationSubjectStatusRow)
     .onConflict((oc) =>
       oc.constraint('moderation_status_unique_idx').doUpdateSet({
@@ -215,8 +229,8 @@ export const adjustModerationSubjectStatus = async (
       }),
     )
 
-  const status = await insertQuery.executeTakeFirst()
-  return status
+  const status = await insertQuery.returningAll().executeTakeFirst()
+  return status || null
 }
 
 type ModerationSubjectStatusFilter =

package/src/mod-service/types.ts

@@ -30,6 +30,7 @@ export type ModEventType =
   | ComAtprotoAdminDefs.ModEventReport
   | ComAtprotoAdminDefs.ModEventMute
   | ComAtprotoAdminDefs.ModEventReverseTakedown
+  | ComAtprotoAdminDefs.ModEventTag
 
 export const UNSPECCED_TAKEDOWN_LABEL = '!unspecced-takedown'
 

package/src/mod-service/views.ts

@@ -1,6 +1,6 @@
 import { sql } from 'kysely'
 import { AtUri, INVALID_HANDLE, normalizeDatetimeAlways } from '@atproto/syntax'
-import AtpAgent from '@atproto/api'
+import AtpAgent, { AppBskyFeedDefs } from '@atproto/api'
 import { dedupeStrs } from '@atproto/common'
 import { BlobRef } from '@atproto/lexicon'
 import { Database } from '../db'
@@ -163,6 +163,11 @@ export class ModerationViews {
       eventView.event.sticky = true
     }
 
+    if (event.action === 'com.atproto.admin.defs#modEventTag') {
+      eventView.event.add = event.addedTags || []
+      eventView.event.remove = event.removedTags || []
+    }
+
     return eventView
   }
 
@@ -217,7 +222,7 @@ export class ModerationViews {
       subjects.map(async (subject) => {
         const uri = new AtUri(subject.uri)
         try {
-          return await this.appviewAgent.api.com.atproto.repo.getRecord(
+          const record = await this.appviewAgent.api.com.atproto.repo.getRecord(
             {
               repo: uri.hostname,
               collection: uri.collection,
@@ -226,6 +231,7 @@ export class ModerationViews {
             },
             auth,
           )
+          return record
         } catch {
           return null
         }
@@ -473,9 +479,22 @@ export class ModerationViews {
       appealed: status.appealed ?? undefined,
       subjectRepoHandle: status.handle ?? undefined,
       subjectBlobCids: status.blobCids || [],
+      tags: status.tags || [],
       subject: subjectFromStatusRow(status).lex(),
     }
   }
+
+  async fetchAuthorFeed(
+    actor: string,
+  ): Promise<AppBskyFeedDefs.FeedViewPost[]> {
+    const auth = await this.appviewAuth()
+    if (!auth) return []
+    const {
+      data: { feed },
+    } = await this.appviewAgent.api.app.bsky.feed.getAuthorFeed({ actor }, auth)
+
+    return feed
+  }
 }
 
 type RecordSubject = { uri: string; cid?: string }

package/tests/__snapshots__/get-record.test.ts.snap

@@ -39,6 +39,9 @@ Object {
       },
       "subjectBlobCids": Array [],
       "subjectRepoHandle": "alice.test",
+      "tags": Array [
+        "lang:und",
+      ],
       "takendown": true,
       "updatedAt": "1970-01-01T00:00:00.000Z",
     },
@@ -133,6 +136,9 @@ Object {
       },
       "subjectBlobCids": Array [],
       "subjectRepoHandle": "alice.test",
+      "tags": Array [
+        "lang:und",
+      ],
       "takendown": true,
       "updatedAt": "1970-01-01T00:00:00.000Z",
     },

package/tests/__snapshots__/get-repo.test.ts.snap

@@ -31,6 +31,9 @@ Object {
       },
       "subjectBlobCids": Array [],
       "subjectRepoHandle": "alice.test",
+      "tags": Array [
+        "lang:und",
+      ],
       "takendown": true,
       "updatedAt": "1970-01-01T00:00:00.000Z",
     },

package/tests/__snapshots__/moderation-events.test.ts.snap

@@ -37,6 +37,9 @@ Object {
           },
           "subjectBlobCids": Array [],
           "subjectRepoHandle": "alice.test",
+          "tags": Array [
+            "lang:und",
+          ],
           "takendown": false,
           "updatedAt": "1970-01-01T00:00:00.000Z",
         },
@@ -98,7 +101,26 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 9,
+    "id": 13,
+    "subject": Object {
+      "$type": "com.atproto.admin.defs#repoRef",
+      "did": "user(0)",
+    },
+    "subjectBlobCids": Array [],
+    "subjectHandle": "bob.test",
+  },
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "createdBy": "user(2)",
+    "event": Object {
+      "$type": "com.atproto.admin.defs#modEventTag",
+      "add": Array [
+        "lang:en",
+        "lang:i",
+      ],
+      "remove": Array [],
+    },
+    "id": 8,
     "subject": Object {
       "$type": "com.atproto.admin.defs#repoRef",
       "did": "user(0)",
@@ -115,7 +137,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 5,
+    "id": 7,
     "subject": Object {
       "$type": "com.atproto.admin.defs#repoRef",
       "did": "user(0)",
@@ -137,7 +159,26 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 8,
+    "id": 12,
+    "subject": Object {
+      "$type": "com.atproto.repo.strongRef",
+      "cid": "cids(0)",
+      "uri": "record(0)",
+    },
+    "subjectBlobCids": Array [],
+    "subjectHandle": "alice.test",
+  },
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "createdBy": "user(1)",
+    "event": Object {
+      "$type": "com.atproto.admin.defs#modEventTag",
+      "add": Array [
+        "lang:und",
+      ],
+      "remove": Array [],
+    },
+    "id": 6,
     "subject": Object {
       "$type": "com.atproto.repo.strongRef",
       "cid": "cids(0)",
@@ -155,7 +196,7 @@ Array [
       "comment": "X",
       "reportType": "com.atproto.moderation.defs#reasonSpam",
     },
-    "id": 4,
+    "id": 5,
     "subject": Object {
       "$type": "com.atproto.repo.strongRef",
       "cid": "cids(0)",

package/tests/__snapshots__/moderation-statuses.test.ts.snap

@@ -1,10 +1,52 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`moderation-statuses query statuses returns statuses filtered by subject language 1`] = `
+Array [
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "id": 7,
+    "lastReportedAt": "1970-01-01T00:00:00.000Z",
+    "reviewState": "com.atproto.admin.defs#reviewOpen",
+    "subject": Object {
+      "$type": "com.atproto.repo.strongRef",
+      "cid": "cids(0)",
+      "uri": "record(0)",
+    },
+    "subjectBlobCids": Array [],
+    "subjectRepoHandle": "bob.test",
+    "tags": Array [
+      "lang:en",
+      "lang:i",
+    ],
+    "takendown": false,
+    "updatedAt": "1970-01-01T00:00:00.000Z",
+  },
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "id": 5,
+    "lastReportedAt": "1970-01-01T00:00:00.000Z",
+    "reviewState": "com.atproto.admin.defs#reviewOpen",
+    "subject": Object {
+      "$type": "com.atproto.admin.defs#repoRef",
+      "did": "user(0)",
+    },
+    "subjectBlobCids": Array [],
+    "subjectRepoHandle": "bob.test",
+    "tags": Array [
+      "lang:en",
+      "lang:i",
+    ],
+    "takendown": false,
+    "updatedAt": "1970-01-01T00:00:00.000Z",
+  },
+]
+`;
+
 exports[`moderation-statuses query statuses returns statuses for subjects that received moderation events 1`] = `
 Array [
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 4,
+    "id": 7,
     "lastReportedAt": "1970-01-01T00:00:00.000Z",
     "reviewState": "com.atproto.admin.defs#reviewOpen",
     "subject": Object {
@@ -14,12 +56,16 @@ Array [
     },
     "subjectBlobCids": Array [],
     "subjectRepoHandle": "bob.test",
+    "tags": Array [
+      "lang:en",
+      "lang:i",
+    ],
     "takendown": false,
     "updatedAt": "1970-01-01T00:00:00.000Z",
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 3,
+    "id": 5,
     "lastReportedAt": "1970-01-01T00:00:00.000Z",
     "reviewState": "com.atproto.admin.defs#reviewOpen",
     "subject": Object {
@@ -28,12 +74,16 @@ Array [
     },
     "subjectBlobCids": Array [],
     "subjectRepoHandle": "bob.test",
+    "tags": Array [
+      "lang:en",
+      "lang:i",
+    ],
     "takendown": false,
     "updatedAt": "1970-01-01T00:00:00.000Z",
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 2,
+    "id": 3,
     "lastReportedAt": "1970-01-01T00:00:00.000Z",
     "reviewState": "com.atproto.admin.defs#reviewOpen",
     "subject": Object {
@@ -43,6 +93,9 @@ Array [
     },
     "subjectBlobCids": Array [],
     "subjectRepoHandle": "alice.test",
+    "tags": Array [
+      "lang:und",
+    ],
     "takendown": false,
     "updatedAt": "1970-01-01T00:00:00.000Z",
   },
@@ -57,6 +110,9 @@ Array [
     },
     "subjectBlobCids": Array [],
     "subjectRepoHandle": "alice.test",
+    "tags": Array [
+      "lang:und",
+    ],
     "takendown": false,
     "updatedAt": "1970-01-01T00:00:00.000Z",
   },

package/tests/__snapshots__/moderation.test.ts.snap

@@ -4,7 +4,7 @@ exports[`moderation reporting creates reports of a record. 1`] = `
 Array [
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 6,
+    "id": 7,
     "reasonType": "com.atproto.moderation.defs#reasonSpam",
     "reportedBy": "user(0)",
     "subject": Object {
@@ -15,7 +15,7 @@ Array [
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 7,
+    "id": 9,
     "reason": "defamation",
     "reasonType": "com.atproto.moderation.defs#reasonOther",
     "reportedBy": "user(1)",
@@ -42,7 +42,7 @@ Array [
   },
   Object {
     "createdAt": "1970-01-01T00:00:00.000Z",
-    "id": 4,
+    "id": 5,
     "reason": "impersonation",
     "reasonType": "com.atproto.moderation.defs#reasonOther",
     "reportedBy": "user(2)",

package/tests/get-record.test.ts

@@ -27,14 +27,6 @@ describe('admin get record view', () => {
   })
 
   beforeAll(async () => {
-    await sc.emitModerationEvent({
-      event: { $type: 'com.atproto.admin.defs#modEventFlag' },
-      subject: {
-        $type: 'com.atproto.repo.strongRef',
-        uri: sc.posts[sc.dids.alice][0].ref.uriStr,
-        cid: sc.posts[sc.dids.alice][0].ref.cidStr,
-      },
-    })
     await sc.createReport({
       reportedBy: sc.dids.bob,
       reasonType: REASONSPAM,