@atproto/ozone 0.1.14 → 0.1.16

package/src/mod-service/views.ts

@@ -28,10 +28,12 @@ import { formatLabel, signLabel } from './util'
 import { LabelRow } from '../db/schema/label'
 import { dbLogger } from '../logger'
 import { httpLogger } from '../logger'
+import { ParsedLabelers } from '../util'
 
 export type AuthHeaders = {
   headers: {
     authorization: string
+    'atproto-accept-labelers'?: string
   }
 }
 
@@ -210,10 +212,14 @@ export class ModerationViews {
     }
   }
 
-  async repoDetail(did: string): Promise<RepoViewDetail | undefined> {
-    const [repos, labels] = await Promise.all([
+  async repoDetail(
+    did: string,
+    labelers?: ParsedLabelers,
+  ): Promise<RepoViewDetail | undefined> {
+    const [repos, localLabels, externalLabels] = await Promise.all([
       this.repos([did]),
       this.labels(did),
+      this.getExternalLabels([did], labelers),
     ])
     const repo = repos.get(did)
     if (!repo) return
@@ -223,7 +229,7 @@ export class ModerationViews {
       moderation: {
         ...repo.moderation,
       },
-      labels,
+      labels: [...localLabels, ...externalLabels],
     }
   }
 
@@ -293,6 +299,7 @@ export class ModerationViews {
 
   async recordDetail(
     subject: RecordSubject,
+    labelers?: ParsedLabelers,
   ): Promise<RecordViewDetail | undefined> {
     const [records, subjectStatusesResult] = await Promise.all([
       this.records([subject]),
@@ -303,9 +310,10 @@ export class ModerationViews {
 
     const status = subjectStatusesResult.get(subject.uri)
 
-    const [blobs, labels, subjectStatus] = await Promise.all([
+    const [blobs, labels, externalLabels, subjectStatus] = await Promise.all([
       this.blob(findBlobRefs(record.value)),
       this.labels(record.uri),
+      this.getExternalLabels([record.uri], labelers),
       status ? this.formatSubjectStatus(status) : Promise.resolve(undefined),
     ])
     const selfLabels = getSelfLabels({
@@ -313,6 +321,7 @@ export class ModerationViews {
       cid: record.cid,
       record: record.value,
     })
+
     return {
       ...record,
       blobs,
@@ -320,10 +329,26 @@ export class ModerationViews {
         ...record.moderation,
         subjectStatus,
       },
-      labels: [...labels, ...selfLabels],
+      labels: [...labels, ...selfLabels, ...externalLabels],
     }
   }
 
+  async getExternalLabels(
+    subjects: string[],
+    labelers?: ParsedLabelers,
+  ): Promise<Label[]> {
+    if (!labelers?.dids.length && !labelers?.redact.size) return []
+
+    const {
+      data: { labels },
+    } = await this.appviewAgent.api.com.atproto.label.queryLabels({
+      uriPatterns: subjects,
+      sources: labelers.dids,
+    })
+
+    return labels
+  }
+
   formatReport(report: ModerationEventRowWithHandle): ReportOutput {
     return {
       id: report.id,

package/src/util.ts

@@ -1,4 +1,5 @@
 import { AxiosError } from 'axios'
+import { parseList } from 'structured-headers'
 import { XRPCError, ResponseType } from '@atproto/xrpc'
 import { RetryOptions, retry } from '@atproto/common'
 import Database from './db'
@@ -45,3 +46,52 @@ export function retryableHttp(err: unknown) {
 const retryableHttpStatusCodes = new Set([
   408, 425, 429, 500, 502, 503, 504, 522, 524,
 ])
+
+export type ParsedLabelers = {
+  dids: string[]
+  redact: Set<string>
+}
+
+export const LABELER_HEADER_NAME = 'atproto-accept-labelers'
+
+export const parseLabelerHeader = (
+  header: string | undefined,
+  ignoreDid?: string,
+): ParsedLabelers | null => {
+  if (!header) return null
+  const labelerDids = new Set<string>()
+  const redactDids = new Set<string>()
+  const parsed = parseList(header)
+  for (const item of parsed) {
+    const did = item[0].toString()
+    if (!did) {
+      return null
+    }
+    if (did === ignoreDid) {
+      continue
+    }
+    labelerDids.add(did)
+    const redact = item[1].get('redact')?.valueOf()
+    if (redact === true) {
+      redactDids.add(did)
+    }
+  }
+  return {
+    dids: [...labelerDids],
+    redact: redactDids,
+  }
+}
+
+export const defaultLabelerHeader = (dids: string[]): ParsedLabelers => {
+  return {
+    dids,
+    redact: new Set(dids),
+  }
+}
+
+export const formatLabelerHeader = (parsed: ParsedLabelers): string => {
+  const parts = parsed.dids.map((did) =>
+    parsed.redact.has(did) ? `${did};redact` : did,
+  )
+  return parts.join(',')
+}

package/tests/3p-labeler.test.ts

@@ -0,0 +1,271 @@
+import {
+  SeedClient,
+  TestNetwork,
+  basicSeed,
+  TestOzone,
+  ModeratorClient,
+  createOzoneDid,
+} from '@atproto/dev-env'
+import AtpAgent from '@atproto/api'
+import { Secp256k1Keypair } from '@atproto/crypto'
+import { LABELER_HEADER_NAME } from '../src/util'
+
+describe('labels from 3p labelers', () => {
+  let network: TestNetwork
+  let ozone: TestOzone
+  let thirdPartyLabeler: TestOzone
+  let agent: AtpAgent
+  let thirdPartyAgent: AtpAgent
+  let sc: SeedClient
+  let modClient: ModeratorClient
+  let thirdPartyModClient: ModeratorClient
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_admin_third_party_labeler',
+    })
+    ozone = network.ozone
+
+    const ozoneKey = await Secp256k1Keypair.create({ exportable: true })
+    const ozoneDid = await createOzoneDid(network.plc.url, ozoneKey)
+    thirdPartyLabeler = await TestOzone.create({
+      port: ozone.port + 10,
+      plcUrl: network.plc.url,
+      signingKey: ozoneKey,
+      serverDid: ozoneDid,
+      dbPostgresSchema: `ozone_admin_third_party_labeler`,
+      dbPostgresUrl: ozone.ctx.cfg.db.postgresUrl,
+      appviewUrl: network.bsky.url,
+      appviewDid: network.bsky.ctx.cfg.serverDid,
+      appviewPushEvents: true,
+      pdsUrl: network.pds.url,
+      pdsDid: network.pds.ctx.cfg.service.did,
+    })
+
+    thirdPartyAgent = thirdPartyLabeler.getClient()
+    agent = ozone.getClient()
+    sc = network.getSeedClient()
+    modClient = ozone.getModClient()
+    thirdPartyModClient = thirdPartyLabeler.getModClient()
+    await basicSeed(sc)
+    await network.processAll()
+  })
+
+  afterAll(async () => {
+    await network.close()
+    await thirdPartyLabeler.close()
+  })
+
+  const getPostSubject = () => ({
+    $type: 'com.atproto.repo.strongRef',
+    uri: sc.posts[sc.dids.alice][0].ref.uriStr,
+    cid: sc.posts[sc.dids.alice][0].ref.cidStr,
+  })
+
+  const adjustLabels = async ({
+    uri,
+    cid,
+    src,
+    createLabelVals = [],
+    negateLabelVals = [],
+  }: {
+    uri: string
+    src: string
+    cid?: string
+    createLabelVals?: string[]
+    negateLabelVals?: string[]
+  }) => {
+    const labelEntries = createLabelVals.map((val) => ({
+      uri,
+      cid: cid || '',
+      val,
+      cts: new Date().toISOString(),
+      neg: false,
+      src,
+    }))
+
+    negateLabelVals.forEach((val) => {
+      labelEntries.push({
+        uri,
+        cid: cid || '',
+        val,
+        cts: new Date().toISOString(),
+        neg: true,
+        src,
+      })
+    })
+    await network.bsky.db.db.insertInto('label').values(labelEntries).execute()
+  }
+
+  const labelAccount = async (
+    client: ModeratorClient,
+    {
+      createLabelVals = [],
+      negateLabelVals = [],
+    }: { createLabelVals?: string[]; negateLabelVals?: string[] },
+  ) => {
+    const subject = {
+      $type: 'com.atproto.admin.defs#repoRef',
+      did: sc.dids.alice,
+    }
+    await client.emitEvent(
+      {
+        subject,
+        event: {
+          $type: 'tools.ozone.moderation.defs#modEventLabel',
+          createLabelVals,
+          negateLabelVals,
+        },
+        createdBy: sc.dids.carol,
+      },
+      'moderator',
+    )
+    await adjustLabels({
+      createLabelVals,
+      negateLabelVals,
+      uri: sc.dids.alice,
+      src: client.ozone.ctx.cfg.service.did,
+    })
+  }
+
+  const labelPost = async (
+    client: ModeratorClient,
+    {
+      createLabelVals = [],
+      negateLabelVals = [],
+    }: { createLabelVals?: string[]; negateLabelVals?: string[] },
+  ) => {
+    const subject = getPostSubject()
+    await client.emitEvent(
+      {
+        subject,
+        event: {
+          $type: 'tools.ozone.moderation.defs#modEventLabel',
+          createLabelVals,
+          negateLabelVals,
+        },
+        createdBy: sc.dids.carol,
+      },
+      'moderator',
+    )
+    await adjustLabels({
+      createLabelVals,
+      negateLabelVals,
+      uri: subject.uri,
+      cid: subject.cid,
+      src: client.ozone.ctx.cfg.service.did,
+    })
+  }
+
+  describe('record labels', () => {
+    it('includes only labels from current authority by default', async () => {
+      await Promise.all([
+        labelPost(modClient, { createLabelVals: ['spam'] }),
+        labelPost(thirdPartyModClient, { createLabelVals: ['weird'] }),
+      ])
+
+      const [
+        { data: recordFromCurrentAuthority },
+        { data: recordFromThirdParty },
+      ] = await Promise.all([
+        agent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          { headers: await ozone.modHeaders() },
+        ),
+        thirdPartyAgent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          { headers: await thirdPartyLabeler.modHeaders() },
+        ),
+      ])
+
+      const currentAuthorityLabels = recordFromCurrentAuthority.labels?.map(
+        (l) => l.val,
+      )
+      const thirdPartyLabels = recordFromThirdParty.labels?.map((l) => l.val)
+      expect(currentAuthorityLabels).toContain('spam')
+      expect(currentAuthorityLabels).not.toContain('weird')
+      expect(thirdPartyLabels).toContain('weird')
+      expect(thirdPartyLabels).not.toContain('spam')
+    })
+
+    it('includes labels from all authorities requested via header', async () => {
+      const authHeaders = await ozone.modHeaders()
+      const { data: recordIncludingExternalLabels } =
+        await agent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          {
+            headers: {
+              ...authHeaders,
+              [LABELER_HEADER_NAME]: [
+                thirdPartyLabeler.ctx.cfg.service.did,
+                ozone.ctx.cfg.service.did,
+              ].join(','),
+            },
+          },
+        )
+      const labelVals = recordIncludingExternalLabels.labels?.map((l) => l.val)
+      const labelSources = recordIncludingExternalLabels.labels?.map(
+        (l) => l.src,
+      )
+      expect(labelVals).toContain('weird')
+      expect(labelVals).toContain('spam')
+      expect(labelSources).toContain(thirdPartyLabeler.ctx.cfg.service.did)
+      expect(labelSources).toContain(ozone.ctx.cfg.service.did)
+    })
+  })
+
+  describe('repo labels', () => {
+    it('includes only labels from current authority by default', async () => {
+      await Promise.all([
+        labelAccount(modClient, { createLabelVals: ['spam'] }),
+        labelAccount(thirdPartyModClient, { createLabelVals: ['weird'] }),
+      ])
+
+      const [{ data: repoFromCurrentAuthority }, { data: repoFromThirdParty }] =
+        await Promise.all([
+          agent.api.tools.ozone.moderation.getRepo(
+            { did: sc.dids.alice },
+            { headers: await ozone.modHeaders() },
+          ),
+          thirdPartyAgent.api.tools.ozone.moderation.getRepo(
+            { did: sc.dids.alice },
+            { headers: await thirdPartyLabeler.modHeaders() },
+          ),
+        ])
+
+      const currentAuthorityLabels = repoFromCurrentAuthority.labels?.map(
+        (l) => l.val,
+      )
+      const thirdPartyLabels = repoFromThirdParty.labels?.map((l) => l.val)
+      expect(currentAuthorityLabels).toContain('spam')
+      expect(currentAuthorityLabels).not.toContain('weird')
+      expect(thirdPartyLabels).toContain('weird')
+      expect(thirdPartyLabels).not.toContain('spam')
+    })
+
+    it('includes labels from all authorities requested via header', async () => {
+      const authHeaders = await ozone.modHeaders()
+      const { data: recordIncludingExternalLabels } =
+        await agent.api.tools.ozone.moderation.getRepo(
+          { did: sc.dids.alice },
+          {
+            headers: {
+              ...authHeaders,
+              [LABELER_HEADER_NAME]: [
+                thirdPartyLabeler.ctx.cfg.service.did,
+                ozone.ctx.cfg.service.did,
+              ].join(','),
+            },
+          },
+        )
+      const labelVals = recordIncludingExternalLabels.labels?.map((l) => l.val)
+      const labelSources = recordIncludingExternalLabels.labels?.map(
+        (l) => l.src,
+      )
+      expect(labelVals).toContain('weird')
+      expect(labelVals).toContain('spam')
+      expect(labelSources).toContain(thirdPartyLabeler.ctx.cfg.service.did)
+      expect(labelSources).toContain(ozone.ctx.cfg.service.did)
+    })
+  })
+})

package/tests/__snapshots__/moderation.test.ts.snap

@@ -1,5 +1,35 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`moderation reporting creates reports of a DM chat. 1`] = `
+Array [
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "id": 12,
+    "reasonType": "com.atproto.moderation.defs#reasonSpam",
+    "reportedBy": "user(0)",
+    "subject": Object {
+      "$type": "chat.bsky.convo.defs#messageRef",
+      "convoId": "testconvoid1",
+      "did": "user(1)",
+      "messageId": "testmessageid1",
+    },
+  },
+  Object {
+    "createdAt": "1970-01-01T00:00:00.000Z",
+    "id": 14,
+    "reason": "defamation",
+    "reasonType": "com.atproto.moderation.defs#reasonOther",
+    "reportedBy": "user(1)",
+    "subject": Object {
+      "$type": "chat.bsky.convo.defs#messageRef",
+      "convoId": "testconvoid2",
+      "did": "user(1)",
+      "messageId": "testmessageid2",
+    },
+  },
+]
+`;
+
 exports[`moderation reporting creates reports of a record. 1`] = `
 Array [
   Object {

package/tests/get-lists.test.ts

@@ -0,0 +1,109 @@
+import {
+  SeedClient,
+  TestNetwork,
+  TestOzone,
+  basicSeed,
+  ModeratorClient,
+  RecordRef,
+} from '@atproto/dev-env'
+import AtpAgent, { BSKY_LABELER_DID } from '@atproto/api'
+import { TAKEDOWN_LABEL } from '../src/mod-service'
+
+describe('admin get lists', () => {
+  let network: TestNetwork
+  let ozone: TestOzone
+  let agent: AtpAgent
+  let appviewAgent: AtpAgent
+  let sc: SeedClient
+  let modClient: ModeratorClient
+  let alicesList: RecordRef
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_admin_get_lists',
+    })
+    ozone = network.ozone
+    agent = ozone.getClient()
+    appviewAgent = network.bsky.getClient()
+    sc = network.getSeedClient()
+    modClient = ozone.getModClient()
+    await basicSeed(sc)
+    alicesList = await sc.createList(sc.dids.alice, "Alice's List", 'mod')
+    AtpAgent.configure({ appLabelers: [ozone.ctx.cfg.service.did] })
+    await network.processAll()
+  })
+
+  afterAll(async () => {
+    AtpAgent.configure({ appLabelers: [BSKY_LABELER_DID] })
+    await network.close()
+  })
+
+  const getAlicesList = async () => {
+    const [{ data: fromOzone }, { data: fromAppview }] = await Promise.all([
+      agent.api.app.bsky.graph.getLists(
+        { actor: sc.dids.alice },
+        { headers: await ozone.modHeaders() },
+      ),
+      appviewAgent.api.app.bsky.graph.getLists({ actor: sc.dids.alice }),
+    ])
+
+    return { fromOzone, fromAppview }
+  }
+
+  it('returns lists from takendown account', async () => {
+    const beforeTakedown = await getAlicesList()
+    expect(beforeTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+    expect(beforeTakedown.fromAppview.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Takedown alice's account
+    await modClient.emitEvent({
+      event: { $type: 'tools.ozone.moderation.defs#modEventTakedown' },
+      subject: {
+        $type: 'com.atproto.admin.defs#repoRef',
+        did: sc.dids.alice,
+      },
+    })
+    await network.processAll()
+
+    const afterTakedown = await getAlicesList()
+
+    // Verify that takendown list is shown when queried through ozone but not through appview
+    expect(afterTakedown.fromAppview.lists.length).toBe(0)
+    expect(afterTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Reverse alice's account takedown
+    await modClient.emitEvent({
+      event: { $type: 'tools.ozone.moderation.defs#modEventReverseTakedown' },
+      subject: {
+        $type: 'com.atproto.admin.defs#repoRef',
+        did: sc.dids.alice,
+      },
+    })
+    await network.processAll()
+  })
+
+  it('returns takendown lists', async () => {
+    const beforeTakedown = await getAlicesList()
+    expect(beforeTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+    expect(beforeTakedown.fromAppview.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Takedown alice's list using a !takedown label
+    await network.bsky.db.db
+      .insertInto('label')
+      .values({
+        src: ozone.ctx.cfg.service.did,
+        uri: alicesList.uriStr,
+        cid: alicesList.cidStr,
+        val: TAKEDOWN_LABEL,
+        neg: false,
+        cts: new Date().toISOString(),
+      })
+      .execute()
+
+    const afterTakedown = await getAlicesList()
+
+    // Verify that takendown list is shown when queried through ozone but not through appview
+    expect(afterTakedown.fromAppview.lists.length).toBe(0)
+    expect(afterTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+  })
+})

package/tests/moderation.test.ts

@@ -149,6 +149,45 @@ describe('moderation', () => {
       })
       await expect(promiseB).resolves.toBeDefined()
     })
+
+    it('creates reports of a DM chat.', async () => {
+      const messageId1 = 'testmessageid1'
+      const messageId2 = 'testmessageid2'
+      const reportA = await sc.createReport({
+        reportedBy: sc.dids.alice,
+        reasonType: REASONSPAM,
+        subject: {
+          $type: 'chat.bsky.convo.defs#messageRef',
+          did: sc.dids.carol,
+          messageId: messageId1,
+          convoId: 'testconvoid1',
+        },
+      })
+      const reportB = await sc.createReport({
+        reportedBy: sc.dids.carol,
+        reasonType: REASONOTHER,
+        reason: 'defamation',
+        subject: {
+          $type: 'chat.bsky.convo.defs#messageRef',
+          did: sc.dids.carol,
+          messageId: messageId2,
+          convoId: 'testconvoid2',
+        },
+      })
+      expect(forSnapshot([reportA, reportB])).toMatchSnapshot()
+      const events = await ozone.ctx.db.db
+        .selectFrom('moderation_event')
+        .selectAll()
+        .where('subjectMessageId', 'in', [messageId1, messageId2])
+        .where('action', '=', 'tools.ozone.moderation.defs#modEventReport')
+        .execute()
+      expect(events.length).toBe(2)
+      expect(
+        events.every(
+          (row) => row.subjectType === 'chat.bsky.convo.defs#messageRef',
+        ),
+      ).toBe(true)
+    })
   })
 
   describe('actioning', () => {