@atproto/ozone 0.1.15 → 0.1.17

package/src/context.ts

@@ -1,3 +1,4 @@
+import express from 'express'
 import * as plc from '@did-plc/lib'
 import { IdResolver } from '@atproto/identity'
 import { AtpAgent } from '@atproto/api'
@@ -17,7 +18,13 @@ import {
 import { BlobDiverter } from './daemon/blob-diverter'
 import { AuthVerifier } from './auth-verifier'
 import { ImageInvalidator } from './image-invalidator'
-import { getSigningKeyId } from './util'
+import {
+  defaultLabelerHeader,
+  getSigningKeyId,
+  LABELER_HEADER_NAME,
+  ParsedLabelers,
+  parseLabelerHeader,
+} from './util'
 
 export type AppContextOptions = {
   db: Database
@@ -234,5 +241,17 @@ export class AppContext {
       ...overrides,
     }
   }
+
+  reqLabelers(req: express.Request): ParsedLabelers {
+    const val = req.header(LABELER_HEADER_NAME)
+    let parsed: ParsedLabelers | null
+    try {
+      parsed = parseLabelerHeader(val, this.cfg.service.did)
+    } catch (err) {
+      parsed = null
+    }
+    if (!parsed) return defaultLabelerHeader([])
+    return parsed
+  }
 }
 export default AppContext

package/src/lexicon/lexicons.ts

@@ -2052,6 +2052,11 @@ export const schemaDict = {
                 description:
                   'A short name for the App Password, to help distinguish them.',
               },
+              privileged: {
+                type: 'boolean',
+                description:
+                  "If an app password has 'privileged' access to possibly sensitive account state. Meant for use with trusted clients.",
+              },
             },
           },
         },
@@ -2082,6 +2087,9 @@ export const schemaDict = {
             type: 'string',
             format: 'datetime',
           },
+          privileged: {
+            type: 'boolean',
+          },
         },
       },
     },
@@ -2629,6 +2637,9 @@ export const schemaDict = {
             type: 'string',
             format: 'datetime',
           },
+          privileged: {
+            type: 'boolean',
+          },
         },
       },
     },

package/src/lexicon/types/com/atproto/server/createAppPassword.ts

@@ -13,6 +13,8 @@ export interface QueryParams {}
 export interface InputSchema {
   /** A short name for the App Password, to help distinguish them. */
   name: string
+  /** If an app password has 'privileged' access to possibly sensitive account state. Meant for use with trusted clients. */
+  privileged?: boolean
   [k: string]: unknown
 }
 
@@ -51,6 +53,7 @@ export interface AppPassword {
   name: string
   password: string
   createdAt: string
+  privileged?: boolean
   [k: string]: unknown
 }
 

package/src/lexicon/types/com/atproto/server/listAppPasswords.ts

@@ -46,6 +46,7 @@ export type Handler<HA extends HandlerAuth = never> = (
 export interface AppPassword {
   name: string
   createdAt: string
+  privileged?: boolean
   [k: string]: unknown
 }
 

package/src/mod-service/index.ts

@@ -48,6 +48,7 @@ import { formatLabel, formatLabelRow, signLabel } from './util'
 import { ImageInvalidator } from '../image-invalidator'
 import { httpLogger as log } from '../logger'
 import { OzoneConfig } from '../config'
+import { LABELER_HEADER_NAME, ParsedLabelers } from '../util'
 
 export type ModerationServiceCreator = (db: Database) => ModerationService
 
@@ -96,7 +97,13 @@ export class ModerationService {
     this.signingKey,
     this.signingKeyId,
     this.appviewAgent,
-    () => this.createAuthHeaders(this.cfg.appview.did),
+    async (labelers?: ParsedLabelers) => {
+      const authHeaders = await this.createAuthHeaders(this.cfg.appview.did)
+      if (labelers?.dids?.length) {
+        authHeaders.headers[LABELER_HEADER_NAME] = labelers.dids.join(', ')
+      }
+      return authHeaders
+    },
   )
 
   async getEvent(id: number): Promise<ModerationEventRow | undefined> {

package/src/mod-service/views.ts

@@ -28,10 +28,12 @@ import { formatLabel, signLabel } from './util'
 import { LabelRow } from '../db/schema/label'
 import { dbLogger } from '../logger'
 import { httpLogger } from '../logger'
+import { ParsedLabelers } from '../util'
 
 export type AuthHeaders = {
   headers: {
     authorization: string
+    'atproto-accept-labelers'?: string
   }
 }
 
@@ -210,10 +212,14 @@ export class ModerationViews {
     }
   }
 
-  async repoDetail(did: string): Promise<RepoViewDetail | undefined> {
-    const [repos, labels] = await Promise.all([
+  async repoDetail(
+    did: string,
+    labelers?: ParsedLabelers,
+  ): Promise<RepoViewDetail | undefined> {
+    const [repos, localLabels, externalLabels] = await Promise.all([
       this.repos([did]),
       this.labels(did),
+      this.getExternalLabels([did], labelers),
     ])
     const repo = repos.get(did)
     if (!repo) return
@@ -223,7 +229,7 @@ export class ModerationViews {
       moderation: {
         ...repo.moderation,
       },
-      labels,
+      labels: [...localLabels, ...externalLabels],
     }
   }
 
@@ -293,6 +299,7 @@ export class ModerationViews {
 
   async recordDetail(
     subject: RecordSubject,
+    labelers?: ParsedLabelers,
   ): Promise<RecordViewDetail | undefined> {
     const [records, subjectStatusesResult] = await Promise.all([
       this.records([subject]),
@@ -303,9 +310,10 @@ export class ModerationViews {
 
     const status = subjectStatusesResult.get(subject.uri)
 
-    const [blobs, labels, subjectStatus] = await Promise.all([
+    const [blobs, labels, externalLabels, subjectStatus] = await Promise.all([
       this.blob(findBlobRefs(record.value)),
       this.labels(record.uri),
+      this.getExternalLabels([record.uri], labelers),
       status ? this.formatSubjectStatus(status) : Promise.resolve(undefined),
     ])
     const selfLabels = getSelfLabels({
@@ -313,6 +321,7 @@ export class ModerationViews {
       cid: record.cid,
       record: record.value,
     })
+
     return {
       ...record,
       blobs,
@@ -320,10 +329,26 @@ export class ModerationViews {
         ...record.moderation,
         subjectStatus,
       },
-      labels: [...labels, ...selfLabels],
+      labels: [...labels, ...selfLabels, ...externalLabels],
     }
   }
 
+  async getExternalLabels(
+    subjects: string[],
+    labelers?: ParsedLabelers,
+  ): Promise<Label[]> {
+    if (!labelers?.dids.length && !labelers?.redact.size) return []
+
+    const {
+      data: { labels },
+    } = await this.appviewAgent.api.com.atproto.label.queryLabels({
+      uriPatterns: subjects,
+      sources: labelers.dids,
+    })
+
+    return labels
+  }
+
   formatReport(report: ModerationEventRowWithHandle): ReportOutput {
     return {
       id: report.id,

package/src/util.ts

@@ -1,4 +1,5 @@
 import { AxiosError } from 'axios'
+import { parseList } from 'structured-headers'
 import { XRPCError, ResponseType } from '@atproto/xrpc'
 import { RetryOptions, retry } from '@atproto/common'
 import Database from './db'
@@ -45,3 +46,52 @@ export function retryableHttp(err: unknown) {
 const retryableHttpStatusCodes = new Set([
   408, 425, 429, 500, 502, 503, 504, 522, 524,
 ])
+
+export type ParsedLabelers = {
+  dids: string[]
+  redact: Set<string>
+}
+
+export const LABELER_HEADER_NAME = 'atproto-accept-labelers'
+
+export const parseLabelerHeader = (
+  header: string | undefined,
+  ignoreDid?: string,
+): ParsedLabelers | null => {
+  if (!header) return null
+  const labelerDids = new Set<string>()
+  const redactDids = new Set<string>()
+  const parsed = parseList(header)
+  for (const item of parsed) {
+    const did = item[0].toString()
+    if (!did) {
+      return null
+    }
+    if (did === ignoreDid) {
+      continue
+    }
+    labelerDids.add(did)
+    const redact = item[1].get('redact')?.valueOf()
+    if (redact === true) {
+      redactDids.add(did)
+    }
+  }
+  return {
+    dids: [...labelerDids],
+    redact: redactDids,
+  }
+}
+
+export const defaultLabelerHeader = (dids: string[]): ParsedLabelers => {
+  return {
+    dids,
+    redact: new Set(dids),
+  }
+}
+
+export const formatLabelerHeader = (parsed: ParsedLabelers): string => {
+  const parts = parsed.dids.map((did) =>
+    parsed.redact.has(did) ? `${did};redact` : did,
+  )
+  return parts.join(',')
+}

package/tests/3p-labeler.test.ts

@@ -0,0 +1,271 @@
+import {
+  SeedClient,
+  TestNetwork,
+  basicSeed,
+  TestOzone,
+  ModeratorClient,
+  createOzoneDid,
+} from '@atproto/dev-env'
+import AtpAgent from '@atproto/api'
+import { Secp256k1Keypair } from '@atproto/crypto'
+import { LABELER_HEADER_NAME } from '../src/util'
+
+describe('labels from 3p labelers', () => {
+  let network: TestNetwork
+  let ozone: TestOzone
+  let thirdPartyLabeler: TestOzone
+  let agent: AtpAgent
+  let thirdPartyAgent: AtpAgent
+  let sc: SeedClient
+  let modClient: ModeratorClient
+  let thirdPartyModClient: ModeratorClient
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_admin_third_party_labeler',
+    })
+    ozone = network.ozone
+
+    const ozoneKey = await Secp256k1Keypair.create({ exportable: true })
+    const ozoneDid = await createOzoneDid(network.plc.url, ozoneKey)
+    thirdPartyLabeler = await TestOzone.create({
+      port: ozone.port + 10,
+      plcUrl: network.plc.url,
+      signingKey: ozoneKey,
+      serverDid: ozoneDid,
+      dbPostgresSchema: `ozone_admin_third_party_labeler`,
+      dbPostgresUrl: ozone.ctx.cfg.db.postgresUrl,
+      appviewUrl: network.bsky.url,
+      appviewDid: network.bsky.ctx.cfg.serverDid,
+      appviewPushEvents: true,
+      pdsUrl: network.pds.url,
+      pdsDid: network.pds.ctx.cfg.service.did,
+    })
+
+    thirdPartyAgent = thirdPartyLabeler.getClient()
+    agent = ozone.getClient()
+    sc = network.getSeedClient()
+    modClient = ozone.getModClient()
+    thirdPartyModClient = thirdPartyLabeler.getModClient()
+    await basicSeed(sc)
+    await network.processAll()
+  })
+
+  afterAll(async () => {
+    await network.close()
+    await thirdPartyLabeler.close()
+  })
+
+  const getPostSubject = () => ({
+    $type: 'com.atproto.repo.strongRef',
+    uri: sc.posts[sc.dids.alice][0].ref.uriStr,
+    cid: sc.posts[sc.dids.alice][0].ref.cidStr,
+  })
+
+  const adjustLabels = async ({
+    uri,
+    cid,
+    src,
+    createLabelVals = [],
+    negateLabelVals = [],
+  }: {
+    uri: string
+    src: string
+    cid?: string
+    createLabelVals?: string[]
+    negateLabelVals?: string[]
+  }) => {
+    const labelEntries = createLabelVals.map((val) => ({
+      uri,
+      cid: cid || '',
+      val,
+      cts: new Date().toISOString(),
+      neg: false,
+      src,
+    }))
+
+    negateLabelVals.forEach((val) => {
+      labelEntries.push({
+        uri,
+        cid: cid || '',
+        val,
+        cts: new Date().toISOString(),
+        neg: true,
+        src,
+      })
+    })
+    await network.bsky.db.db.insertInto('label').values(labelEntries).execute()
+  }
+
+  const labelAccount = async (
+    client: ModeratorClient,
+    {
+      createLabelVals = [],
+      negateLabelVals = [],
+    }: { createLabelVals?: string[]; negateLabelVals?: string[] },
+  ) => {
+    const subject = {
+      $type: 'com.atproto.admin.defs#repoRef',
+      did: sc.dids.alice,
+    }
+    await client.emitEvent(
+      {
+        subject,
+        event: {
+          $type: 'tools.ozone.moderation.defs#modEventLabel',
+          createLabelVals,
+          negateLabelVals,
+        },
+        createdBy: sc.dids.carol,
+      },
+      'moderator',
+    )
+    await adjustLabels({
+      createLabelVals,
+      negateLabelVals,
+      uri: sc.dids.alice,
+      src: client.ozone.ctx.cfg.service.did,
+    })
+  }
+
+  const labelPost = async (
+    client: ModeratorClient,
+    {
+      createLabelVals = [],
+      negateLabelVals = [],
+    }: { createLabelVals?: string[]; negateLabelVals?: string[] },
+  ) => {
+    const subject = getPostSubject()
+    await client.emitEvent(
+      {
+        subject,
+        event: {
+          $type: 'tools.ozone.moderation.defs#modEventLabel',
+          createLabelVals,
+          negateLabelVals,
+        },
+        createdBy: sc.dids.carol,
+      },
+      'moderator',
+    )
+    await adjustLabels({
+      createLabelVals,
+      negateLabelVals,
+      uri: subject.uri,
+      cid: subject.cid,
+      src: client.ozone.ctx.cfg.service.did,
+    })
+  }
+
+  describe('record labels', () => {
+    it('includes only labels from current authority by default', async () => {
+      await Promise.all([
+        labelPost(modClient, { createLabelVals: ['spam'] }),
+        labelPost(thirdPartyModClient, { createLabelVals: ['weird'] }),
+      ])
+
+      const [
+        { data: recordFromCurrentAuthority },
+        { data: recordFromThirdParty },
+      ] = await Promise.all([
+        agent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          { headers: await ozone.modHeaders() },
+        ),
+        thirdPartyAgent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          { headers: await thirdPartyLabeler.modHeaders() },
+        ),
+      ])
+
+      const currentAuthorityLabels = recordFromCurrentAuthority.labels?.map(
+        (l) => l.val,
+      )
+      const thirdPartyLabels = recordFromThirdParty.labels?.map((l) => l.val)
+      expect(currentAuthorityLabels).toContain('spam')
+      expect(currentAuthorityLabels).not.toContain('weird')
+      expect(thirdPartyLabels).toContain('weird')
+      expect(thirdPartyLabels).not.toContain('spam')
+    })
+
+    it('includes labels from all authorities requested via header', async () => {
+      const authHeaders = await ozone.modHeaders()
+      const { data: recordIncludingExternalLabels } =
+        await agent.api.tools.ozone.moderation.getRecord(
+          { uri: sc.posts[sc.dids.alice][0].ref.uriStr },
+          {
+            headers: {
+              ...authHeaders,
+              [LABELER_HEADER_NAME]: [
+                thirdPartyLabeler.ctx.cfg.service.did,
+                ozone.ctx.cfg.service.did,
+              ].join(','),
+            },
+          },
+        )
+      const labelVals = recordIncludingExternalLabels.labels?.map((l) => l.val)
+      const labelSources = recordIncludingExternalLabels.labels?.map(
+        (l) => l.src,
+      )
+      expect(labelVals).toContain('weird')
+      expect(labelVals).toContain('spam')
+      expect(labelSources).toContain(thirdPartyLabeler.ctx.cfg.service.did)
+      expect(labelSources).toContain(ozone.ctx.cfg.service.did)
+    })
+  })
+
+  describe('repo labels', () => {
+    it('includes only labels from current authority by default', async () => {
+      await Promise.all([
+        labelAccount(modClient, { createLabelVals: ['spam'] }),
+        labelAccount(thirdPartyModClient, { createLabelVals: ['weird'] }),
+      ])
+
+      const [{ data: repoFromCurrentAuthority }, { data: repoFromThirdParty }] =
+        await Promise.all([
+          agent.api.tools.ozone.moderation.getRepo(
+            { did: sc.dids.alice },
+            { headers: await ozone.modHeaders() },
+          ),
+          thirdPartyAgent.api.tools.ozone.moderation.getRepo(
+            { did: sc.dids.alice },
+            { headers: await thirdPartyLabeler.modHeaders() },
+          ),
+        ])
+
+      const currentAuthorityLabels = repoFromCurrentAuthority.labels?.map(
+        (l) => l.val,
+      )
+      const thirdPartyLabels = repoFromThirdParty.labels?.map((l) => l.val)
+      expect(currentAuthorityLabels).toContain('spam')
+      expect(currentAuthorityLabels).not.toContain('weird')
+      expect(thirdPartyLabels).toContain('weird')
+      expect(thirdPartyLabels).not.toContain('spam')
+    })
+
+    it('includes labels from all authorities requested via header', async () => {
+      const authHeaders = await ozone.modHeaders()
+      const { data: recordIncludingExternalLabels } =
+        await agent.api.tools.ozone.moderation.getRepo(
+          { did: sc.dids.alice },
+          {
+            headers: {
+              ...authHeaders,
+              [LABELER_HEADER_NAME]: [
+                thirdPartyLabeler.ctx.cfg.service.did,
+                ozone.ctx.cfg.service.did,
+              ].join(','),
+            },
+          },
+        )
+      const labelVals = recordIncludingExternalLabels.labels?.map((l) => l.val)
+      const labelSources = recordIncludingExternalLabels.labels?.map(
+        (l) => l.src,
+      )
+      expect(labelVals).toContain('weird')
+      expect(labelVals).toContain('spam')
+      expect(labelSources).toContain(thirdPartyLabeler.ctx.cfg.service.did)
+      expect(labelSources).toContain(ozone.ctx.cfg.service.did)
+    })
+  })
+})

package/tests/get-lists.test.ts

@@ -0,0 +1,109 @@
+import {
+  SeedClient,
+  TestNetwork,
+  TestOzone,
+  basicSeed,
+  ModeratorClient,
+  RecordRef,
+} from '@atproto/dev-env'
+import AtpAgent, { BSKY_LABELER_DID } from '@atproto/api'
+import { TAKEDOWN_LABEL } from '../src/mod-service'
+
+describe('admin get lists', () => {
+  let network: TestNetwork
+  let ozone: TestOzone
+  let agent: AtpAgent
+  let appviewAgent: AtpAgent
+  let sc: SeedClient
+  let modClient: ModeratorClient
+  let alicesList: RecordRef
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_admin_get_lists',
+    })
+    ozone = network.ozone
+    agent = ozone.getClient()
+    appviewAgent = network.bsky.getClient()
+    sc = network.getSeedClient()
+    modClient = ozone.getModClient()
+    await basicSeed(sc)
+    alicesList = await sc.createList(sc.dids.alice, "Alice's List", 'mod')
+    AtpAgent.configure({ appLabelers: [ozone.ctx.cfg.service.did] })
+    await network.processAll()
+  })
+
+  afterAll(async () => {
+    AtpAgent.configure({ appLabelers: [BSKY_LABELER_DID] })
+    await network.close()
+  })
+
+  const getAlicesList = async () => {
+    const [{ data: fromOzone }, { data: fromAppview }] = await Promise.all([
+      agent.api.app.bsky.graph.getLists(
+        { actor: sc.dids.alice },
+        { headers: await ozone.modHeaders() },
+      ),
+      appviewAgent.api.app.bsky.graph.getLists({ actor: sc.dids.alice }),
+    ])
+
+    return { fromOzone, fromAppview }
+  }
+
+  it('returns lists from takendown account', async () => {
+    const beforeTakedown = await getAlicesList()
+    expect(beforeTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+    expect(beforeTakedown.fromAppview.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Takedown alice's account
+    await modClient.emitEvent({
+      event: { $type: 'tools.ozone.moderation.defs#modEventTakedown' },
+      subject: {
+        $type: 'com.atproto.admin.defs#repoRef',
+        did: sc.dids.alice,
+      },
+    })
+    await network.processAll()
+
+    const afterTakedown = await getAlicesList()
+
+    // Verify that takendown list is shown when queried through ozone but not through appview
+    expect(afterTakedown.fromAppview.lists.length).toBe(0)
+    expect(afterTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Reverse alice's account takedown
+    await modClient.emitEvent({
+      event: { $type: 'tools.ozone.moderation.defs#modEventReverseTakedown' },
+      subject: {
+        $type: 'com.atproto.admin.defs#repoRef',
+        did: sc.dids.alice,
+      },
+    })
+    await network.processAll()
+  })
+
+  it('returns takendown lists', async () => {
+    const beforeTakedown = await getAlicesList()
+    expect(beforeTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+    expect(beforeTakedown.fromAppview.lists[0].uri).toEqual(alicesList.uriStr)
+
+    // Takedown alice's list using a !takedown label
+    await network.bsky.db.db
+      .insertInto('label')
+      .values({
+        src: ozone.ctx.cfg.service.did,
+        uri: alicesList.uriStr,
+        cid: alicesList.cidStr,
+        val: TAKEDOWN_LABEL,
+        neg: false,
+        cts: new Date().toISOString(),
+      })
+      .execute()
+
+    const afterTakedown = await getAlicesList()
+
+    // Verify that takendown list is shown when queried through ozone but not through appview
+    expect(afterTakedown.fromAppview.lists.length).toBe(0)
+    expect(afterTakedown.fromOzone.lists[0].uri).toEqual(alicesList.uriStr)
+  })
+})