@atproto/bsky 0.0.39 → 0.0.40

package/src/data-plane/server/routes/labels.ts

@@ -3,28 +3,44 @@ import { noUndefinedVals } from '@atproto/common'
 import { ServiceImpl } from '@connectrpc/connect'
 import { Service } from '../../../proto/bsky_connect'
 import { Database } from '../db'
+import { Selectable } from 'kysely'
+import { Label } from '../db/tables/label'
+
+type LabelRow = Selectable<Label>
 
 export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   async getLabels(req) {
     const { subjects, issuers } = req
     if (subjects.length === 0 || issuers.length === 0) {
-      return { records: [] }
+      return { labels: [] }
     }
-    const res = await db.db
+    const res: LabelRow[] = await db.db
       .selectFrom('label')
       .where('uri', 'in', subjects)
       .where('src', 'in', issuers)
       .selectAll()
       .execute()
 
-    const labels = res.map((l) => {
-      const formatted = noUndefinedVals({
-        ...l,
-        cid: l.cid === '' ? undefined : l.cid,
-        neg: l.neg === true ? true : undefined,
+    const labelsBySubject = new Map<string, LabelRow[]>()
+    res.forEach((l) => {
+      const labels = labelsBySubject.get(l.uri) ?? []
+      labels.push(l)
+      labelsBySubject.set(l.uri, labels)
+    })
+
+    // intentionally duplicate label results, appview frontend should be defensive to this
+    const labels = subjects.flatMap((sub) => {
+      const labelsForSub = labelsBySubject.get(sub) ?? []
+      return labelsForSub.map((l) => {
+        const formatted = noUndefinedVals({
+          ...l,
+          cid: l.cid === '' ? undefined : l.cid,
+          neg: l.neg === true ? true : undefined,
+        })
+        return ui8.fromString(JSON.stringify(formatted), 'utf8')
       })
-      return ui8.fromString(JSON.stringify(formatted), 'utf8')
     })
+
     return { labels }
   },
 })

package/src/hydration/hydrator.ts

@@ -29,7 +29,13 @@ import {
   Labelers,
   Labels,
 } from './label'
-import { HydrationMap, RecordInfo, didFromUri, urisByCollection } from './util'
+import {
+  HydrationMap,
+  Merges,
+  RecordInfo,
+  didFromUri,
+  urisByCollection,
+} from './util'
 import {
   FeedGenAggs,
   FeedGens,
@@ -47,7 +53,17 @@ import {
 } from './feed'
 import { ParsedLabelers } from '../util'
 
-export type HydrateCtx = {
+export class HydrateCtx {
+  labelers = this.vals.labelers
+  viewer = this.vals.viewer
+  includeTakedowns = this.vals.includeTakedowns
+  constructor(private vals: HydrateCtxVals) {}
+  copy<V extends Partial<HydrateCtxVals>>(vals?: V): HydrateCtx & V {
+    return new HydrateCtx({ ...this.vals, ...vals }) as HydrateCtx & V
+  }
+}
+
+export type HydrateCtxVals = {
   labelers: ParsedLabelers
   viewer: string | null
   includeTakedowns?: boolean
@@ -91,12 +107,17 @@ export class Hydrator {
   feed: FeedHydrator
   graph: GraphHydrator
   label: LabelHydrator
+  serviceLabelers: Set<string>
 
-  constructor(public dataplane: DataPlaneClient) {
+  constructor(
+    public dataplane: DataPlaneClient,
+    serviceLabelers: string[] = [],
+  ) {
     this.actor = new ActorHydrator(dataplane)
     this.feed = new FeedHydrator(dataplane)
     this.graph = new GraphHydrator(dataplane)
     this.label = new LabelHydrator(dataplane)
+    this.serviceLabelers = new Set(serviceLabelers)
   }
 
   // app.bsky.actor.defs#profileView
@@ -549,13 +570,14 @@ export class Hydrator {
   ): Promise<HydrationState> {
     const [labelers, labelerAggs, labelerViewers, profileState] =
       await Promise.all([
-        this.label.getLabelers(dids),
+        this.label.getLabelers(dids, ctx.includeTakedowns),
         this.label.getLabelerAggregates(dids),
         ctx.viewer
           ? this.label.getLabelerViewerStates(dids, ctx.viewer)
           : undefined,
-        this.hydrateProfiles(dids.map(didFromUri), ctx),
+        this.hydrateProfiles(dids, ctx),
       ])
+    actionTakedownLabels(dids, labelers, profileState.labels ?? new Labels())
     return mergeStates(profileState, {
       labelers,
       labelerAggs,
@@ -613,8 +635,10 @@ export class Hydrator {
         undefined
       )
     } else if (collection === ids.AppBskyLabelerService) {
+      if (parsed.rkey !== 'self') return
+      const did = parsed.hostname
       return (
-        (await this.label.getLabelers([uri], includeTakedowns)).get(uri) ??
+        (await this.label.getLabelers([did], includeTakedowns)).get(did) ??
         undefined
       )
     } else if (collection === ids.AppBskyActorProfile) {
@@ -631,6 +655,30 @@ export class Hydrator {
       }
     }
   }
+
+  async createContext(vals: HydrateCtxVals) {
+    // ensures we're only apply labelers that exist and are not taken down
+    const labelers = vals.labelers.dids
+    const nonServiceLabelers = labelers.filter(
+      (did) => !this.serviceLabelers.has(did),
+    )
+    const labelerActors = await this.actor.getActors(
+      nonServiceLabelers,
+      vals.includeTakedowns,
+    )
+    const availableDids = labelers.filter(
+      (did) => this.serviceLabelers.has(did) || !!labelerActors.get(did),
+    )
+    const availableLabelers = {
+      dids: availableDids,
+      redact: vals.labelers.redact,
+    }
+    return new HydrateCtx({
+      labelers: availableLabelers,
+      viewer: vals.viewer,
+      includeTakedowns: vals.includeTakedowns,
+    })
+  }
 }
 
 const listUrisFromProfileViewer = (item: ProfileViewerState | null) => {
@@ -770,10 +818,7 @@ export const mergeStates = (
   }
 }
 
-const mergeMaps = <T>(
-  mapA?: HydrationMap<T>,
-  mapB?: HydrationMap<T>,
-): HydrationMap<T> | undefined => {
+const mergeMaps = <M extends Merges>(mapA?: M, mapB?: M): M | undefined => {
   if (!mapA) return mapB
   if (!mapB) return mapA
   return mapA.merge(mapB)

package/src/hydration/label.ts

@@ -3,6 +3,7 @@ import { Label } from '../lexicon/types/com/atproto/label/defs'
 import { Record as LabelerRecord } from '../lexicon/types/app/bsky/labeler/service'
 import {
   HydrationMap,
+  Merges,
   RecordInfo,
   parseJsonBytes,
   parseRecord,
@@ -16,10 +17,36 @@ export type { Label } from '../lexicon/types/com/atproto/label/defs'
 
 export type SubjectLabels = {
   isTakendown: boolean
-  labels: Label[]
+  labels: HydrationMap<Label> // src + val -> label
 }
 
-export type Labels = HydrationMap<SubjectLabels>
+export class Labels extends HydrationMap<SubjectLabels> implements Merges {
+  static key(label: Label) {
+    return `${label.src}::${label.val}`
+  }
+  merge(map: Labels): this {
+    map.forEach((theirs, key) => {
+      if (!theirs) return
+      const mine = this.get(key)
+      if (mine) {
+        mine.isTakendown = mine.isTakendown || theirs.isTakendown
+        mine.labels = mine.labels.merge(theirs.labels)
+      } else {
+        this.set(key, theirs)
+      }
+    })
+    return this
+  }
+  getBySubject(sub: string): Label[] {
+    const it = this.get(sub)?.labels.values()
+    if (!it) return []
+    const labels: Label[] = []
+    for (const label of it) {
+      if (label) labels.push(label)
+    }
+    return labels
+  }
+}
 
 export type LabelerAgg = {
   likes: number
@@ -43,8 +70,7 @@ export class LabelHydrator {
     subjects: string[],
     labelers: ParsedLabelers,
   ): Promise<Labels> {
-    if (!subjects.length || !labelers.dids.length)
-      return new HydrationMap<SubjectLabels>()
+    if (!subjects.length || !labelers.dids.length) return new Labels()
     const res = await this.dataplane.getLabels({
       subjects,
       issuers: labelers.dids,
@@ -57,11 +83,11 @@ export class LabelHydrator {
       if (!entry) {
         entry = {
           isTakendown: false,
-          labels: [],
+          labels: new HydrationMap(),
         }
         acc.set(label.uri, entry)
       }
-      entry.labels.push(label)
+      entry.labels.set(Labels.key(label), label)
       if (
         TAKEDOWN_LABELS.includes(label.val) &&
         !label.neg &&
@@ -70,7 +96,7 @@ export class LabelHydrator {
         entry.isTakendown = true
       }
       return acc
-    }, new HydrationMap<SubjectLabels>())
+    }, new Labels())
   }
 
   async getLabelers(

package/src/hydration/util.ts

@@ -5,8 +5,8 @@ import * as ui8 from 'uint8arrays'
 import { lexicons } from '../lexicon/lexicons'
 import { Record } from '../proto/bsky_pb'
 
-export class HydrationMap<T> extends Map<string, T | null> {
-  merge(map: HydrationMap<T>): HydrationMap<T> {
+export class HydrationMap<T> extends Map<string, T | null> implements Merges {
+  merge(map: HydrationMap<T>): this {
     map.forEach((val, key) => {
       this.set(key, val)
     })
@@ -14,6 +14,10 @@ export class HydrationMap<T> extends Map<string, T | null> {
   }
 }
 
+export interface Merges {
+  merge<T extends this>(map: T): this
+}
+
 export type RecordInfo<T> = {
   record: T
   cid: string

package/src/index.ts

@@ -77,7 +77,7 @@ export class BskyAppView {
       httpVersion: config.dataplaneHttpVersion,
       rejectUnauthorized: !config.dataplaneIgnoreBadTls,
     })
-    const hydrator = new Hydrator(dataplane)
+    const hydrator = new Hydrator(dataplane, config.labelsFromIssuerDids)
     const views = new Views(imgUriBuilder)
 
     const bsyncClient = createBsyncClient({

package/src/views/index.ts

@@ -136,8 +136,8 @@ export class Views {
       'self',
     ).toString()
     const labels = [
-      ...(state.labels?.get(did)?.labels ?? []),
-      ...(state.labels?.get(profileUri)?.labels ?? []),
+      ...(state.labels?.getBySubject(did) ?? []),
+      ...(state.labels?.getBySubject(profileUri) ?? []),
       ...this.selfLabels({
         uri: profileUri,
         cid: actor.profileCid?.toString(),
@@ -225,7 +225,7 @@ export class Views {
       return undefined
     }
     const listViewer = state.listViewers?.get(uri)
-    const labels = state.labels?.get(uri)?.labels ?? []
+    const labels = state.labels?.getBySubject(uri) ?? []
     const creator = new AtUri(uri).hostname
     return {
       uri,
@@ -281,7 +281,7 @@ export class Views {
 
     const uri = AtUri.make(did, ids.AppBskyLabelerService, 'self').toString()
     const labels = [
-      ...(state.labels?.get(uri)?.labels ?? []),
+      ...(state.labels?.getBySubject(uri) ?? []),
       ...this.selfLabels({
         uri,
         cid: labeler.cid.toString(),
@@ -351,7 +351,7 @@ export class Views {
     if (!creator) return
     const viewer = state.feedgenViewers?.get(uri)
     const aggs = state.feedgenAggs?.get(uri)
-    const labels = state.labels?.get(uri)?.labels ?? []
+    const labels = state.labels?.getBySubject(uri) ?? []
 
     return {
       uri,
@@ -408,7 +408,7 @@ export class Views {
       parsedUri.rkey,
     ).toString()
     const labels = [
-      ...(state.labels?.get(uri)?.labels ?? []),
+      ...(state.labels?.getBySubject(uri) ?? []),
       ...this.selfLabels({
         uri,
         cid: post.cid,
@@ -886,7 +886,7 @@ export class Views {
       recordInfo = state.follows?.get(notif.uri)
     }
     if (!recordInfo) return
-    const labels = state.labels?.get(notif.uri)?.labels ?? []
+    const labels = state.labels?.getBySubject(notif.uri) ?? []
     const selfLabels = this.selfLabels({
       uri: notif.uri,
       cid: recordInfo.cid,

package/tests/label-hydration.test.ts

@@ -4,6 +4,7 @@ import axios from 'axios'
 
 describe('label hydration', () => {
   let network: TestNetwork
+  let agent: AtpAgent
   let pdsAgent: AtpAgent
   let sc: SeedClient
 
@@ -16,6 +17,7 @@ describe('label hydration', () => {
     network = await TestNetwork.create({
       dbPostgresSchema: 'bsky_label_hydration',
     })
+    agent = network.bsky.getClient()
     pdsAgent = network.pds.getClient()
     sc = network.getSeedClient()
     await basicSeed(sc)
@@ -93,6 +95,35 @@ describe('label hydration', () => {
     )
   })
 
+  it('hydrates labels without duplication', async () => {
+    AtpAgent.configure({ appLabelers: [alice] })
+    pdsAgent.configureLabelersHeader([])
+    const res = await pdsAgent.api.app.bsky.actor.getProfiles(
+      { actors: [carol, carol] },
+      { headers: sc.getHeaders(bob) },
+    )
+    const { labels = [] } = res.data.profiles[0]
+    expect(labels.map((l) => ({ val: l.val, src: l.src }))).toEqual([
+      { src: alice, val: 'spam' },
+    ])
+  })
+
+  it('does not hydrate labels from takendown labeler', async () => {
+    AtpAgent.configure({ appLabelers: [alice, sc.dids.dan] })
+    pdsAgent.configureLabelersHeader([])
+    await network.bsky.ctx.dataplane.takedownActor({ did: alice })
+    const res = await pdsAgent.api.app.bsky.actor.getProfile(
+      { actor: carol },
+      { headers: sc.getHeaders(bob) },
+    )
+    const { labels = [] } = res.data
+    expect(labels).toEqual([])
+    expect(res.headers['atproto-content-labelers']).toEqual(
+      `${sc.dids.dan};redact`, // does not include alice
+    )
+    await network.bsky.ctx.dataplane.untakedownActor({ did: alice })
+  })
+
   it('hydrates labels onto list views.', async () => {
     AtpAgent.configure({ appLabelers: [labelerDid] })
     pdsAgent.configureLabelersHeader([])

package/tests/views/labeler-service.test.ts

@@ -136,10 +136,7 @@ describe('labeler service views', () => {
   })
 
   it('blocked by labeler takedown', async () => {
-    await network.bsky.ctx.dataplane.takedownRecord({
-      recordUri: aliceService.uriStr,
-    })
-
+    await network.bsky.ctx.dataplane.takedownActor({ did: alice })
     const res = await agent.api.app.bsky.labeler.getServices(
       { dids: [alice, bob] },
       { headers: await network.serviceHeaders(bob) },
@@ -149,8 +146,6 @@ describe('labeler service views', () => {
     expect(res.data.views[0].creator.did).toEqual(bob)
 
     // Cleanup
-    await network.bsky.ctx.dataplane.untakedownRecord({
-      recordUri: aliceService.uriStr,
-    })
+    await network.bsky.ctx.dataplane.untakedownActor({ did: alice })
   })
 })

package/tests/views/takedown-labels.test.ts

@@ -1,5 +1,6 @@
 import AtpAgent from '@atproto/api'
 import { TestNetwork, SeedClient, basicSeed, RecordRef } from '@atproto/dev-env'
+import { ids } from '../../src/lexicon/lexicons'
 
 describe('bsky takedown labels', () => {
   let network: TestNetwork
@@ -40,6 +41,48 @@ describe('bsky takedown labels', () => {
       'carol generator',
     )
 
+    // labelers
+    await sc.createAccount('labeler1', {
+      email: 'lab1@test.com',
+      handle: 'lab1.test',
+      password: 'lab1',
+    })
+    await sc.agent.api.com.atproto.repo.createRecord(
+      {
+        repo: sc.dids.labeler1,
+        collection: ids.AppBskyLabelerService,
+        rkey: 'self',
+        record: {
+          policies: { labelValues: ['spam'] },
+          createdAt: new Date().toISOString(),
+        },
+      },
+      {
+        headers: sc.getHeaders(sc.dids.labeler1),
+        encoding: 'application/json',
+      },
+    )
+    await sc.createAccount('labeler2', {
+      email: 'lab2@test.com',
+      handle: 'lab2.test',
+      password: 'lab2',
+    })
+    await sc.agent.api.com.atproto.repo.createRecord(
+      {
+        repo: sc.dids.labeler2,
+        collection: ids.AppBskyLabelerService,
+        rkey: 'self',
+        record: {
+          policies: { labelValues: ['spam'] },
+          createdAt: new Date().toISOString(),
+        },
+      },
+      {
+        headers: sc.getHeaders(sc.dids.labeler2),
+        encoding: 'application/json',
+      },
+    )
+
     await network.processAll()
 
     takendownSubjects = [
@@ -47,6 +90,7 @@ describe('bsky takedown labels', () => {
       sc.dids.carol,
       aliceListRef.uriStr,
       aliceGenRef.uriStr,
+      sc.dids.labeler1,
     ]
     const src = network.ozone.ctx.cfg.service.did
     const cts = new Date().toISOString()
@@ -123,6 +167,14 @@ describe('bsky takedown labels', () => {
     expect(res.data.feeds.at(0)?.uri).toEqual(bobGenRef.uriStr)
   })
 
+  it('takesdown labelers', async () => {
+    const res = await agent.api.app.bsky.labeler.getServices({
+      dids: [sc.dids.labeler1, sc.dids.labeler2],
+    })
+    expect(res.data.views.length).toBe(1)
+    expect(res.data.views[0].creator?.['did']).toBe(sc.dids.labeler2)
+  })
+
   it('only applies if the relevant labeler is configured', async () => {
     AtpAgent.configure({ appLabelers: ['did:web:example.com'] })
     const res = await agent.api.app.bsky.actor.getProfile({