@atproto/bsky 0.0.37 → 0.0.39

package/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts

@@ -2,7 +2,7 @@ import { mapDefined } from '@atproto/common'
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
 import { parseString } from '../../../../hydration/util'
-import { clearlyBadCursor } from '../../../util'
+import { clearlyBadCursor, resHeaders } from '../../../util'
 
 // THIS IS A TEMPORARY UNSPECCED ROUTE
 // @TODO currently mirrors getSuggestedFeeds and ignores the "query" param.
@@ -10,8 +10,10 @@ import { clearlyBadCursor } from '../../../util'
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.unspecced.getPopularFeedGenerators({
     auth: ctx.authVerifier.standardOptional,
-    handler: async ({ auth, params }) => {
+    handler: async ({ auth, params, req }) => {
       const viewer = auth.credentials.iss
+      const labelers = ctx.reqLabelers(req)
+      const hydrateCtx = { viewer, labelers }
 
       if (clearlyBadCursor(params.cursor)) {
         return {
@@ -40,7 +42,7 @@ export default function (server: Server, ctx: AppContext) {
         cursor = parseString(res.cursor)
       }
 
-      const hydration = await ctx.hydrator.hydrateFeedGens(uris, viewer)
+      const hydration = await ctx.hydrator.hydrateFeedGens(uris, hydrateCtx)
       const feedViews = mapDefined(uris, (uri) =>
         ctx.views.feedGenerator(uri, hydration),
       )
@@ -51,6 +53,7 @@ export default function (server: Server, ctx: AppContext) {
           feeds: feedViews,
           cursor,
         },
+        headers: resHeaders({ labelers }),
       }
     },
   })

package/src/api/com/atproto/admin/getAccountInfos.ts

@@ -5,18 +5,25 @@ import { INVALID_HANDLE } from '@atproto/syntax'
 
 export default function (server: Server, ctx: AppContext) {
   server.com.atproto.admin.getAccountInfos({
-    auth: ctx.authVerifier.roleOrModService,
-    handler: async ({ params }) => {
+    auth: ctx.authVerifier.optionalStandardOrRole,
+    handler: async ({ params, auth }) => {
       const { dids } = params
+      const { includeTakedowns } = ctx.authVerifier.parseCreds(auth)
+
       const actors = await ctx.hydrator.actor.getActors(dids, true)
 
       const infos = mapDefined(dids, (did) => {
         const info = actors.get(did)
         if (!info) return
+        if (info.takedownRef && !includeTakedowns) return
+        const profileRecord =
+          !info.profileTakedownRef || includeTakedowns
+            ? info.profile
+            : undefined
         return {
           did,
           handle: info.handle ?? INVALID_HANDLE,
-          relatedRecords: info.profile ? [info.profile] : undefined,
+          relatedRecords: profileRecord ? [profileRecord] : undefined,
           indexedAt: (info.sortedAt ?? new Date(0)).toISOString(),
         }
       })

package/src/api/index.ts

@@ -30,6 +30,7 @@ import unmuteActor from './app/bsky/graph/unmuteActor'
 import muteActorList from './app/bsky/graph/muteActorList'
 import unmuteActorList from './app/bsky/graph/unmuteActorList'
 import getSuggestedFollowsByActor from './app/bsky/graph/getSuggestedFollowsByActor'
+import getLabelerServices from './app/bsky/labeler/getServices'
 import searchActors from './app/bsky/actor/searchActors'
 import searchActorsTypeahead from './app/bsky/actor/searchActorsTypeahead'
 import getSuggestions from './app/bsky/actor/getSuggestions'
@@ -84,6 +85,7 @@ export default function (server: Server, ctx: AppContext) {
   muteActorList(server, ctx)
   unmuteActorList(server, ctx)
   getSuggestedFollowsByActor(server, ctx)
+  getLabelerServices(server, ctx)
   searchActors(server, ctx)
   searchActorsTypeahead(server, ctx)
   getSuggestions(server, ctx)

package/src/api/util.ts

@@ -1,9 +1,24 @@
-import express from 'express'
+import { ParsedLabelers, formatLabelerHeader } from '../util'
 
-export const setRepoRev = (res: express.Response, rev: string | null) => {
-  if (rev !== null) {
-    res.setHeader('Atproto-Repo-Rev', rev)
+export const ATPROTO_CONTENT_LABELERS = 'Atproto-Content-Labelers'
+export const ATPROTO_REPO_REV = 'Atproto-Repo-Rev'
+
+type ResHeaderOpts = {
+  labelers: ParsedLabelers
+  repoRev: string | null
+}
+
+export const resHeaders = (
+  opts: Partial<ResHeaderOpts>,
+): Record<string, string> => {
+  const headers = {}
+  if (opts.labelers) {
+    headers[ATPROTO_CONTENT_LABELERS] = formatLabelerHeader(opts.labelers)
+  }
+  if (opts.repoRev) {
+    headers[ATPROTO_REPO_REV] = opts.repoRev
   }
+  return headers
 }
 
 export const clearlyBadCursor = (cursor?: string) => {

package/src/auth-verifier.ts

@@ -258,7 +258,7 @@ export class AuthVerifier {
   ) {
     const viewer =
       creds.credentials.type === 'standard' ? creds.credentials.iss : null
-    const canViewTakedowns =
+    const includeTakedowns =
       (creds.credentials.type === 'role' && creds.credentials.admin) ||
       creds.credentials.type === 'mod_service' ||
       (creds.credentials.type === 'standard' &&
@@ -269,7 +269,7 @@ export class AuthVerifier {
 
     return {
       viewer,
-      canViewTakedowns,
+      includeTakedowns,
       canPerformTakedown,
     }
   }

package/src/context.ts

@@ -1,3 +1,4 @@
+import express from 'express'
 import * as plc from '@did-plc/lib'
 import { IdResolver } from '@atproto/identity'
 import AtpAgent from '@atproto/api'
@@ -10,6 +11,12 @@ import { Views } from './views'
 import { AuthVerifier } from './auth-verifier'
 import { BsyncClient } from './bsync'
 import { CourierClient } from './courier'
+import {
+  ParsedLabelers,
+  defaultLabelerHeader,
+  parseLabelerHeader,
+} from './util'
+import { httpLogger as log } from './logger'
 
 export class AppContext {
   constructor(
@@ -79,6 +86,19 @@ export class AppContext {
       keypair: this.signingKey,
     })
   }
+
+  reqLabelers(req: express.Request): ParsedLabelers {
+    const val = req.header('atproto-accept-labelers')
+    let parsed: ParsedLabelers | null
+    try {
+      parsed = parseLabelerHeader(val)
+    } catch (err) {
+      parsed = null
+      log.info({ err, val }, 'failed to parse labeler header')
+    }
+    if (!parsed) return defaultLabelerHeader(this.cfg.labelsFromIssuerDids)
+    return parsed
+  }
 }
 
 export default AppContext

package/src/data-plane/server/db/database-schema.ts

@@ -25,7 +25,6 @@ import * as record from './tables/record'
 import * as notification from './tables/notification'
 import * as notificationPushToken from './tables/notification-push-token'
 import * as didCache from './tables/did-cache'
-import * as moderation from './tables/moderation'
 import * as label from './tables/label'
 import * as algo from './tables/algo'
 import * as viewParam from './tables/view-param'
@@ -33,6 +32,7 @@ import * as suggestedFollow from './tables/suggested-follow'
 import * as suggestedFeed from './tables/suggested-feed'
 import * as taggedSuggestion from './tables/tagged-suggestion'
 import * as blobTakedown from './tables/blob-takedown'
+import * as labeler from './tables/labeler'
 
 export type DatabaseSchemaType = duplicateRecord.PartialDB &
   profile.PartialDB &
@@ -60,14 +60,14 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   notification.PartialDB &
   notificationPushToken.PartialDB &
   didCache.PartialDB &
-  moderation.PartialDB &
   label.PartialDB &
   algo.PartialDB &
   viewParam.PartialDB &
   suggestedFollow.PartialDB &
   suggestedFeed.PartialDB &
-  taggedSuggestion.PartialDB &
-  blobTakedown.PartialDB
+  blobTakedown.PartialDB &
+  labeler.PartialDB &
+  taggedSuggestion.PartialDB
 
 export type DatabaseSchema = Kysely<DatabaseSchemaType>
 

package/src/data-plane/server/db/migrations/20240226T225725627Z-labelers.ts

@@ -0,0 +1,27 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('labeler')
+    .addColumn('uri', 'varchar', (col) => col.primaryKey())
+    .addColumn('cid', 'varchar', (col) => col.notNull())
+    .addColumn('creator', 'varchar', (col) => col.notNull())
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('indexedAt', 'varchar', (col) => col.notNull())
+    .addColumn('sortAt', 'varchar', (col) =>
+      col
+        .generatedAlwaysAs(sql`least("createdAt", "indexedAt")`)
+        .stored()
+        .notNull(),
+    )
+    .execute()
+  await db.schema
+    .createIndex('labeler_order_by_idx')
+    .on('labeler')
+    .columns(['sortAt', 'cid'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('labeler').execute()
+}

package/src/data-plane/server/db/migrations/index.ts

@@ -33,3 +33,4 @@ export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-index
 export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
 export * as _20231220T225126090Z from './20231220T225126090Z-blob-takedowns'
 export * as _20240124T023719200Z from './20240124T023719200Z-tagged-suggestions'
+export * as _20240226T225725627Z from './20240226T225725627Z-labelers'

package/src/data-plane/server/db/tables/labeler.ts

@@ -0,0 +1,16 @@
+import { GeneratedAlways } from 'kysely'
+
+export const tableName = 'labeler'
+
+export interface Labeler {
+  uri: string
+  cid: string
+  creator: string
+  createdAt: string
+  indexedAt: string
+  sortAt: GeneratedAlways<string>
+}
+
+export type PartialDB = {
+  [tableName]: Labeler
+}

package/src/data-plane/server/indexing/index.ts

@@ -26,6 +26,7 @@ import * as ListItem from './plugins/list-item'
 import * as ListBlock from './plugins/list-block'
 import * as Block from './plugins/block'
 import * as FeedGenerator from './plugins/feed-generator'
+import * as Labeler from './plugins/labeler'
 import RecordProcessor from './processor'
 import { subLogger } from '../../../logger'
 import { retryHttp } from '../../../util/retry'
@@ -44,6 +45,7 @@ export class IndexingService {
     listBlock: ListBlock.PluginType
     block: Block.PluginType
     feedGenerator: FeedGenerator.PluginType
+    labeler: Labeler.PluginType
   }
 
   constructor(
@@ -63,6 +65,7 @@ export class IndexingService {
       listBlock: ListBlock.makePlugin(this.db, this.background),
       block: Block.makePlugin(this.db, this.background),
       feedGenerator: FeedGenerator.makePlugin(this.db, this.background),
+      labeler: Labeler.makePlugin(this.db, this.background),
     }
   }
 
@@ -298,6 +301,7 @@ export class IndexingService {
       .deleteFrom('feed_generator')
       .where('creator', '=', did)
       .execute()
+    await this.db.db.deleteFrom('labeler').where('creator', '=', did).execute()
     // lists
     await this.db.db
       .deleteFrom('list_item')

package/src/data-plane/server/indexing/plugins/labeler.ts

@@ -0,0 +1,77 @@
+import { Selectable } from 'kysely'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
+import { CID } from 'multiformats/cid'
+import * as Labeler from '../../../../lexicon/types/app/bsky/labeler/service'
+import * as lex from '../../../../lexicon/lexicons'
+import { Database } from '../../db'
+import { DatabaseSchema, DatabaseSchemaType } from '../../db/database-schema'
+import RecordProcessor from '../processor'
+import { BackgroundQueue } from '../../background'
+
+const lexId = lex.ids.AppBskyLabelerService
+type IndexedLabeler = Selectable<DatabaseSchemaType['labeler']>
+
+const insertFn = async (
+  db: DatabaseSchema,
+  uri: AtUri,
+  cid: CID,
+  obj: Labeler.Record,
+  timestamp: string,
+): Promise<IndexedLabeler | null> => {
+  if (uri.rkey !== 'self') return null
+  const inserted = await db
+    .insertInto('labeler')
+    .values({
+      uri: uri.toString(),
+      cid: cid.toString(),
+      creator: uri.host,
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
+      indexedAt: timestamp,
+    })
+    .onConflict((oc) => oc.doNothing())
+    .returningAll()
+    .executeTakeFirst()
+  return inserted || null
+}
+
+const findDuplicate = async (): Promise<AtUri | null> => {
+  return null
+}
+
+const notifsForInsert = () => {
+  return []
+}
+
+const deleteFn = async (
+  db: DatabaseSchema,
+  uri: AtUri,
+): Promise<IndexedLabeler | null> => {
+  const deleted = await db
+    .deleteFrom('labeler')
+    .where('uri', '=', uri.toString())
+    .returningAll()
+    .executeTakeFirst()
+  return deleted || null
+}
+
+const notifsForDelete = () => {
+  return { notifs: [], toDelete: [] }
+}
+
+export type PluginType = RecordProcessor<Labeler.Record, IndexedLabeler>
+
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue,
+): PluginType => {
+  return new RecordProcessor(db, background, {
+    lexId,
+    insertFn,
+    findDuplicate,
+    deleteFn,
+    notifsForInsert,
+    notifsForDelete,
+  })
+}
+
+export default makePlugin

package/src/data-plane/server/routes/interactions.ts

@@ -2,6 +2,7 @@ import { keyBy } from '@atproto/common'
 import { ServiceImpl } from '@connectrpc/connect'
 import { Service } from '../../../proto/bsky_connect'
 import { Database } from '../db'
+import { countAll } from '../db/util'
 
 export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   async getInteractionCounts(req) {
@@ -25,16 +26,31 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
     if (req.dids.length === 0) {
       return { followers: [], following: [], posts: [] }
     }
+    const { ref } = db.db.dynamic
     const res = await db.db
       .selectFrom('profile_agg')
-      .selectAll()
       .where('did', 'in', req.dids)
+      .selectAll('profile_agg')
+      .select([
+        db.db
+          .selectFrom('feed_generator')
+          .whereRef('creator', '=', ref('profile_agg.did'))
+          .select(countAll.as('val'))
+          .as('feedGensCount'),
+        db.db
+          .selectFrom('list')
+          .whereRef('creator', '=', ref('profile_agg.did'))
+          .select(countAll.as('val'))
+          .as('listsCount'),
+      ])
       .execute()
     const byDid = keyBy(res, 'did')
     return {
       followers: req.dids.map((uri) => byDid[uri]?.followersCount ?? 0),
       following: req.dids.map((uri) => byDid[uri]?.followsCount ?? 0),
       posts: req.dids.map((uri) => byDid[uri]?.postsCount ?? 0),
+      lists: req.dids.map((uri) => byDid[uri]?.listsCount ?? 0),
+      feeds: req.dids.map((uri) => byDid[uri]?.feedGensCount ?? 0),
     }
   },
 })

package/src/data-plane/server/routes/labels.ts

@@ -1,4 +1,5 @@
 import * as ui8 from 'uint8arrays'
+import { noUndefinedVals } from '@atproto/common'
 import { ServiceImpl } from '@connectrpc/connect'
 import { Service } from '../../../proto/bsky_connect'
 import { Database } from '../db'
@@ -17,10 +18,11 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
       .execute()
 
     const labels = res.map((l) => {
-      const formatted = {
+      const formatted = noUndefinedVals({
         ...l,
         cid: l.cid === '' ? undefined : l.cid,
-      }
+        neg: l.neg === true ? true : undefined,
+      })
       return ui8.fromString(JSON.stringify(formatted), 'utf8')
     })
     return { labels }

package/src/data-plane/server/routes/profile.ts

@@ -3,6 +3,7 @@ import { Service } from '../../../proto/bsky_connect'
 import { keyBy } from '@atproto/common'
 import { getRecords } from './records'
 import { Database } from '../db'
+import { sql } from 'kysely'
 
 export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   async getActors(req) {
@@ -13,8 +14,20 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
     const profileUris = dids.map(
       (did) => `at://${did}/app.bsky.actor.profile/self`,
     )
+    const { ref } = db.db.dynamic
     const [handlesRes, profiles] = await Promise.all([
-      db.db.selectFrom('actor').where('did', 'in', dids).selectAll().execute(),
+      db.db
+        .selectFrom('actor')
+        .where('did', 'in', dids)
+        .selectAll('actor')
+        .select([
+          db.db
+            .selectFrom('labeler')
+            .whereRef('creator', '=', ref('actor.did'))
+            .select(sql<true>`${true}`.as('val'))
+            .as('isLabeler'),
+        ])
+        .execute(),
       getRecords(db)({ uris: profileUris }),
     ])
     const byDid = keyBy(handlesRes, 'did')
@@ -27,6 +40,7 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
         takenDown: !!row?.takedownRef,
         takedownRef: row?.takedownRef || undefined,
         tombstonedAt: undefined, // in current implementation, tombstoned actors are deleted
+        labeler: row?.isLabeler ?? false,
       }
     })
     return { actors }

package/src/data-plane/server/routes/records.ts

@@ -20,6 +20,7 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   getProfileRecords: getRecords(db, ids.AppBskyActorProfile),
   getRepostRecords: getRecords(db, ids.AppBskyFeedRepost),
   getThreadGateRecords: getRecords(db, ids.AppBskyFeedThreadgate),
+  getLabelerRecords: getRecords(db, ids.AppBskyLabelerService),
 })
 
 export const getRecords =

package/src/hydration/actor.ts

@@ -15,6 +15,7 @@ export type Actor = {
   profileTakedownRef?: string
   sortedAt?: Date
   takedownRef?: string
+  isLabeler: boolean
 }
 
 export type Actors = HydrationMap<Actor>
@@ -36,6 +37,8 @@ export type ProfileAgg = {
   followers: number
   follows: number
   posts: number
+  lists: number
+  feeds: number
 }
 
 export type ProfileAggs = HydrationMap<ProfileAgg>
@@ -100,6 +103,7 @@ export class ActorHydrator {
         profileTakedownRef: safeTakedownRef(profile),
         sortedAt: profile?.sortedAt?.toDate(),
         takedownRef: safeTakedownRef(actor),
+        isLabeler: actor.labeler ?? false,
       })
     }, new HydrationMap<Actor>())
   }
@@ -143,6 +147,8 @@ export class ActorHydrator {
         followers: counts.followers[i] ?? 0,
         follows: counts.following[i] ?? 0,
         posts: counts.posts[i] ?? 0,
+        lists: counts.lists[i] ?? 0,
+        feeds: counts.feeds[i] ?? 0,
       })
     }, new HydrationMap<ProfileAgg>())
   }