@atproto/bsky 0.0.66 → 0.0.68

package/src/api/app/bsky/feed/getRepostedBy.ts

@@ -23,9 +23,13 @@ export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getRepostedBy({
     auth: ctx.authVerifier.standardOptional,
     handler: async ({ params, auth, req }) => {
-      const viewer = auth.credentials.iss
+      const { viewer, includeTakedowns } = ctx.authVerifier.parseCreds(auth)
       const labelers = ctx.reqLabelers(req)
-      const hydrateCtx = await ctx.hydrator.createContext({ labelers, viewer })
+      const hydrateCtx = await ctx.hydrator.createContext({
+        labelers,
+        viewer,
+        includeTakedowns,
+      })
       const result = await getRepostedBy({ ...params, hydrateCtx }, ctx)
 
       return {

package/src/api/app/bsky/graph/getLists.ts

@@ -1,12 +1,13 @@
 import { mapDefined } from '@atproto/common'
 import { Server } from '../../../../lexicon'
 import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getLists'
+import { REFERENCELIST } from '../../../../lexicon/types/app/bsky/graph/defs'
 import AppContext from '../../../../context'
 import {
   createPipeline,
   HydrationFnInput,
-  noRules,
   PresentationFnInput,
+  RulesFnInput,
   SkeletonFnInput,
 } from '../../../../pipeline'
 import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator'
@@ -14,7 +15,12 @@ import { Views } from '../../../../views'
 import { clearlyBadCursor, resHeaders } from '../../../util'
 
 export default function (server: Server, ctx: AppContext) {
-  const getLists = createPipeline(skeleton, hydration, noRules, presentation)
+  const getLists = createPipeline(
+    skeleton,
+    hydration,
+    noReferenceLists,
+    presentation,
+  )
   server.app.bsky.graph.getLists({
     auth: ctx.authVerifier.optionalStandardOrRole,
     handler: async ({ params, auth, req }) => {
@@ -59,6 +65,17 @@ const hydration = async (
   return ctx.hydrator.hydrateLists(listUris, params.hydrateCtx)
 }
 
+const noReferenceLists = (
+  input: RulesFnInput<Context, Params, SkeletonState>,
+) => {
+  const { skeleton, hydration } = input
+  skeleton.listUris = skeleton.listUris.filter((uri) => {
+    const list = hydration.lists?.get(uri)
+    return list?.record.purpose !== REFERENCELIST
+  })
+  return skeleton
+}
+
 const presentation = (
   input: PresentationFnInput<Context, Params, SkeletonState>,
 ) => {

package/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts

@@ -1,5 +1,6 @@
-import { mapDefined } from '@atproto/common'
+import { mapDefined, noUndefinedVals } from '@atproto/common'
 import { InvalidRequestError } from '@atproto/xrpc-server'
+import AtpAgent from '@atproto/api'
 import { Server } from '../../../../lexicon'
 import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getSuggestedFollowsByActor'
 import AppContext from '../../../../context'
@@ -27,14 +28,27 @@ export default function (server: Server, ctx: AppContext) {
       const viewer = auth.credentials.iss
       const labelers = ctx.reqLabelers(req)
       const hydrateCtx = await ctx.hydrator.createContext({ labelers, viewer })
-      const result = await getSuggestedFollowsByActor(
-        { ...params, hydrateCtx: hydrateCtx.copy({ viewer }) },
-        ctx,
-      )
+      const headers = noUndefinedVals({
+        'accept-language': req.headers['accept-language'],
+        'x-bsky-topics': Array.isArray(req.headers['x-bsky-topics'])
+          ? req.headers['x-bsky-topics'].join(',')
+          : req.headers['x-bsky-topics'],
+      })
+      const { headers: resultHeaders, ...result } =
+        await getSuggestedFollowsByActor(
+          { ...params, hydrateCtx: hydrateCtx.copy({ viewer }), headers },
+          ctx,
+        )
+      const responseHeaders = noUndefinedVals({
+        'content-language': resultHeaders?.['content-language'],
+      })
       return {
         encoding: 'application/json',
         body: result,
-        headers: resHeaders({ labelers: hydrateCtx.labelers }),
+        headers: {
+          ...responseHeaders,
+          ...resHeaders({ labelers: hydrateCtx.labelers }),
+        },
       }
     },
   })
@@ -42,17 +56,39 @@ export default function (server: Server, ctx: AppContext) {
 
 const skeleton = async (input: SkeletonFnInput<Context, Params>) => {
   const { params, ctx } = input
+  const gates = ctx.featureGates
   const [relativeToDid] = await ctx.hydrator.actor.getDids([params.actor])
   if (!relativeToDid) {
     throw new InvalidRequestError('Actor not found')
   }
-  const { dids, cursor } = await ctx.hydrator.dataplane.getFollowSuggestions({
-    actorDid: params.hydrateCtx.viewer,
-    relativeToDid,
-  })
-  return {
-    suggestedDids: dids,
-    cursor: cursor || undefined,
+
+  if (
+    ctx.suggestionsAgent &&
+    gates.check(
+      await gates.user({ did: params.hydrateCtx.viewer }),
+      gates.ids.NewSuggestedFollowsByActor,
+    )
+  ) {
+    const res =
+      await ctx.suggestionsAgent.api.app.bsky.unspecced.getSuggestionsSkeleton(
+        {
+          viewer: params.hydrateCtx.viewer ?? undefined,
+          relativeToDid,
+        },
+        { headers: params.headers },
+      )
+    return {
+      suggestedDids: res.data.actors.map((a) => a.did),
+      headers: res.headers,
+    }
+  } else {
+    const { dids } = await ctx.hydrator.dataplane.getFollowSuggestions({
+      actorDid: params.hydrateCtx.viewer,
+      relativeToDid,
+    })
+    return {
+      suggestedDids: dids,
+    }
   }
 }
 
@@ -80,22 +116,26 @@ const presentation = (
   input: PresentationFnInput<Context, Params, SkeletonState>,
 ) => {
   const { ctx, hydration, skeleton } = input
-  const { suggestedDids } = skeleton
+  const { suggestedDids, headers } = skeleton
   const suggestions = mapDefined(suggestedDids, (did) =>
     ctx.views.profileDetailed(did, hydration),
   )
-  return { suggestions }
+  return { suggestions, headers }
 }
 
 type Context = {
   hydrator: Hydrator
   views: Views
+  suggestionsAgent: AtpAgent | undefined
+  featureGates: AppContext['featureGates']
 }
 
 type Params = QueryParams & {
   hydrateCtx: HydrateCtx & { viewer: string }
+  headers: Record<string, string>
 }
 
 type SkeletonState = {
   suggestedDids: string[]
+  headers?: Record<string, string>
 }

package/src/api/com/atproto/repo/getRecord.ts

@@ -4,27 +4,36 @@ import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
-  server.com.atproto.repo.getRecord(async ({ params }) => {
-    const { repo, collection, rkey, cid } = params
-    const [did] = await ctx.hydrator.actor.getDids([repo])
-    if (!did) {
-      throw new InvalidRequestError(`Could not find repo: ${repo}`)
-    }
+  server.com.atproto.repo.getRecord({
+    auth: ctx.authVerifier.optionalStandardOrRole,
+    handler: async ({ auth, params }) => {
+      const { repo, collection, rkey, cid } = params
+      const { includeTakedowns } = ctx.authVerifier.parseCreds(auth)
+      const [did] = await ctx.hydrator.actor.getDids([repo])
+      if (!did) {
+        throw new InvalidRequestError(`Could not find repo: ${repo}`)
+      }
 
-    const uri = AtUri.make(did, collection, rkey).toString()
-    const result = await ctx.hydrator.getRecord(uri, true)
+      const actors = await ctx.hydrator.actor.getActors([did], includeTakedowns)
+      if (!actors.get(did)) {
+        throw new InvalidRequestError(`Could not find repo: ${repo}`)
+      }
 
-    if (!result || (cid && result.cid !== cid)) {
-      throw new InvalidRequestError(`Could not locate record: ${uri}`)
-    }
+      const uri = AtUri.make(did, collection, rkey).toString()
+      const result = await ctx.hydrator.getRecord(uri, includeTakedowns)
 
-    return {
-      encoding: 'application/json' as const,
-      body: {
-        uri: uri,
-        cid: result.cid,
-        value: result.record,
-      },
-    }
+      if (!result || (cid && result.cid !== cid)) {
+        throw new InvalidRequestError(`Could not locate record: ${uri}`)
+      }
+
+      return {
+        encoding: 'application/json' as const,
+        body: {
+          uri: uri,
+          cid: result.cid,
+          value: result.record,
+        },
+      }
+    },
   })
 }

package/src/config.ts

@@ -35,6 +35,8 @@ export interface ServerConfigValues {
   labelsFromIssuerDids?: string[]
   // misc/dev
   blobCacheLocation?: string
+  statsigKey?: string
+  statsigEnv?: string
 }
 
 export class ServerConfig {
@@ -102,6 +104,14 @@ export class ServerConfig {
     assert(modServiceDid)
     assert(dataplaneUrls.length)
     assert(dataplaneHttpVersion === '1.1' || dataplaneHttpVersion === '2')
+    const statsigKey =
+      process.env.NODE_ENV === 'test'
+        ? 'secret-key'
+        : process.env.BSKY_STATSIG_KEY || undefined
+    const statsigEnv =
+      process.env.NODE_ENV === 'test'
+        ? 'test'
+        : process.env.BSKY_STATSIG_ENV || 'development'
     return new ServerConfig({
       version,
       debugMode,
@@ -132,6 +142,8 @@ export class ServerConfig {
       blobRateLimitBypassHostname,
       adminPasswords,
       modServiceDid,
+      statsigKey,
+      statsigEnv,
       ...stripUndefineds(overrides ?? {}),
     })
   }
@@ -264,6 +276,14 @@ export class ServerConfig {
   get blobCacheLocation() {
     return this.cfg.blobCacheLocation
   }
+
+  get statsigKey() {
+    return this.cfg.statsigKey
+  }
+
+  get statsigEnv() {
+    return this.cfg.statsigEnv
+  }
 }
 
 function stripUndefineds(

package/src/context.ts

@@ -11,6 +11,7 @@ import { Views } from './views'
 import { AuthVerifier } from './auth-verifier'
 import { BsyncClient } from './bsync'
 import { CourierClient } from './courier'
+import { FeatureGates } from './feature-gates'
 import {
   ParsedLabelers,
   defaultLabelerHeader,
@@ -32,6 +33,7 @@ export class AppContext {
       bsyncClient: BsyncClient
       courierClient: CourierClient
       authVerifier: AuthVerifier
+      featureGates: FeatureGates
     },
   ) {}
 
@@ -83,6 +85,10 @@ export class AppContext {
     return this.opts.authVerifier
   }
 
+  get featureGates(): FeatureGates {
+    return this.opts.featureGates
+  }
+
   async serviceAuthJwt(aud: string) {
     const iss = this.cfg.serverDid
     return createServiceJwt({

package/src/feature-gates.ts

@@ -0,0 +1,66 @@
+import { Statsig, StatsigUser } from 'statsig-node'
+import { sha256Hex } from '@atproto/crypto'
+
+import { featureGatesLogger } from './logger'
+import type { ServerConfig } from './config'
+
+export type Config = {
+  apiKey?: string
+  env?: 'development' | 'staging' | 'production' | string
+}
+
+export enum GateID {
+  NewSuggestedFollowsByActor = 'new_sugg_foll_by_actor',
+}
+
+/**
+ * @see https://docs.statsig.com/server/nodejsServerSDK
+ */
+export class FeatureGates {
+  ready = false
+  private statsig = Statsig
+  ids = GateID
+
+  constructor(private config: Config) {}
+
+  async start() {
+    try {
+      if (this.config.apiKey) {
+        /**
+         * Special handling in test mode, see {@link ServerConfig}
+         *
+         * {@link https://docs.statsig.com/server/nodejsServerSDK#local-overrides}
+         */
+        await this.statsig.initialize(this.config.apiKey, {
+          localMode: this.config.env === 'test',
+          environment: {
+            tier: this.config.env || 'development',
+          },
+        })
+        this.ready = true
+      }
+    } catch (err) {
+      featureGatesLogger.error({ err }, 'Failed to initialize StatSig')
+      this.ready = false
+    }
+  }
+
+  destroy() {
+    if (this.ready) {
+      this.ready = false
+      this.statsig.shutdown()
+    }
+  }
+
+  async user({ did }: { did: string }): Promise<StatsigUser> {
+    const userID = await sha256Hex(did)
+    return {
+      userID,
+    }
+  }
+
+  check(user: StatsigUser, gate: GateID) {
+    if (!this.ready) return false
+    return this.statsig.checkGateSync(user, gate)
+  }
+}

package/src/hydration/hydrator.ts

@@ -36,11 +36,13 @@ import {
 } from './label'
 import {
   HydrationMap,
-  Merges,
   RecordInfo,
   ItemRef,
   didFromUri,
   urisByCollection,
+  mergeMaps,
+  mergeNestedMaps,
+  mergeManyMaps,
 } from './util'
 import {
   FeedGenAggs,
@@ -57,6 +59,7 @@ import {
   FeedItem,
 } from './feed'
 import { ParsedLabelers } from '../util'
+import starterPack from '../data-plane/server/indexing/plugins/starter-pack'
 
 export class HydrateCtx {
   labelers = this.vals.labelers
@@ -102,6 +105,7 @@ export type HydrationState = {
   labelerViewers?: LabelerViewerStates
   labelerAggs?: LabelerAggs
   knownFollowers?: KnownFollowers
+  bidirectionalBlocks?: BidirectionalBlocks
 }
 
 export type PostBlock = { embed: boolean; reply: boolean }
@@ -111,6 +115,8 @@ type PostBlockPairs = { embed?: RelationshipPair; reply?: RelationshipPair }
 export type FollowBlock = boolean
 export type FollowBlocks = HydrationMap<FollowBlock>
 
+export type BidirectionalBlocks = HydrationMap<HydrationMap<boolean>>
+
 export class Hydrator {
   actor: ActorHydrator
   feed: FeedHydrator
@@ -215,13 +221,23 @@ export class Hydrator {
       )
     }
 
-    const knownFollowersDids = Array.from(knownFollowers.values())
+    const subjectsToKnownFollowersMap = Array.from(
+      knownFollowers.keys(),
+    ).reduce((acc, did) => {
+      const known = knownFollowers.get(did)
+      if (known) {
+        acc.set(did, known.followers)
+      }
+      return acc
+    }, new Map<string, string[]>())
+    const allKnownFollowerDids = Array.from(knownFollowers.values())
       .filter(Boolean)
       .flatMap((f) => f!.followers)
-    const allDids = Array.from(new Set(dids.concat(knownFollowersDids)))
-    const [state, profileAggs] = await Promise.all([
+    const allDids = Array.from(new Set(dids.concat(allKnownFollowerDids)))
+    const [state, profileAggs, bidirectionalBlocks] = await Promise.all([
       this.hydrateProfiles(allDids, ctx),
       this.actor.getProfileAggregates(dids),
+      this.hydrateBidirectionalBlocks(subjectsToKnownFollowersMap),
     ])
     const starterPackUriSet = new Set<string>()
     state.actors?.forEach((actor) => {
@@ -237,6 +253,7 @@ export class Hydrator {
       profileAggs,
       knownFollowers,
       ctx,
+      bidirectionalBlocks,
     })
   }
 
@@ -352,6 +369,10 @@ export class Hydrator {
       ...(urisLayer1ByCollection.get(ids.AppBskyLabelerService) ?? []),
       ...(urisLayer2ByCollection.get(ids.AppBskyLabelerService) ?? []),
     ].map((uri) => new AtUri(uri).hostname)
+    const nestedStarterPackUris = [
+      ...(urisLayer1ByCollection.get(ids.AppBskyGraphStarterpack) ?? []),
+      ...(urisLayer2ByCollection.get(ids.AppBskyGraphStarterpack) ?? []),
+    ]
     const posts =
       mergeManyMaps(postsLayer0, postsLayer1, postsLayer2) ?? postsLayer0
     const allPostUris = [...posts.keys()]
@@ -374,6 +395,7 @@ export class Hydrator {
       listState,
       feedGenState,
       labelerState,
+      starterPackState,
     ] = await Promise.all([
       this.feed.getPostAggregates(allRefs),
       ctx.viewer
@@ -385,6 +407,7 @@ export class Hydrator {
       this.hydrateLists([...nestedListUris, ...gateListUris], ctx),
       this.hydrateFeedGens(nestedFeedGenUris, ctx),
       this.hydrateLabelers(nestedLabelerDids, ctx),
+      this.hydrateStarterPacksBasic(nestedStarterPackUris, ctx),
     ])
     if (!ctx.includeTakedowns) {
       actionTakedownLabels(allPostUris, posts, labels)
@@ -395,6 +418,7 @@ export class Hydrator {
       listState,
       feedGenState,
       labelerState,
+      starterPackState,
       {
         posts,
         postAggs,
@@ -661,7 +685,7 @@ export class Hydrator {
   //     - list basic
   async hydrateLikes(uris: string[], ctx: HydrateCtx): Promise<HydrationState> {
     const [likes, profileState] = await Promise.all([
-      this.feed.getLikes(uris),
+      this.feed.getLikes(uris, ctx.includeTakedowns),
       this.hydrateProfiles(uris.map(didFromUri), ctx),
     ])
     return mergeStates(profileState, { likes, ctx })
@@ -673,7 +697,7 @@ export class Hydrator {
   //     - list basic
   async hydrateReposts(uris: string[], ctx: HydrateCtx) {
     const [reposts, profileState] = await Promise.all([
-      this.feed.getReposts(uris),
+      this.feed.getReposts(uris, ctx.includeTakedowns),
       this.hydrateProfiles(uris.map(didFromUri), ctx),
     ])
     return mergeStates(profileState, { reposts, ctx })
@@ -737,6 +761,30 @@ export class Hydrator {
     return { follows, followBlocks }
   }
 
+  async hydrateBidirectionalBlocks(
+    didMap: Map<string, string[]>, // DID -> DID[]
+  ): Promise<BidirectionalBlocks> {
+    const pairs: RelationshipPair[] = []
+    for (const [source, targets] of didMap) {
+      for (const target of targets) {
+        pairs.push([source, target])
+      }
+    }
+
+    const result = new HydrationMap<HydrationMap<boolean>>()
+    const blocks = await this.graph.getBidirectionalBlocks(pairs)
+
+    for (const [source, targets] of didMap) {
+      const didBlocks = new HydrationMap<boolean>()
+      for (const target of targets) {
+        didBlocks.set(target, blocks.isBlocked(source, target))
+      }
+      result.set(source, didBlocks)
+    }
+
+    return result
+  }
+
   // app.bsky.labeler.def#labelerViewDetailed
   // - labeler
   //   - profile
@@ -1022,23 +1070,17 @@ export const mergeStates = (
     labelerAggs: mergeMaps(stateA.labelerAggs, stateB.labelerAggs),
     labelerViewers: mergeMaps(stateA.labelerViewers, stateB.labelerViewers),
     knownFollowers: mergeMaps(stateA.knownFollowers, stateB.knownFollowers),
+    bidirectionalBlocks: mergeNestedMaps(
+      stateA.bidirectionalBlocks,
+      stateB.bidirectionalBlocks,
+    ),
   }
 }
 
-const mergeMaps = <M extends Merges>(mapA?: M, mapB?: M): M | undefined => {
-  if (!mapA) return mapB
-  if (!mapB) return mapA
-  return mapA.merge(mapB)
-}
-
 const mergeManyStates = (...states: HydrationState[]) => {
   return states.reduce(mergeStates, {} as HydrationState)
 }
 
-const mergeManyMaps = <T>(...maps: HydrationMap<T>[]) => {
-  return maps.reduce(mergeMaps, undefined as HydrationMap<T> | undefined)
-}
-
 const actionTakedownLabels = <T>(
   keys: string[],
   hydrationMap: HydrationMap<T>,

package/src/hydration/util.ts

@@ -25,6 +25,34 @@ export type RecordInfo<T> = {
   takedownRef: string | undefined
 }
 
+export const mergeMaps = <V, M extends HydrationMap<V>>(
+  mapA?: M,
+  mapB?: M,
+): M | undefined => {
+  if (!mapA) return mapB
+  if (!mapB) return mapA
+  return mapA.merge(mapB)
+}
+
+export const mergeNestedMaps = <V, M extends HydrationMap<HydrationMap<V>>>(
+  mapA?: M,
+  mapB?: M,
+): M | undefined => {
+  if (!mapA) return mapB
+  if (!mapB) return mapA
+
+  for (const [key, map] of mapB) {
+    const merged = mergeMaps(mapA.get(key) ?? undefined, map ?? undefined)
+    mapA.set(key, merged ?? null)
+  }
+
+  return mapA
+}
+
+export const mergeManyMaps = <T>(...maps: HydrationMap<T>[]) => {
+  return maps.reduce(mergeMaps, undefined as HydrationMap<T> | undefined)
+}
+
 export type ItemRef = { uri: string; cid?: string }
 
 export const parseRecord = <T>(

package/src/index.ts

@@ -23,6 +23,7 @@ import { Views } from './views'
 import { AuthVerifier } from './auth-verifier'
 import { authWithApiKey as bsyncAuth, createBsyncClient } from './bsync'
 import { authWithApiKey as courierAuth, createCourierClient } from './courier'
+import { FeatureGates } from './feature-gates'
 
 export * from './data-plane'
 export type { ServerConfigValues } from './config'
@@ -116,6 +117,11 @@ export class BskyAppView {
       adminPasses: config.adminPasswords,
     })
 
+    const featureGates = new FeatureGates({
+      apiKey: config.statsigKey,
+      env: config.statsigEnv,
+    })
+
     const ctx = new AppContext({
       cfg: config,
       dataplane,
@@ -128,6 +134,7 @@ export class BskyAppView {
       bsyncClient,
       courierClient,
       authVerifier,
+      featureGates,
     })
 
     let server = createServer({
@@ -154,6 +161,7 @@ export class BskyAppView {
   }
 
   async start(): Promise<http.Server> {
+    await this.ctx.featureGates.start()
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
@@ -166,6 +174,7 @@ export class BskyAppView {
 
   async destroy(): Promise<void> {
     await this.terminator?.terminate()
+    this.ctx.featureGates.destroy()
   }
 }
 

package/src/lexicon/lexicons.ts

@@ -4167,6 +4167,7 @@ export const schemaDict = {
             'lex:app.bsky.actor.defs#mutedWordsPref',
             'lex:app.bsky.actor.defs#hiddenPostsPref',
             'lex:app.bsky.actor.defs#bskyAppStatePref',
+            'lex:app.bsky.actor.defs#labelersPref',
           ],
         },
       },
@@ -4957,6 +4958,7 @@ export const schemaDict = {
               'lex:app.bsky.feed.defs#generatorView',
               'lex:app.bsky.graph.defs#listView',
               'lex:app.bsky.labeler.defs#labelerView',
+              'lex:app.bsky.graph.defs#starterPackViewBasic',
             ],
           },
         },
@@ -8710,6 +8712,12 @@ export const schemaDict = {
             cursor: {
               type: 'string',
             },
+            relativeToDid: {
+              type: 'string',
+              format: 'did',
+              description:
+                'DID of the account to get suggestions relative to. If not provided, suggestions will be based on the viewer.',
+            },
           },
         },
         output: {

package/src/lexicon/types/app/bsky/actor/defs.ts

@@ -185,6 +185,7 @@ export type Preferences = (
   | MutedWordsPref
   | HiddenPostsPref
   | BskyAppStatePref
+  | LabelersPref
   | { $type: string; [k: string]: unknown }
 )[]
 

package/src/lexicon/types/app/bsky/embed/record.ts

@@ -41,6 +41,7 @@ export interface View {
     | AppBskyFeedDefs.GeneratorView
     | AppBskyGraphDefs.ListView
     | AppBskyLabelerDefs.LabelerView
+    | AppBskyGraphDefs.StarterPackViewBasic
     | { $type: string; [k: string]: unknown }
   [k: string]: unknown
 }