@atproto/bsky 0.0.14 → 0.0.15

package/dist/lexicon/index.d.ts

@@ -65,6 +65,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos';
 import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate';
 import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl';
 import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos';
+import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels';
 import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences';
 import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile';
 import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles';
@@ -149,6 +150,7 @@ export declare class AtprotoNS {
     repo: RepoNS;
     server: ServerNS;
     sync: SyncNS;
+    temp: TempNS;
     constructor(server: Server);
 }
 export declare class AdminNS {
@@ -245,6 +247,11 @@ export declare class SyncNS {
     requestCrawl<AV extends AuthVerifier>(cfg: ConfigOf<AV, ComAtprotoSyncRequestCrawl.Handler<ExtractAuth<AV>>, ComAtprotoSyncRequestCrawl.HandlerReqCtx<ExtractAuth<AV>>>): void;
     subscribeRepos<AV extends StreamAuthVerifier>(cfg: ConfigOf<AV, ComAtprotoSyncSubscribeRepos.Handler<ExtractAuth<AV>>, ComAtprotoSyncSubscribeRepos.HandlerReqCtx<ExtractAuth<AV>>>): void;
 }
+export declare class TempNS {
+    _server: Server;
+    constructor(server: Server);
+    fetchLabels<AV extends AuthVerifier>(cfg: ConfigOf<AV, ComAtprotoTempFetchLabels.Handler<ExtractAuth<AV>>, ComAtprotoTempFetchLabels.HandlerReqCtx<ExtractAuth<AV>>>): void;
+}
 export declare class AppNS {
     _server: Server;
     bsky: BskyNS;

package/dist/lexicon/lexicons.d.ts

@@ -358,6 +358,10 @@ export declare const schemaDict: {
                     inviteNote: {
                         type: string;
                     };
+                    emailConfirmedAt: {
+                        type: string;
+                        format: string;
+                    };
                 };
             };
             accountView: {
@@ -393,6 +397,10 @@ export declare const schemaDict: {
                     invitesDisabled: {
                         type: string;
                     };
+                    emailConfirmedAt: {
+                        type: string;
+                        format: string;
+                    };
                     inviteNote: {
                         type: string;
                     };
@@ -3585,6 +3593,46 @@ export declare const schemaDict: {
             };
         };
     };
+    ComAtprotoTempFetchLabels: {
+        lexicon: number;
+        id: string;
+        defs: {
+            main: {
+                type: string;
+                description: string;
+                parameters: {
+                    type: string;
+                    properties: {
+                        since: {
+                            type: string;
+                        };
+                        limit: {
+                            type: string;
+                            minimum: number;
+                            maximum: number;
+                            default: number;
+                        };
+                    };
+                };
+                output: {
+                    encoding: string;
+                    schema: {
+                        type: string;
+                        required: string[];
+                        properties: {
+                            labels: {
+                                type: string;
+                                items: {
+                                    type: string;
+                                    ref: string;
+                                };
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
     AppBskyActorDefs: {
         lexicon: number;
         id: string;
@@ -3889,6 +3937,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     required: string[];
@@ -3915,6 +3964,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     required: string[];
@@ -3997,6 +4047,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 key: string;
                 record: {
                     type: string;
@@ -5188,6 +5239,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     required: string[];
@@ -5316,6 +5368,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     required: string[];
@@ -5403,6 +5456,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     required: string[];
@@ -5551,6 +5605,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 key: string;
                 record: {
                     type: string;
@@ -5575,6 +5630,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 key: string;
                 record: {
                     type: string;
@@ -5691,6 +5747,7 @@ export declare const schemaDict: {
         id: string;
         defs: {
             main: {
+                description: string;
                 type: string;
                 key: string;
                 record: {
@@ -6609,6 +6666,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     properties: {
@@ -6639,6 +6697,7 @@ export declare const schemaDict: {
         defs: {
             main: {
                 type: string;
+                description: string;
                 parameters: {
                     type: string;
                     properties: {
@@ -7207,6 +7266,7 @@ export declare const ids: {
     ComAtprotoSyncNotifyOfUpdate: string;
     ComAtprotoSyncRequestCrawl: string;
     ComAtprotoSyncSubscribeRepos: string;
+    ComAtprotoTempFetchLabels: string;
     AppBskyActorDefs: string;
     AppBskyActorGetPreferences: string;
     AppBskyActorGetProfile: string;

package/dist/lexicon/types/com/atproto/admin/defs.d.ts

@@ -128,6 +128,7 @@ export interface RepoViewDetail {
     invites?: ComAtprotoServerDefs.InviteCode[];
     invitesDisabled?: boolean;
     inviteNote?: string;
+    emailConfirmedAt?: string;
     [k: string]: unknown;
 }
 export declare function isRepoViewDetail(v: unknown): v is RepoViewDetail;
@@ -140,6 +141,7 @@ export interface AccountView {
     invitedBy?: ComAtprotoServerDefs.InviteCode;
     invites?: ComAtprotoServerDefs.InviteCode[];
     invitesDisabled?: boolean;
+    emailConfirmedAt?: string;
     inviteNote?: string;
     [k: string]: unknown;
 }

package/dist/lexicon/types/com/atproto/temp/fetchLabels.d.ts

@@ -0,0 +1,33 @@
+import express from 'express';
+import { HandlerAuth } from '@atproto/xrpc-server';
+import * as ComAtprotoLabelDefs from '../label/defs';
+export interface QueryParams {
+    since?: number;
+    limit: number;
+}
+export declare type InputSchema = undefined;
+export interface OutputSchema {
+    labels: ComAtprotoLabelDefs.Label[];
+    [k: string]: unknown;
+}
+export declare type HandlerInput = undefined;
+export interface HandlerSuccess {
+    encoding: 'application/json';
+    body: OutputSchema;
+    headers?: {
+        [key: string]: string;
+    };
+}
+export interface HandlerError {
+    status: number;
+    message?: string;
+}
+export declare type HandlerOutput = HandlerError | HandlerSuccess;
+export declare type HandlerReqCtx<HA extends HandlerAuth = never> = {
+    auth: HA;
+    params: QueryParams;
+    input: HandlerInput;
+    req: express.Request;
+    res: express.Response;
+};
+export declare type Handler<HA extends HandlerAuth = never> = (ctx: HandlerReqCtx<HA>) => Promise<HandlerOutput> | HandlerOutput;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/bsky",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "license": "MIT",
   "description": "Reference implementation of app.bsky App View (Bluesky API)",
   "keywords": [
@@ -32,17 +32,17 @@
     "pg": "^8.10.0",
     "pino": "^8.15.0",
     "pino-http": "^8.2.1",
-    "sharp": "^0.31.2",
+    "sharp": "^0.32.6",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
     "@atproto/api": "^0.6.23",
     "@atproto/common": "^0.3.3",
-    "@atproto/crypto": "^0.2.3",
+    "@atproto/crypto": "^0.3.0",
     "@atproto/syntax": "^0.1.4",
-    "@atproto/identity": "^0.3.1",
+    "@atproto/identity": "^0.3.2",
     "@atproto/lexicon": "^0.3.0",
-    "@atproto/repo": "^0.3.4",
-    "@atproto/xrpc-server": "^0.4.0"
+    "@atproto/repo": "^0.3.5",
+    "@atproto/xrpc-server": "^0.4.1"
   },
   "devDependencies": {
     "@did-plc/server": "^0.0.1",
@@ -51,12 +51,11 @@
     "@types/express-serve-static-core": "^4.17.36",
     "@types/pg": "^8.6.6",
     "@types/qs": "^6.9.7",
-    "@types/sharp": "^0.31.0",
     "axios": "^0.27.2",
     "@atproto/api": "^0.6.23",
-    "@atproto/dev-env": "^0.2.14",
+    "@atproto/dev-env": "^0.2.15",
     "@atproto/lex-cli": "^0.2.4",
-    "@atproto/pds": "^0.3.2",
+    "@atproto/pds": "^0.3.3",
     "@atproto/xrpc": "^0.4.0"
   },
   "scripts": {

package/src/api/app/bsky/actor/getSuggestions.ts

@@ -42,12 +42,12 @@ const skeleton = async (
   ctx: Context,
 ): Promise<SkeletonState> => {
   const { db } = ctx
-  const { limit, cursor, viewer } = params
+  const { viewer } = params
+  const alreadyIncluded = parseCursor(params.cursor)
   const { ref } = db.db.dynamic
-  let suggestionsQb = db.db
+  const suggestions = await db.db
     .selectFrom('suggested_follow')
     .innerJoin('actor', 'actor.did', 'suggested_follow.did')
-    .innerJoin('profile_agg', 'profile_agg.did', 'actor.did')
     .where(notSoftDeletedClause(ref('actor')))
     .where('suggested_follow.did', '!=', viewer ?? '')
     .whereNotExists((qb) =>
@@ -57,27 +57,30 @@ const skeleton = async (
         .where('creator', '=', viewer ?? '')
         .whereRef('subjectDid', '=', ref('actor.did')),
     )
+    .if(alreadyIncluded.length > 0, (qb) =>
+      qb.where('suggested_follow.order', 'not in', alreadyIncluded),
+    )
     .selectAll()
-    .select('profile_agg.postsCount as postsCount')
-    .limit(limit)
     .orderBy('suggested_follow.order', 'asc')
+    .execute()
 
-  if (cursor) {
-    const cursorRow = await db.db
-      .selectFrom('suggested_follow')
-      .where('did', '=', cursor)
-      .selectAll()
-      .executeTakeFirst()
-    if (cursorRow) {
-      suggestionsQb = suggestionsQb.where(
-        'suggested_follow.order',
-        '>',
-        cursorRow.order,
-      )
-    }
-  }
-  const suggestions = await suggestionsQb.execute()
-  return { params, suggestions, cursor: suggestions.at(-1)?.did }
+  // always include first two
+  const firstTwo = suggestions.filter(
+    (row) => row.order === 1 || row.order === 2,
+  )
+  const rest = suggestions.filter((row) => row.order !== 1 && row.order !== 2)
+  const limited = firstTwo.concat(shuffle(rest)).slice(0, params.limit)
+
+  // if the result set ends up getting larger, consider using a seed included in the cursor for for the randomized shuffle
+  const cursor =
+    limited.length > 0
+      ? limited
+          .map((row) => row.order.toString())
+          .concat(alreadyIncluded.map((id) => id.toString()))
+          .join(':')
+      : undefined
+
+  return { params, suggestions: limited, cursor }
 }
 
 const hydration = async (state: SkeletonState, ctx: Context) => {
@@ -110,6 +113,27 @@ const presentation = (state: HydrationState) => {
   return { actors: suggestedActors, cursor }
 }
 
+const parseCursor = (cursor?: string): number[] => {
+  if (!cursor) {
+    return []
+  }
+  try {
+    return cursor
+      .split(':')
+      .map((id) => parseInt(id, 10))
+      .filter((id) => !isNaN(id))
+  } catch {
+    return []
+  }
+}
+
+const shuffle = <T>(arr: T[]): T[] => {
+  return arr
+    .map((value) => ({ value, sort: Math.random() }))
+    .sort((a, b) => a.sort - b.sort)
+    .map(({ value }) => value)
+}
+
 type Context = {
   db: Database
   actorService: ActorService

package/src/api/app/bsky/feed/getPostThread.ts

@@ -255,11 +255,15 @@ const getThreadData = async (
       .orderBy('sortAt', 'desc')
       .execute(),
   ])
-  const parentsByUri = parents.reduce((acc, parent) => {
-    return Object.assign(acc, { [parent.postUri]: parent })
+  // prevent self-referential loops
+  const includedPosts = new Set<string>([uri])
+  const parentsByUri = parents.reduce((acc, post) => {
+    return Object.assign(acc, { [post.uri]: post })
   }, {} as Record<string, FeedRow>)
   const childrenByParentUri = children.reduce((acc, child) => {
     if (!child.replyParent) return acc
+    if (includedPosts.has(child.uri)) return acc
+    includedPosts.add(child.uri)
     acc[child.replyParent] ??= []
     acc[child.replyParent].push(child)
     return acc
@@ -269,7 +273,12 @@ const getThreadData = async (
   return {
     post,
     parent: post.replyParent
-      ? getParentData(parentsByUri, post.replyParent, parentHeight)
+      ? getParentData(
+          parentsByUri,
+          includedPosts,
+          post.replyParent,
+          parentHeight,
+        )
       : undefined,
     replies: getChildrenData(childrenByParentUri, uri, depth),
   }
@@ -277,16 +286,19 @@ const getThreadData = async (
 
 const getParentData = (
   postsByUri: Record<string, FeedRow>,
+  includedPosts: Set<string>,
   uri: string,
   depth: number,
 ): PostThread | ParentNotFoundError | undefined => {
   if (depth < 1) return undefined
+  if (includedPosts.has(uri)) return undefined
+  includedPosts.add(uri)
   const post = postsByUri[uri]
   if (!post) return new ParentNotFoundError(uri)
   return {
     post,
     parent: post.replyParent
-      ? getParentData(postsByUri, post.replyParent, depth - 1)
+      ? getParentData(postsByUri, includedPosts, post.replyParent, depth - 1)
       : undefined,
     replies: [],
   }

package/src/api/app/bsky/feed/searchPosts.ts

@@ -0,0 +1,127 @@
+import AppContext from '../../../../context'
+import { Server } from '../../../../lexicon'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import AtpAgent from '@atproto/api'
+import { AtUri } from '@atproto/syntax'
+import { mapDefined } from '@atproto/common'
+import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/searchPosts'
+import { Database } from '../../../../db'
+import { FeedHydrationState, FeedService } from '../../../../services/feed'
+import { ActorService } from '../../../../services/actor'
+import { createPipeline } from '../../../../pipeline'
+
+export default function (server: Server, ctx: AppContext) {
+  const searchPosts = createPipeline(
+    skeleton,
+    hydration,
+    noBlocks,
+    presentation,
+  )
+  server.app.bsky.feed.searchPosts({
+    auth: ctx.authOptionalVerifier,
+    handler: async ({ auth, params }) => {
+      const viewer = auth.credentials.did
+      const db = ctx.db.getReplica('search')
+      const feedService = ctx.services.feed(db)
+      const actorService = ctx.services.actor(db)
+      const searchAgent = ctx.searchAgent
+      if (!searchAgent) {
+        throw new InvalidRequestError('Search not available')
+      }
+
+      const results = await searchPosts(
+        { ...params, viewer },
+        { db, feedService, actorService, searchAgent },
+      )
+
+      return {
+        encoding: 'application/json',
+        body: results,
+      }
+    },
+  })
+}
+
+const skeleton = async (
+  params: Params,
+  ctx: Context,
+): Promise<SkeletonState> => {
+  const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton({
+    q: params.q,
+    cursor: params.cursor,
+    limit: params.limit,
+  })
+  return {
+    params,
+    postUris: res.data.posts.map((a) => a.uri),
+    cursor: res.data.cursor,
+    hitsTotal: res.data.hitsTotal,
+  }
+}
+
+const hydration = async (
+  state: SkeletonState,
+  ctx: Context,
+): Promise<HydrationState> => {
+  const { feedService } = ctx
+  const { params, postUris } = state
+  const uris = new Set<string>(postUris)
+  const dids = new Set<string>(postUris.map((uri) => new AtUri(uri).hostname))
+  const hydrated = await feedService.feedHydration({
+    uris,
+    dids,
+    viewer: params.viewer,
+  })
+  return { ...state, ...hydrated }
+}
+
+const noBlocks = (state: HydrationState): HydrationState => {
+  const { viewer } = state.params
+  state.postUris = state.postUris.filter((uri) => {
+    const post = state.posts[uri]
+    if (!viewer || !post) return true
+    return !state.bam.block([viewer, post.creator])
+  })
+  return state
+}
+
+const presentation = (state: HydrationState, ctx: Context) => {
+  const { feedService, actorService } = ctx
+  const { postUris, profiles, params } = state
+  const actors = actorService.views.profileBasicPresentation(
+    Object.keys(profiles),
+    state,
+    { viewer: params.viewer },
+  )
+
+  const postViews = mapDefined(postUris, (uri) =>
+    feedService.views.formatPostView(
+      uri,
+      actors,
+      state.posts,
+      state.threadgates,
+      state.embeds,
+      state.labels,
+      state.lists,
+    ),
+  )
+  return { posts: postViews, cursor: state.cursor, hitsTotal: state.hitsTotal }
+}
+
+type Context = {
+  db: Database
+  feedService: FeedService
+  actorService: ActorService
+  searchAgent: AtpAgent
+}
+
+type Params = QueryParams & { viewer: string | null }
+
+type SkeletonState = {
+  params: Params
+  postUris: string[]
+  hitsTotal?: number
+  cursor?: string
+}
+
+type HydrationState = SkeletonState & FeedHydrationState

package/src/api/com/atproto/admin/reverseModerationAction.ts

@@ -85,9 +85,9 @@ export default function (server: Server, ctx: AppContext) {
         return { result, restored }
       })
 
-      if (restored) {
-        const { did, subjects } = restored
-        const agent = await ctx.pdsAdminAgent(did)
+      if (restored && ctx.moderationPushAgent) {
+        const agent = ctx.moderationPushAgent
+        const { subjects } = restored
         const results = await Promise.allSettled(
           subjects.map((subject) =>
             retryHttp(() =>

package/src/api/com/atproto/admin/takeModerationAction.ts

@@ -111,10 +111,10 @@ export default function (server: Server, ctx: AppContext) {
         return { result, takenDown }
       })
 
-      if (takenDown) {
+      if (takenDown && ctx.moderationPushAgent) {
+        const agent = ctx.moderationPushAgent
         const { did, subjects } = takenDown
         if (did && subjects.length > 0) {
-          const agent = await ctx.pdsAdminAgent(did)
           const results = await Promise.allSettled(
             subjects.map((subject) =>
               retryHttp(() =>

package/src/api/com/atproto/admin/util.ts

@@ -9,8 +9,9 @@ export const getPdsAccountInfo = async (
   ctx: AppContext,
   did: string,
 ): Promise<AccountView | null> => {
+  const agent = ctx.moderationPushAgent
+  if (!agent) return null
   try {
-    const agent = await ctx.pdsAdminAgent(did)
     const res = await agent.api.com.atproto.admin.getAccountInfo({ did })
     return res.data
   } catch (err) {
@@ -31,6 +32,7 @@ export const addAccountInfoToRepoViewDetail = (
     invitesDisabled: accountInfo.invitesDisabled,
     inviteNote: accountInfo.inviteNote,
     invites: accountInfo.invites,
+    emailConfirmedAt: accountInfo.emailConfirmedAt,
   }
 }
 

package/src/api/com/atproto/temp/fetchLabels.ts

@@ -0,0 +1,30 @@
+import { Server } from '../../../../lexicon'
+import AppContext from '../../../../context'
+
+export default function (server: Server, ctx: AppContext) {
+  server.com.atproto.temp.fetchLabels(async ({ params }) => {
+    const { limit } = params
+    const db = ctx.db.getReplica()
+    const since =
+      params.since !== undefined ? new Date(params.since).toISOString() : ''
+    const labelRes = await db.db
+      .selectFrom('label')
+      .selectAll()
+      .orderBy('label.cts', 'asc')
+      .where('cts', '>', since)
+      .limit(limit)
+      .execute()
+
+    const labels = labelRes.map((l) => ({
+      ...l,
+      cid: l.cid === '' ? undefined : l.cid,
+    }))
+
+    return {
+      encoding: 'application/json',
+      body: {
+        labels,
+      },
+    }
+  })
+}

package/src/api/index.ts

@@ -13,6 +13,7 @@ import getLikes from './app/bsky/feed/getLikes'
 import getListFeed from './app/bsky/feed/getListFeed'
 import getPostThread from './app/bsky/feed/getPostThread'
 import getPosts from './app/bsky/feed/getPosts'
+import searchPosts from './app/bsky/feed/searchPosts'
 import getActorLikes from './app/bsky/feed/getActorLikes'
 import getProfile from './app/bsky/actor/getProfile'
 import getProfiles from './app/bsky/actor/getProfiles'
@@ -52,6 +53,7 @@ import getModerationReport from './com/atproto/admin/getModerationReport'
 import getModerationReports from './com/atproto/admin/getModerationReports'
 import resolveHandle from './com/atproto/identity/resolveHandle'
 import getRecord from './com/atproto/repo/getRecord'
+import fetchLabels from './com/atproto/temp/fetchLabels'
 
 export * as health from './health'
 
@@ -74,6 +76,7 @@ export default function (server: Server, ctx: AppContext) {
   getListFeed(server, ctx)
   getPostThread(server, ctx)
   getPosts(server, ctx)
+  searchPosts(server, ctx)
   getActorLikes(server, ctx)
   getProfile(server, ctx)
   getProfiles(server, ctx)
@@ -114,5 +117,6 @@ export default function (server: Server, ctx: AppContext) {
   getModerationReports(server, ctx)
   resolveHandle(server, ctx)
   getRecord(server, ctx)
+  fetchLabels(server, ctx)
   return server
 }