@atproto/bsky 0.0.243 → 0.0.245

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/bsky",
-  "version": "0.0.243",
+  "version": "0.0.245",
   "license": "MIT",
   "description": "Reference implementation of app.bsky App View (Bluesky API)",
   "keywords": [
@@ -46,19 +46,19 @@
     "structured-headers": "^1.0.1",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "^5.0.0",
-    "undici": "^6.19.8",
+    "undici": "^8.5.0",
     "zod": "3.23.8",
-    "@atproto-labs/xrpc-utils": "^0.1.1",
-    "@atproto/api": "^0.20.16",
+    "@atproto-labs/fetch-node": "^0.3.2",
     "@atproto/common": "^0.6.3",
-    "@atproto-labs/fetch-node": "^0.3.1",
-    "@atproto/crypto": "^0.5.1",
+    "@atproto-labs/xrpc-utils": "^0.1.1",
+    "@atproto/api": "^0.20.18",
     "@atproto/did": "^0.5.1",
+    "@atproto/crypto": "^0.5.1",
     "@atproto/identity": "^0.5.1",
-    "@atproto/repo": "^0.10.1",
     "@atproto/lex": "^0.1.4",
-    "@atproto/sync": "^0.3.2",
+    "@atproto/sync": "^0.3.3",
     "@atproto/syntax": "^0.6.2",
+    "@atproto/repo": "^0.10.1",
     "@atproto/xrpc-server": "^0.11.2"
   },
   "devDependencies": {
@@ -73,7 +73,7 @@
     "@types/qs": "^6.9.7",
     "ts-node": "^10.8.2",
     "vitest": "^4.0.16",
-    "@atproto/pds": "^0.5.6"
+    "@atproto/pds": "^0.5.7"
   },
   "type": "module",
   "exports": {

package/proto/bsky.proto

@@ -815,20 +815,12 @@ message NotificationPreference {
   NotificationChannelPush push = 2;
 }
 
-enum ChatNotificationInclude {
-  CHAT_NOTIFICATION_INCLUDE_UNSPECIFIED = 0;
-  CHAT_NOTIFICATION_INCLUDE_ALL = 1;
-  CHAT_NOTIFICATION_INCLUDE_ACCEPTED = 2;
-}
-
-message ChatNotificationPreference {
-  ChatNotificationInclude include = 1;
-  NotificationChannelPush push = 2;
-}
-
 message NotificationPreferences {
   bytes entry = 1;
-  ChatNotificationPreference chat = 2;
+
+  reserved 2; // formerly "chat". This is now stored on the chat service.
+  reserved "chat";
+
   FilterableNotificationPreference follow = 3;
   FilterableNotificationPreference like = 4;
   FilterableNotificationPreference like_via_repost = 5;
@@ -1144,10 +1136,10 @@ enum SearchQueryLanguage {
   SEARCH_QUERY_LANGUAGE_ZH = 2; // Chinese (smartcn)
   SEARCH_QUERY_LANGUAGE_KO = 3; // Korean (nori)
   SEARCH_QUERY_LANGUAGE_TH = 4; // Thai
+  SEARCH_QUERY_LANGUAGE_AR = 5; // Arabic
 }
 
-// Shared request params
-
+// Shared request params for paginated methods
 message SearchParams {
   string query = 1;
   string viewer = 2;
@@ -1157,6 +1149,13 @@ message SearchParams {
   string cursor = 4;
 }
 
+// Shared params for typeahead-style searches, which aren't paginated.
+message SearchTypeaheadParams {
+  string query = 1;
+  string viewer = 2;
+  int32 limit = 3;
+}
+
 // Shared result types
 
 // SearchRecordResult represents a generic result hit corresponding to a record.
@@ -1195,7 +1194,8 @@ message PostsFilters {
   // Content filters
   repeated string domains = 3;
   repeated string urls = 4;
-  repeated string embed_uris = 5;
+  // Canonical request field for posts embedding these AT-URI records.
+  repeated string embedded_at_uris = 5;
   repeated string hashtags = 6;
 }
 
@@ -1246,6 +1246,11 @@ message SearchPostsV2Request {
 message SearchPostsV2Response {
   repeated SearchRecordResult posts = 1;
   PageInfo page_info = 2;
+  // Detected CJK/Thai/Arabic languages of the query, usually one. Han-only text
+  // returns ["zh", "ja"] since it's ambiguous between Chinese and Japanese.
+  // set query_language on the request to disambiguate.
+  // Currently left empty for other scripts (e.g. Latin-based languages like English).
+  repeated string detected_query_languages = 3;
 }
 
 message SearchActorsV2Request {
@@ -1258,9 +1263,7 @@ message SearchActorsV2Response {
 }
 
 message SearchActorsTypeaheadRequest {
-  string viewer = 1;
-  string query = 2;
-  int32 limit = 3;
+  SearchTypeaheadParams params = 1;
 }
 
 message SearchActorsTypeaheadResponse {

package/src/api/app/bsky/actor/searchActorsTypeahead.ts

@@ -95,9 +95,11 @@ const skeletonV2 = async (
   const term = params.q ?? params.term ?? ''
 
   const res = await ctx.dataplane.searchActorsTypeahead({
-    viewer: params.hydrateCtx.viewer ?? undefined,
-    query: term,
-    limit: params.limit,
+    params: {
+      query: term,
+      viewer: params.hydrateCtx.viewer ?? undefined,
+      limit: params.limit,
+    },
   })
   return {
     dids: res.actors.map(({ did }) => did as DidString),

package/src/api/app/bsky/feed/searchPostsV2.ts

@@ -0,0 +1,232 @@
+import { Timestamp } from '@bufbuild/protobuf'
+import { mapDefined } from '@atproto/common'
+import { AtUriString } from '@atproto/lex'
+import { InvalidRequestError, Server } from '@atproto/xrpc-server'
+import { AppContext } from '../../../../context.js'
+import { DataPlaneClient } from '../../../../data-plane/index.js'
+import {
+  PostSearchQuery,
+  parsePostSearchQuery,
+} from '../../../../data-plane/server/util.js'
+import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator.js'
+import { parseString } from '../../../../hydration/util.js'
+import { app } from '../../../../lexicons/index.js'
+import {
+  HydrationFnInput,
+  PresentationFnInput,
+  RulesFnInput,
+  SkeletonFnInput,
+  createPipeline,
+} from '../../../../pipeline.js'
+import {
+  SearchQueryLanguage,
+  SearchSortOrder,
+} from '../../../../proto/bsky_pb.js'
+import { uriToDid as creatorFromUri } from '../../../../util/uris.js'
+import { Views } from '../../../../views/index.js'
+import { resHeaders, resolveSearchV2Override } from '../../../util.js'
+
+export default function (server: Server, ctx: AppContext) {
+  const searchPostsV2 = createPipeline(
+    skeleton,
+    hydration,
+    noBlocksOrTagged,
+    presentation,
+  )
+  server.add(app.bsky.feed.searchPostsV2, {
+    auth: ctx.authVerifier.standard,
+    handler: async ({ auth, params, req }) => {
+      const { viewer, isModService, skipViewerBlocks } =
+        ctx.authVerifier.parseCreds(auth)
+
+      const labelers = ctx.reqLabelers(req)
+      const hydrateCtx = await ctx.hydrator.createContext({
+        labelers,
+        viewer,
+        skipViewerBlocks,
+        features: ctx.featureGatesClient.scope(
+          ctx.featureGatesClient.parseUserContextFromHandler({
+            viewer,
+            req,
+          }),
+        ),
+      })
+      const isV2Enabled =
+        hydrateCtx.features.checkGate(
+          hydrateCtx.features.Gate.SearchV2Enable,
+        ) || resolveSearchV2Override(req, ctx.cfg)
+      if (!isV2Enabled) {
+        throw new InvalidRequestError('Search v2 is not enabled')
+      }
+
+      const results = await searchPostsV2(
+        {
+          ...params,
+          hydrateCtx,
+          isModService,
+        },
+        ctx,
+      )
+      return {
+        encoding: 'application/json',
+        body: results,
+        headers: resHeaders({ labelers: hydrateCtx.labelers }),
+      }
+    },
+  })
+}
+
+const skeleton = async (
+  inputs: SkeletonFnInput<Context, Params>,
+): Promise<Skeleton> => {
+  const { ctx, params } = inputs
+  const query = params.query ?? ''
+  const parsedQuery = parsePostSearchQuery(query, {
+    author: params.authors?.[0],
+  })
+
+  const res = await ctx.dataplane.searchPostsV2({
+    params: {
+      query,
+      viewer: params.hydrateCtx.viewer ?? undefined,
+      limit: params.limit,
+      cursor: params.cursor,
+    },
+    sort: postSortToV2(params.sort),
+    filters: {
+      authors: params.authors ?? [],
+      mentions: params.mentions ?? [],
+      domains: params.domains ?? [],
+      urls: params.urls ?? [],
+      embeddedAtUris: params.embeddedAtUris ?? [],
+      hashtags: params.hashtags ?? [],
+    },
+    exclude: {
+      authors: params.excludeAuthors ?? [],
+      mentions: params.excludeMentions ?? [],
+      domains: params.excludeDomains ?? [],
+      urls: params.excludeUrls ?? [],
+      embeddedAtUris: params.excludeEmbeddedAtUris ?? [],
+      hashtags: params.excludeHashtags ?? [],
+    },
+    since: parseTimestamp(params.since),
+    until: parseTimestamp(params.until),
+    allTime: params.allTime,
+    language: params.language,
+    hasMedia: params.hasMedia,
+    hasVideo: params.hasVideo,
+    replyParentUri: params.replyParentUri,
+    threadRootUri: params.threadRootUri,
+    excludeReplies: params.excludeReplies,
+    repliesOnly: params.repliesOnly,
+    following: params.following,
+    queryLanguage: queryLanguageToV2(params.queryLanguage),
+  })
+  return {
+    posts: res.posts.map(({ uri }) => uri as AtUriString),
+    cursor: parseString(res.pageInfo?.cursor),
+    hitsTotal: res.pageInfo?.hitsTotal
+      ? Number(res.pageInfo.hitsTotal)
+      : undefined,
+    detectedQueryLanguages: res.detectedQueryLanguages,
+    parsedQuery,
+  }
+}
+
+const hydration = async (
+  inputs: HydrationFnInput<Context, Params, Skeleton>,
+) => {
+  const { ctx, params, skeleton } = inputs
+  return ctx.hydrator.hydratePosts(
+    skeleton.posts.map((uri) => ({ uri })),
+    params.hydrateCtx,
+  )
+}
+
+const noBlocksOrTagged = (inputs: RulesFnInput<Context, Params, Skeleton>) => {
+  const { ctx, params, skeleton, hydration } = inputs
+  const { parsedQuery } = skeleton
+
+  skeleton.posts = skeleton.posts.filter((uri) => {
+    const post = hydration.posts?.get(uri)
+    if (!post) return
+
+    const creator = creatorFromUri(uri)
+    const isCuratedSearch = params.sort === 'top'
+    const isPostByViewer = creator === params.hydrateCtx.viewer
+
+    if (isPostByViewer) return true
+    if (params.isModService) return true
+
+    if (ctx.views.viewerBlockExists(creator, hydration)) return false
+
+    const tagged = [...ctx.cfg.searchTagsHide].some((t) => post.tags.has(t))
+
+    if (isCuratedSearch && tagged) return false
+    if (!(parsedQuery.author || params.authors?.length) && tagged) return false
+    return true
+  })
+  return skeleton
+}
+
+const presentation = (
+  inputs: PresentationFnInput<Context, Params, Skeleton>,
+) => {
+  const { ctx, skeleton, hydration } = inputs
+  const posts = mapDefined(skeleton.posts, (uri) => {
+    const post = hydration.posts?.get(uri)
+    if (!post) return
+
+    return ctx.views.post(uri, hydration)
+  })
+  return {
+    posts,
+    cursor: skeleton.cursor,
+    hitsTotal: skeleton.hitsTotal,
+    detectedQueryLanguages: skeleton.detectedQueryLanguages,
+  }
+}
+
+type Context = {
+  cfg: AppContext['cfg']
+  dataplane: DataPlaneClient
+  hydrator: Hydrator
+  views: Views
+}
+
+type Params = app.bsky.feed.searchPostsV2.$Params & {
+  hydrateCtx: HydrateCtx
+  isModService: boolean
+}
+
+type Skeleton = {
+  posts: AtUriString[]
+  hitsTotal?: number
+  cursor?: string
+  detectedQueryLanguages?: string[]
+  parsedQuery: PostSearchQuery
+}
+
+const postSortToV2 = (sort: string): SearchSortOrder => {
+  if (sort === 'top') return SearchSortOrder.TOP
+  if (sort === 'recent') return SearchSortOrder.RECENT
+  return SearchSortOrder.UNSPECIFIED
+}
+
+const queryLanguageToV2 = (
+  lang: string | undefined,
+): SearchQueryLanguage | undefined => {
+  if (lang === 'ja') return SearchQueryLanguage.JA
+  if (lang === 'zh') return SearchQueryLanguage.ZH
+  if (lang === 'ko') return SearchQueryLanguage.KO
+  if (lang === 'th') return SearchQueryLanguage.TH
+  if (lang === 'ar') return SearchQueryLanguage.AR
+  return undefined
+}
+
+const parseTimestamp = (value: string | undefined): Timestamp | undefined => {
+  if (!value) return undefined
+  const date = new Date(value)
+  if (isNaN(date.getTime())) return undefined
+  return Timestamp.fromDate(date)
+}

package/src/api/app/bsky/notification/putPreferencesV2.ts

@@ -5,7 +5,7 @@ import { AppContext } from '../../../../context.js'
 import { app } from '../../../../lexicons/index.js'
 import { GetNotificationPreferencesResponse } from '../../../../proto/bsky_pb.js'
 import { Namespaces } from '../../../../stash.js'
-import { protobufToLex } from './util.js'
+import { DEFAULT_CHAT_PREFERENCE, protobufToLex } from './util.js'
 
 export default function (server: Server, ctx: AppContext) {
   server.add(app.bsky.notification.putPreferencesV2, {
@@ -57,5 +57,8 @@ const computePreferences = async (
 
   const currentPreferences = protobufToLex(res.preferences[0])
   const preferences = { ...currentPreferences, ...input.body }
+  // NOTE: See the deprecation notice on the lexicon. This field returns a static default value and shouldn't be used.
+  // Use the chat.bsky.notification.defs#preferences type instead.
+  preferences.chat = DEFAULT_CHAT_PREFERENCE
   return preferences
 }

package/src/api/app/bsky/notification/util.ts

@@ -1,7 +1,5 @@
 import { app } from '../../../../lexicons/index.js'
 import {
-  ChatNotificationInclude,
-  ChatNotificationPreference,
   FilterableNotificationPreference,
   NotificationInclude,
   NotificationPreference,
@@ -71,15 +69,11 @@ const ensurePreferences = (
   }
 }
 
-const protobufChatPreferenceToLex = (
-  p?: DeepPartial<ChatNotificationPreference>,
-): Partial<app.bsky.notification.defs.ChatPreference> => {
-  return {
-    include:
-      p?.include === ChatNotificationInclude.ACCEPTED ? 'accepted' : 'all',
-    push: p?.push?.enabled,
+export const DEFAULT_CHAT_PREFERENCE: app.bsky.notification.defs.ChatPreference =
+  {
+    include: 'all',
+    push: true,
   }
-}
 
 const protobufFilterablePreferenceToLex = (
   p?: DeepPartial<FilterableNotificationPreference>,
@@ -104,7 +98,9 @@ export const protobufToLex = (
   res: DeepPartial<NotificationPreferences>,
 ): app.bsky.notification.defs.Preferences => {
   return ensurePreferences({
-    chat: protobufChatPreferenceToLex(res.chat),
+    // NOTE: See the deprecation notice on the lexicon. This field returns a static default value and shouldn't be used.
+    // Use the chat.bsky.notification.defs#preferences type instead.
+    chat: DEFAULT_CHAT_PREFERENCE,
     follow: protobufFilterablePreferenceToLex(res.follow),
     like: protobufFilterablePreferenceToLex(res.like),
     likeViaRepost: protobufFilterablePreferenceToLex(res.likeViaRepost),

package/src/api/blob-dispatcher.ts

@@ -1,4 +1,4 @@
-import { Agent, Dispatcher, Pool, RetryAgent } from 'undici'
+import { Agent, Dispatcher, Pool, interceptors } from 'undici'
 import { isUnicastIp, unicastLookup } from '@atproto-labs/fetch-node'
 import { ServerConfig } from '../config.js'
 import { RETRYABLE_HTTP_STATUS_CODES } from '../util/retry.js'
@@ -27,11 +27,15 @@ export function createBlobDispatcher(cfg: ServerConfig): Dispatcher {
     },
   })
 
-  return cfg.proxyMaxRetries > 0
-    ? new RetryAgent(baseDispatcher, {
-        statusCodes: [...RETRYABLE_HTTP_STATUS_CODES],
-        methods: ['GET', 'HEAD'],
-        maxRetries: cfg.proxyMaxRetries,
-      })
-    : baseDispatcher
+  return baseDispatcher.compose(
+    interceptors.redirect({
+      maxRedirections: 10,
+      throwOnMaxRedirect: true,
+    }),
+    interceptors.retry({
+      statusCodes: [...RETRYABLE_HTTP_STATUS_CODES],
+      methods: ['GET', 'HEAD'],
+      maxRetries: cfg.proxyMaxRetries,
+    }),
+  )
 }

package/src/api/blob-resolver.ts

@@ -197,7 +197,6 @@ export async function streamBlob(
         path: url.pathname + url.search,
         headers,
         signal: options.signal,
-        maxRedirections: 10,
       },
       (upstream) => {
         headersReceived = true

package/src/api/index.ts

@@ -39,6 +39,7 @@ import getRepostedBy from './app/bsky/feed/getRepostedBy.js'
 import getSuggestedFeeds from './app/bsky/feed/getSuggestedFeeds.js'
 import getTimeline from './app/bsky/feed/getTimeline.js'
 import searchPosts from './app/bsky/feed/searchPosts.js'
+import searchPostsV2 from './app/bsky/feed/searchPostsV2.js'
 import getActorStarterPacks from './app/bsky/graph/getActorStarterPacks.js'
 import getBlocks from './app/bsky/graph/getBlocks.js'
 import getFollowers from './app/bsky/graph/getFollowers.js'
@@ -142,6 +143,7 @@ export default function (server: Server, ctx: AppContext) {
   getPostThreadV2(server, ctx)
   getPosts(server, ctx)
   searchPosts(server, ctx)
+  searchPostsV2(server, ctx)
   getActorLikes(server, ctx)
   getProfile(server, ctx)
   getProfiles(server, ctx)

package/src/data-plane/bsync/index.ts

@@ -4,6 +4,8 @@ import http from 'node:http'
 import { ConnectRouter } from '@connectrpc/connect'
 import { expressConnectMiddleware } from '@connectrpc/connect-express'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { TID } from '@atproto/common'
 import { lexParse } from '@atproto/lex'
 import { AtUri } from '@atproto/syntax'
@@ -20,7 +22,11 @@ import { Database } from '../server/db/index.js'
 import { countAll, excluded } from '../server/db/util.js'
 
 export class MockBsync {
-  constructor(public server: http.Server) {}
+  private terminator: httpTerminator.HttpTerminator
+
+  constructor(public server: http.Server) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(db: Database, port: number) {
     const app = express()
@@ -32,15 +38,11 @@ export class MockBsync {
   }
 
   async destroy() {
-    return new Promise<void>((resolve, reject) => {
-      this.server.close((err) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve()
-        }
-      })
-    })
+    await this.terminator.terminate()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/data-plane/server/background.ts

@@ -1,35 +1,66 @@
 import PQueue from 'p-queue'
 import { dbLogger } from '../../logger.js'
-import { Database } from './db/index.js'
 
 // A simple queue for in-process, out-of-band/backgrounded work
 
-export class BackgroundQueue {
-  queue: InstanceType<typeof PQueue> = new PQueue()
-  destroyed = false
-  constructor(public db: Database) {}
+type Task<TContext> = (ctx: TContext, signal: AbortSignal) => Promise<void>
 
-  add(task: Task) {
-    if (this.destroyed) {
-      return
-    }
-    this.queue
-      .add(() => task(this.db))
-      .catch((err) => {
+export type BackgroundQueueOptions = NonNullable<
+  ConstructorParameters<typeof PQueue>[0]
+> & {
+  concurrency: number
+}
+
+// @NOTE Keep this in sync with the BackgroundQueue in
+// - packages/bsky/src/data-plane/server/background.ts
+// - packages/ozone/src/background.ts
+// - packages/pds/src/background.ts
+export class BackgroundQueue<TContext = unknown> {
+  private abortController = new AbortController()
+  private queue: PQueue
+
+  public get signal(): AbortSignal {
+    return this.abortController.signal
+  }
+
+  public get destroyed() {
+    return this.signal.aborted
+  }
+
+  constructor(
+    private readonly context: TContext,
+    options?: BackgroundQueueOptions,
+  ) {
+    this.queue = new PQueue(options)
+  }
+
+  add(task: Task<TContext>) {
+    if (this.destroyed) return
+
+    this.queue.add<void>(async () => {
+      try {
+        // The task will receive a signal allowing it to abort if the
+        // backgroundQueue is destroyed.
+        await task(this.context, this.signal)
+      } catch (err) {
         dbLogger.error({ err }, 'background queue task failed')
-      })
+      }
+    })
   }
 
   async processAll() {
-    await this.queue.onIdle()
+    const { queue } = this
+    while (queue.size || queue.pending) await queue.onIdle()
   }
 
   // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks.
   // The application calls this only once http connections have drained (tasks no longer being added).
   async destroy() {
-    this.destroyed = true
-    await this.queue.onIdle()
+    if (this.destroyed) {
+      dbLogger.warn('BackgroundQueue.destroy() called multiple times')
+    }
+
+    this.abortController.abort()
+    return this.processAll()
   }
 }
-
-type Task = (db: Database) => Promise<void>

package/src/data-plane/server/index.ts

@@ -2,6 +2,8 @@ import events from 'node:events'
 import http from 'node:http'
 import { expressConnectMiddleware } from '@connectrpc/connect-express'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { IdResolver, MemoryCache } from '@atproto/identity'
 import { Database, DatabaseSchema } from './db/index.js'
 import createRoutes from './routes/index.js'
@@ -11,10 +13,14 @@ export type { DatabaseSchema }
 export { RepoSubscription } from './subscription.js'
 
 export class DataPlaneServer {
+  private terminator: httpTerminator.HttpTerminator
+
   constructor(
     public server: http.Server,
     public idResolver: IdResolver,
-  ) {}
+  ) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(db: Database, port: number, plcUrl?: string) {
     const app = express()
@@ -28,14 +34,10 @@ export class DataPlaneServer {
   }
 
   async destroy() {
-    return new Promise<void>((resolve, reject) => {
-      this.server.close((err) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve()
-        }
-      })
-    })
+    await this.terminator.terminate()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }

package/src/data-plane/server/indexing/index.ts

@@ -62,7 +62,7 @@ export class IndexingService {
   constructor(
     public db: Database,
     public idResolver: IdResolver,
-    public background: BackgroundQueue,
+    public background: BackgroundQueue<Database>,
   ) {
     this.records = {
       post: Post.makePlugin(this.db, this.background),

package/src/data-plane/server/indexing/plugins/block.ts

@@ -67,7 +67,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.block.main,
     insertFn,