@atproto/bsky 0.0.243 → 0.0.244

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/bsky",
-  "version": "0.0.243",
+  "version": "0.0.244",
   "license": "MIT",
   "description": "Reference implementation of app.bsky App View (Bluesky API)",
   "keywords": [
@@ -46,18 +46,18 @@
     "structured-headers": "^1.0.1",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "^5.0.0",
-    "undici": "^6.19.8",
+    "undici": "^8.5.0",
     "zod": "3.23.8",
-    "@atproto-labs/xrpc-utils": "^0.1.1",
-    "@atproto/api": "^0.20.16",
-    "@atproto/common": "^0.6.3",
-    "@atproto-labs/fetch-node": "^0.3.1",
+    "@atproto-labs/fetch-node": "^0.3.2",
+    "@atproto/api": "^0.20.17",
     "@atproto/crypto": "^0.5.1",
+    "@atproto/common": "^0.6.3",
     "@atproto/did": "^0.5.1",
-    "@atproto/identity": "^0.5.1",
-    "@atproto/repo": "^0.10.1",
+    "@atproto-labs/xrpc-utils": "^0.1.1",
     "@atproto/lex": "^0.1.4",
-    "@atproto/sync": "^0.3.2",
+    "@atproto/repo": "^0.10.1",
+    "@atproto/identity": "^0.5.1",
+    "@atproto/sync": "^0.3.3",
     "@atproto/syntax": "^0.6.2",
     "@atproto/xrpc-server": "^0.11.2"
   },
@@ -73,7 +73,7 @@
     "@types/qs": "^6.9.7",
     "ts-node": "^10.8.2",
     "vitest": "^4.0.16",
-    "@atproto/pds": "^0.5.6"
+    "@atproto/pds": "^0.5.7"
   },
   "type": "module",
   "exports": {

package/proto/bsky.proto

@@ -815,20 +815,12 @@ message NotificationPreference {
   NotificationChannelPush push = 2;
 }
 
-enum ChatNotificationInclude {
-  CHAT_NOTIFICATION_INCLUDE_UNSPECIFIED = 0;
-  CHAT_NOTIFICATION_INCLUDE_ALL = 1;
-  CHAT_NOTIFICATION_INCLUDE_ACCEPTED = 2;
-}
-
-message ChatNotificationPreference {
-  ChatNotificationInclude include = 1;
-  NotificationChannelPush push = 2;
-}
-
 message NotificationPreferences {
   bytes entry = 1;
-  ChatNotificationPreference chat = 2;
+
+  reserved 2; // formerly "chat". This is now stored on the chat service.
+  reserved "chat";
+
   FilterableNotificationPreference follow = 3;
   FilterableNotificationPreference like = 4;
   FilterableNotificationPreference like_via_repost = 5;
@@ -1144,10 +1136,10 @@ enum SearchQueryLanguage {
   SEARCH_QUERY_LANGUAGE_ZH = 2; // Chinese (smartcn)
   SEARCH_QUERY_LANGUAGE_KO = 3; // Korean (nori)
   SEARCH_QUERY_LANGUAGE_TH = 4; // Thai
+  SEARCH_QUERY_LANGUAGE_AR = 5; // Arabic
 }
 
-// Shared request params
-
+// Shared request params for paginated methods
 message SearchParams {
   string query = 1;
   string viewer = 2;
@@ -1157,6 +1149,13 @@ message SearchParams {
   string cursor = 4;
 }
 
+// Shared params for typeahead-style searches, which aren't paginated.
+message SearchTypeaheadParams {
+  string query = 1;
+  string viewer = 2;
+  int32 limit = 3;
+}
+
 // Shared result types
 
 // SearchRecordResult represents a generic result hit corresponding to a record.
@@ -1246,6 +1245,11 @@ message SearchPostsV2Request {
 message SearchPostsV2Response {
   repeated SearchRecordResult posts = 1;
   PageInfo page_info = 2;
+  // Detected CJK/Thai/Arabic languages of the query, usually one. Han-only text
+  // returns ["zh", "ja"] since it's ambiguous between Chinese and Japanese.
+  // set query_language on the request to disambiguate.
+  // Currently left empty for other scripts (e.g. Latin-based languages like English).
+  repeated string detected_query_languages = 3;
 }
 
 message SearchActorsV2Request {
@@ -1258,9 +1262,7 @@ message SearchActorsV2Response {
 }
 
 message SearchActorsTypeaheadRequest {
-  string viewer = 1;
-  string query = 2;
-  int32 limit = 3;
+  SearchTypeaheadParams params = 1;
 }
 
 message SearchActorsTypeaheadResponse {

package/src/api/app/bsky/actor/searchActorsTypeahead.ts

@@ -95,9 +95,11 @@ const skeletonV2 = async (
   const term = params.q ?? params.term ?? ''
 
   const res = await ctx.dataplane.searchActorsTypeahead({
-    viewer: params.hydrateCtx.viewer ?? undefined,
-    query: term,
-    limit: params.limit,
+    params: {
+      query: term,
+      viewer: params.hydrateCtx.viewer ?? undefined,
+      limit: params.limit,
+    },
   })
   return {
     dids: res.actors.map(({ did }) => did as DidString),

package/src/api/app/bsky/notification/putPreferencesV2.ts

@@ -5,7 +5,7 @@ import { AppContext } from '../../../../context.js'
 import { app } from '../../../../lexicons/index.js'
 import { GetNotificationPreferencesResponse } from '../../../../proto/bsky_pb.js'
 import { Namespaces } from '../../../../stash.js'
-import { protobufToLex } from './util.js'
+import { DEFAULT_CHAT_PREFERENCE, protobufToLex } from './util.js'
 
 export default function (server: Server, ctx: AppContext) {
   server.add(app.bsky.notification.putPreferencesV2, {
@@ -57,5 +57,8 @@ const computePreferences = async (
 
   const currentPreferences = protobufToLex(res.preferences[0])
   const preferences = { ...currentPreferences, ...input.body }
+  // NOTE: See the deprecation notice on the lexicon. This field returns a static default value and shouldn't be used.
+  // Use the chat.bsky.notification.defs#preferences type instead.
+  preferences.chat = DEFAULT_CHAT_PREFERENCE
   return preferences
 }

package/src/api/app/bsky/notification/util.ts

@@ -1,7 +1,5 @@
 import { app } from '../../../../lexicons/index.js'
 import {
-  ChatNotificationInclude,
-  ChatNotificationPreference,
   FilterableNotificationPreference,
   NotificationInclude,
   NotificationPreference,
@@ -71,15 +69,11 @@ const ensurePreferences = (
   }
 }
 
-const protobufChatPreferenceToLex = (
-  p?: DeepPartial<ChatNotificationPreference>,
-): Partial<app.bsky.notification.defs.ChatPreference> => {
-  return {
-    include:
-      p?.include === ChatNotificationInclude.ACCEPTED ? 'accepted' : 'all',
-    push: p?.push?.enabled,
+export const DEFAULT_CHAT_PREFERENCE: app.bsky.notification.defs.ChatPreference =
+  {
+    include: 'all',
+    push: true,
   }
-}
 
 const protobufFilterablePreferenceToLex = (
   p?: DeepPartial<FilterableNotificationPreference>,
@@ -104,7 +98,9 @@ export const protobufToLex = (
   res: DeepPartial<NotificationPreferences>,
 ): app.bsky.notification.defs.Preferences => {
   return ensurePreferences({
-    chat: protobufChatPreferenceToLex(res.chat),
+    // NOTE: See the deprecation notice on the lexicon. This field returns a static default value and shouldn't be used.
+    // Use the chat.bsky.notification.defs#preferences type instead.
+    chat: DEFAULT_CHAT_PREFERENCE,
     follow: protobufFilterablePreferenceToLex(res.follow),
     like: protobufFilterablePreferenceToLex(res.like),
     likeViaRepost: protobufFilterablePreferenceToLex(res.likeViaRepost),

package/src/api/blob-dispatcher.ts

@@ -1,4 +1,4 @@
-import { Agent, Dispatcher, Pool, RetryAgent } from 'undici'
+import { Agent, Dispatcher, Pool, interceptors } from 'undici'
 import { isUnicastIp, unicastLookup } from '@atproto-labs/fetch-node'
 import { ServerConfig } from '../config.js'
 import { RETRYABLE_HTTP_STATUS_CODES } from '../util/retry.js'
@@ -27,11 +27,15 @@ export function createBlobDispatcher(cfg: ServerConfig): Dispatcher {
     },
   })
 
-  return cfg.proxyMaxRetries > 0
-    ? new RetryAgent(baseDispatcher, {
-        statusCodes: [...RETRYABLE_HTTP_STATUS_CODES],
-        methods: ['GET', 'HEAD'],
-        maxRetries: cfg.proxyMaxRetries,
-      })
-    : baseDispatcher
+  return baseDispatcher.compose(
+    interceptors.redirect({
+      maxRedirections: 10,
+      throwOnMaxRedirect: true,
+    }),
+    interceptors.retry({
+      statusCodes: [...RETRYABLE_HTTP_STATUS_CODES],
+      methods: ['GET', 'HEAD'],
+      maxRetries: cfg.proxyMaxRetries,
+    }),
+  )
 }

package/src/api/blob-resolver.ts

@@ -197,7 +197,6 @@ export async function streamBlob(
         path: url.pathname + url.search,
         headers,
         signal: options.signal,
-        maxRedirections: 10,
       },
       (upstream) => {
         headersReceived = true

package/src/data-plane/bsync/index.ts

@@ -4,6 +4,8 @@ import http from 'node:http'
 import { ConnectRouter } from '@connectrpc/connect'
 import { expressConnectMiddleware } from '@connectrpc/connect-express'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { TID } from '@atproto/common'
 import { lexParse } from '@atproto/lex'
 import { AtUri } from '@atproto/syntax'
@@ -20,7 +22,11 @@ import { Database } from '../server/db/index.js'
 import { countAll, excluded } from '../server/db/util.js'
 
 export class MockBsync {
-  constructor(public server: http.Server) {}
+  private terminator: httpTerminator.HttpTerminator
+
+  constructor(public server: http.Server) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(db: Database, port: number) {
     const app = express()
@@ -32,15 +38,11 @@ export class MockBsync {
   }
 
   async destroy() {
-    return new Promise<void>((resolve, reject) => {
-      this.server.close((err) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve()
-        }
-      })
-    })
+    await this.terminator.terminate()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/data-plane/server/background.ts

@@ -1,35 +1,66 @@
 import PQueue from 'p-queue'
 import { dbLogger } from '../../logger.js'
-import { Database } from './db/index.js'
 
 // A simple queue for in-process, out-of-band/backgrounded work
 
-export class BackgroundQueue {
-  queue: InstanceType<typeof PQueue> = new PQueue()
-  destroyed = false
-  constructor(public db: Database) {}
+type Task<TContext> = (ctx: TContext, signal: AbortSignal) => Promise<void>
 
-  add(task: Task) {
-    if (this.destroyed) {
-      return
-    }
-    this.queue
-      .add(() => task(this.db))
-      .catch((err) => {
+export type BackgroundQueueOptions = NonNullable<
+  ConstructorParameters<typeof PQueue>[0]
+> & {
+  concurrency: number
+}
+
+// @NOTE Keep this in sync with the BackgroundQueue in
+// - packages/bsky/src/data-plane/server/background.ts
+// - packages/ozone/src/background.ts
+// - packages/pds/src/background.ts
+export class BackgroundQueue<TContext = unknown> {
+  private abortController = new AbortController()
+  private queue: PQueue
+
+  public get signal(): AbortSignal {
+    return this.abortController.signal
+  }
+
+  public get destroyed() {
+    return this.signal.aborted
+  }
+
+  constructor(
+    private readonly context: TContext,
+    options?: BackgroundQueueOptions,
+  ) {
+    this.queue = new PQueue(options)
+  }
+
+  add(task: Task<TContext>) {
+    if (this.destroyed) return
+
+    this.queue.add<void>(async () => {
+      try {
+        // The task will receive a signal allowing it to abort if the
+        // backgroundQueue is destroyed.
+        await task(this.context, this.signal)
+      } catch (err) {
         dbLogger.error({ err }, 'background queue task failed')
-      })
+      }
+    })
   }
 
   async processAll() {
-    await this.queue.onIdle()
+    const { queue } = this
+    while (queue.size || queue.pending) await queue.onIdle()
   }
 
   // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks.
   // The application calls this only once http connections have drained (tasks no longer being added).
   async destroy() {
-    this.destroyed = true
-    await this.queue.onIdle()
+    if (this.destroyed) {
+      dbLogger.warn('BackgroundQueue.destroy() called multiple times')
+    }
+
+    this.abortController.abort()
+    return this.processAll()
   }
 }
-
-type Task = (db: Database) => Promise<void>

package/src/data-plane/server/index.ts

@@ -2,6 +2,8 @@ import events from 'node:events'
 import http from 'node:http'
 import { expressConnectMiddleware } from '@connectrpc/connect-express'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { IdResolver, MemoryCache } from '@atproto/identity'
 import { Database, DatabaseSchema } from './db/index.js'
 import createRoutes from './routes/index.js'
@@ -11,10 +13,14 @@ export type { DatabaseSchema }
 export { RepoSubscription } from './subscription.js'
 
 export class DataPlaneServer {
+  private terminator: httpTerminator.HttpTerminator
+
   constructor(
     public server: http.Server,
     public idResolver: IdResolver,
-  ) {}
+  ) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(db: Database, port: number, plcUrl?: string) {
     const app = express()
@@ -28,14 +34,10 @@ export class DataPlaneServer {
   }
 
   async destroy() {
-    return new Promise<void>((resolve, reject) => {
-      this.server.close((err) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve()
-        }
-      })
-    })
+    await this.terminator.terminate()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }

package/src/data-plane/server/indexing/index.ts

@@ -62,7 +62,7 @@ export class IndexingService {
   constructor(
     public db: Database,
     public idResolver: IdResolver,
-    public background: BackgroundQueue,
+    public background: BackgroundQueue<Database>,
   ) {
     this.records = {
       post: Post.makePlugin(this.db, this.background),

package/src/data-plane/server/indexing/plugins/block.ts

@@ -67,7 +67,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.block.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/chat-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   const processor = new RecordProcessor(db, background, {
     schema: chat.bsky.actor.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/feed-generator.ts

@@ -63,7 +63,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.generator.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/follow.ts

@@ -114,7 +114,10 @@ const updateAggregates = async (db: DatabaseSchema, follow: IndexedFollow) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.follow.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/germ-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: com.germnetwork.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/labeler.ts

@@ -57,7 +57,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.labeler.service.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/like.ts

@@ -133,7 +133,10 @@ const updateAggregates = async (db: DatabaseSchema, like: IndexedLike) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.like.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list-block.ts

@@ -67,7 +67,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.listblock.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list-item.ts

@@ -75,7 +75,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.listitem.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list.ts

@@ -63,7 +63,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.list.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/notif-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.notification.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/post-gate.ts

@@ -84,7 +84,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.postgate.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/post.ts

@@ -535,7 +535,10 @@ const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.post.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/profile.ts

@@ -73,7 +73,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.actor.profile.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/repost.ts

@@ -158,7 +158,10 @@ const updateAggregates = async (db: DatabaseSchema, repost: IndexedRepost) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.repost.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/starter-pack.ts

@@ -57,7 +57,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.starterpack.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/status.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.actor.status.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/thread-gate.ts

@@ -84,7 +84,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.threadgate.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/verification.ts

@@ -96,7 +96,10 @@ const notifsForDelete = (
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.verification.main,
     insertFn,