@atproto/bsky 0.0.243 → 0.0.245

package/src/data-plane/server/indexing/plugins/chat-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   const processor = new RecordProcessor(db, background, {
     schema: chat.bsky.actor.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/feed-generator.ts

@@ -63,7 +63,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.generator.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/follow.ts

@@ -114,7 +114,10 @@ const updateAggregates = async (db: DatabaseSchema, follow: IndexedFollow) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.follow.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/germ-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: com.germnetwork.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/labeler.ts

@@ -57,7 +57,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.labeler.service.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/like.ts

@@ -133,7 +133,10 @@ const updateAggregates = async (db: DatabaseSchema, like: IndexedLike) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.like.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list-block.ts

@@ -67,7 +67,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.listblock.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list-item.ts

@@ -75,7 +75,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.listitem.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/list.ts

@@ -63,7 +63,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.list.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/notif-declaration.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.notification.declaration.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/post-gate.ts

@@ -84,7 +84,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.postgate.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/post.ts

@@ -535,7 +535,10 @@ const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.post.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/profile.ts

@@ -73,7 +73,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.actor.profile.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/repost.ts

@@ -158,7 +158,10 @@ const updateAggregates = async (db: DatabaseSchema, repost: IndexedRepost) => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.repost.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/starter-pack.ts

@@ -57,7 +57,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.starterpack.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/status.ts

@@ -40,7 +40,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.actor.status.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/thread-gate.ts

@@ -84,7 +84,10 @@ const notifsForDelete = () => {
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.feed.threadgate.main,
     insertFn,

package/src/data-plane/server/indexing/plugins/verification.ts

@@ -96,7 +96,10 @@ const notifsForDelete = (
 }
 
 export type PluginType = ReturnType<typeof makePlugin>
-export const makePlugin = (db: Database, background: BackgroundQueue) => {
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue<Database>,
+) => {
   return new RecordProcessor(db, background, {
     schema: app.bsky.graph.verification.main,
     insertFn,

package/src/data-plane/server/indexing/processor.ts

@@ -37,7 +37,7 @@ type Notif = Insertable<Notification>
 export class RecordProcessor<TSchema extends l.RecordSchema, TRow> {
   constructor(
     private appDb: Database,
-    private background: BackgroundQueue,
+    private background: BackgroundQueue<Database>,
     private options: RecordProcessorOptions<TSchema, TRow>,
   ) {}
 

package/src/data-plane/server/routes/notifs.ts

@@ -6,8 +6,6 @@ import { lexParse } from '@atproto/lex'
 import { app } from '../../../lexicons/index.js'
 import { Service } from '../../../proto/bsky_connect.js'
 import {
-  ChatNotificationInclude,
-  ChatNotificationPreference,
   FilterableNotificationPreference,
   NotificationInclude,
   NotificationPreference,
@@ -203,17 +201,6 @@ export const notificationPreferencesLexToProtobuf = (
   p: app.bsky.notification.defs.Preferences,
   json: string,
 ): NotificationPreferences => {
-  const lexChatPreferenceToProtobuf = (
-    p: app.bsky.notification.defs.ChatPreference,
-  ): ChatNotificationPreference =>
-    new ChatNotificationPreference({
-      include:
-        p.include === 'accepted'
-          ? ChatNotificationInclude.ACCEPTED
-          : ChatNotificationInclude.ALL,
-      push: { enabled: p.push ?? true },
-    })
-
   const lexFilterablePreferenceToProtobuf = (
     p: app.bsky.notification.defs.FilterablePreference,
   ): FilterableNotificationPreference =>
@@ -236,7 +223,6 @@ export const notificationPreferencesLexToProtobuf = (
 
   return new NotificationPreferences({
     entry: Buffer.from(json),
-    chat: lexChatPreferenceToProtobuf(p.chat),
     follow: lexFilterablePreferenceToProtobuf(p.follow),
     like: lexFilterablePreferenceToProtobuf(p.like),
     likeViaRepost: lexFilterablePreferenceToProtobuf(p.likeViaRepost),

package/src/data-plane/server/routes/search.ts

@@ -139,8 +139,8 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => {
 
     async searchActorsTypeahead(req) {
       const { dids } = await searchActorsImpl({
-        term: req.query,
-        limit: req.limit || 10,
+        term: req.params?.query ?? '',
+        limit: req.params?.limit || 10,
       })
       return {
         actors: dids.map((did) => ({ did, score: 0 })),
@@ -159,6 +159,7 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => {
       return {
         posts: uris.map((uri) => ({ uri, score: 0 })),
         pageInfo: { cursor: cursor ?? '', hitsTotal: 0n },
+        detectedQueryLanguages: [],
       }
     },
 

package/src/data-plane/server/subscription.ts

@@ -7,52 +7,95 @@ import { Database } from './db/index.js'
 import { IndexingService } from './indexing/index.js'
 
 export class RepoSubscription {
-  firehose: Firehose
-  runner: MemoryRunner
-  background: BackgroundQueue
-  indexingSvc: IndexingService
+  private readonly abortController = new AbortController()
+  private current?: ReturnType<typeof createFirehose>
+  readonly background: BackgroundQueue<Database>
+  readonly indexingSvc: IndexingService
+  private readonly service: string
+  private readonly idResolver: IdResolver
 
-  constructor(
-    public opts: { service: string; db: Database; idResolver: IdResolver },
-  ) {
-    const { service, db, idResolver } = opts
-    this.background = new BackgroundQueue(db)
+  constructor({
+    service,
+    db,
+    idResolver,
+  }: {
+    service: string
+    db: Database
+    idResolver: IdResolver
+  }) {
+    this.service = service
+    this.idResolver = idResolver
+    this.background = new BackgroundQueue(db, {
+      // @TODO This has historically been using the default concurrency
+      // (Infinity) but we may want to limit this in the future to avoid
+      // overloading the database with indexing tasks.
+      concurrency: Number.POSITIVE_INFINITY,
+    })
     this.indexingSvc = new IndexingService(db, idResolver, this.background)
+  }
 
-    const { runner, firehose } = createFirehose({
-      idResolver,
-      service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
+  get destroyed() {
+    return this.abortController.signal.aborted
   }
 
-  start() {
-    this.firehose.start()
+  get running() {
+    return this.current != null
+  }
+
+  getCursor() {
+    return this.current?.runner.getCursor()
+  }
+
+  async start() {
+    this.abortController.signal.throwIfAborted()
+
+    if (!this.current) {
+      this.current = createFirehose({
+        idResolver: this.idResolver,
+        service: this.service,
+        indexingSvc: this.indexingSvc,
+      })
+      void this.current.firehose.start()
+    }
+  }
+
+  async stop() {
+    try {
+      if (this.current) {
+        const { firehose, runner } = this.current
+        this.current = undefined
+        try {
+          await firehose.destroy()
+        } finally {
+          await runner.destroy()
+        }
+      }
+    } finally {
+      await this.background.processAll()
+    }
   }
 
   async restart() {
-    await this.destroy()
-    const { runner, firehose } = createFirehose({
-      idResolver: this.opts.idResolver,
-      service: this.opts.service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
-    this.start()
+    await this.stop()
+    void this.start()
   }
 
   async processAll() {
-    await this.runner.processAll()
+    await this.current?.runner.processAll()
     await this.background.processAll()
   }
 
   async destroy() {
-    await this.firehose.destroy()
-    await this.runner.destroy()
-    await this.background.processAll()
+    this.abortController.abort()
+    try {
+      await this.stop()
+    } finally {
+      await this.background.destroy()
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/feature-gates/index.ts

@@ -72,10 +72,10 @@ export class FeatureGatesClient {
       this.client = new GrowthBookClient({
         apiHost: this.config.growthBookApiHost,
         clientKey: this.config.growthBookClientKey,
-        onFeatureUsage: (feature, result, userContext) => {
+        onFeatureUsage: async (feature, result, userContext) => {
           if (IGNORE_METRICS_FOR_GATES.has(feature as Gate)) return
 
-          this.metrics.track(
+          await this.metrics.track(
             'feature:viewed',
             {
               featureId: feature,
@@ -88,7 +88,7 @@ export class FeatureGatesClient {
             ),
           )
         },
-        trackingCallback: (experiment, result, userContext) => {
+        trackingCallback: async (experiment, result, userContext) => {
           /**
            * Experiments are only fired in a feature gate has an Experiment
            * attached in Growthbook. Howerver, we want to be extra sure that a
@@ -101,7 +101,7 @@ export class FeatureGatesClient {
           )
             return
 
-          this.metrics.track(
+          await this.metrics.track(
             'experiment:viewed',
             {
               experimentId: experiment.key,
@@ -154,14 +154,18 @@ export class FeatureGatesClient {
     }
   }
 
-  destroy() {
+  async destroy() {
     if (this.ready) {
       this.ready = false
       if (this.refreshInterval) {
         clearInterval(this.refreshInterval)
       }
     }
-    this.metrics.stop()
+    await this.metrics.stop()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 
   /**

package/src/feature-gates/metrics.test.ts

@@ -32,8 +32,8 @@ describe('MetricsClient', () => {
     global.fetch = fetchMock
   })
 
-  afterEach(() => {
-    client?.stop()
+  afterEach(async () => {
+    await client?.stop()
     vi.useRealTimers()
     vi.clearAllMocks()
   })
@@ -42,8 +42,8 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
-    client.track('view', { screen: 'home' })
+    await client.track('click', { button: 'submit' })
+    await client.track('view', { screen: 'home' })
 
     expect(fetchRequests).toHaveLength(0)
 
@@ -65,13 +65,13 @@ describe('MetricsClient', () => {
 
     // Add events up to maxBatchSize (should not flush yet)
     for (let i = 0; i < 5; i++) {
-      client.track('click', { button: `btn-${i}` })
+      await client.track('click', { button: `btn-${i}` })
     }
 
     expect(fetchRequests).toHaveLength(0)
 
     // One more event should trigger flush (> maxBatchSize)
-    client.track('click', { button: 'btn-trigger' })
+    await client.track('click', { button: 'btn-trigger' })
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -90,7 +90,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -119,7 +119,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush - should not throw
     vi.advanceTimersByTime(10_000)
@@ -140,12 +140,12 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     expect(fetchRequests).toHaveLength(0)
 
     // Stop should flush remaining events
-    client.stop()
+    await client.stop()
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -155,7 +155,7 @@ describe('MetricsClient', () => {
 
   it('does not send if trackingEndpoint is not configured', async () => {
     client = new MetricsClient<TestEvents>({})
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -170,7 +170,7 @@ describe('MetricsClient', () => {
     })
 
     // track() calls start() internally
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
     client.start()
     client.start()
 

package/src/feature-gates/metrics.ts

@@ -38,24 +38,24 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     if (this.started) return
     this.started = true
     this.flushInterval = setInterval(() => {
-      this.flush()
+      void this.flush()
     }, 10_000)
   }
 
-  stop() {
+  async stop() {
     if (this.flushInterval) {
       clearInterval(this.flushInterval)
       this.flushInterval = null
     }
-    this.flush()
+    await this.flush()
   }
 
-  track<E extends keyof M>(
+  async track<E extends keyof M>(
     event: E,
     payload: M[E],
     metadata: Record<string, any> = {},
   ) {
-    this.start()
+    await this.start()
 
     const e = {
       source: 'appview',
@@ -67,14 +67,14 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     this.queue.push(e)
 
     if (this.queue.length > this.maxBatchSize) {
-      this.flush()
+      await this.flush()
     }
   }
 
-  flush() {
+  async flush() {
     if (!this.queue.length) return
     const events = this.queue.splice(0, this.queue.length)
-    this.sendBatch(events)
+    await this.sendBatch(events)
   }
 
   private async sendBatch(events: Event<M>[]) {

package/src/index.ts

@@ -5,11 +5,8 @@ import compression from 'compression'
 import cors from 'cors'
 import { Etcd3 } from 'etcd3'
 import express from 'express'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
+// eslint-disable-next-line import/default
 import httpTerminator from 'http-terminator'
-// eslint-disable-next-line import/no-named-as-default-member
-const { createHttpTerminator } = httpTerminator
-type HttpTerminator = ReturnType<typeof createHttpTerminator>
 import { DAY, SECOND } from '@atproto/common'
 import { Keypair } from '@atproto/crypto'
 import { IdResolver } from '@atproto/identity'
@@ -63,7 +60,7 @@ export class BskyAppView {
   public ctx: AppContext
   public app: express.Application
   public server?: http.Server
-  private terminator?: HttpTerminator
+  private terminator?: httpTerminator.HttpTerminator
 
   constructor(opts: { ctx: AppContext; app: express.Application }) {
     this.ctx = opts.ctx
@@ -288,7 +285,7 @@ export class BskyAppView {
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
-    this.terminator = createHttpTerminator({ server })
+    this.terminator = httpTerminator.createHttpTerminator({ server })
     await events.once(server, 'listening')
     const { port } = server.address() as AddressInfo
     this.ctx.cfg.assignPort(port)
@@ -296,9 +293,19 @@ export class BskyAppView {
   }
 
   async destroy(): Promise<void> {
-    this.ctx.featureGatesClient.destroy()
-    await this.terminator?.terminate()
-    await this.ctx.etcd?.close()
+    try {
+      await this.ctx.featureGatesClient.destroy()
+    } finally {
+      try {
+        await this.terminator?.terminate()
+      } finally {
+        await this.ctx.etcd?.close()
+      }
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }