@atproto/bsky 0.0.243 → 0.0.244

package/src/data-plane/server/indexing/processor.ts

@@ -37,7 +37,7 @@ type Notif = Insertable<Notification>
 export class RecordProcessor<TSchema extends l.RecordSchema, TRow> {
   constructor(
     private appDb: Database,
-    private background: BackgroundQueue,
+    private background: BackgroundQueue<Database>,
     private options: RecordProcessorOptions<TSchema, TRow>,
   ) {}
 

package/src/data-plane/server/routes/notifs.ts

@@ -6,8 +6,6 @@ import { lexParse } from '@atproto/lex'
 import { app } from '../../../lexicons/index.js'
 import { Service } from '../../../proto/bsky_connect.js'
 import {
-  ChatNotificationInclude,
-  ChatNotificationPreference,
   FilterableNotificationPreference,
   NotificationInclude,
   NotificationPreference,
@@ -203,17 +201,6 @@ export const notificationPreferencesLexToProtobuf = (
   p: app.bsky.notification.defs.Preferences,
   json: string,
 ): NotificationPreferences => {
-  const lexChatPreferenceToProtobuf = (
-    p: app.bsky.notification.defs.ChatPreference,
-  ): ChatNotificationPreference =>
-    new ChatNotificationPreference({
-      include:
-        p.include === 'accepted'
-          ? ChatNotificationInclude.ACCEPTED
-          : ChatNotificationInclude.ALL,
-      push: { enabled: p.push ?? true },
-    })
-
   const lexFilterablePreferenceToProtobuf = (
     p: app.bsky.notification.defs.FilterablePreference,
   ): FilterableNotificationPreference =>
@@ -236,7 +223,6 @@ export const notificationPreferencesLexToProtobuf = (
 
   return new NotificationPreferences({
     entry: Buffer.from(json),
-    chat: lexChatPreferenceToProtobuf(p.chat),
     follow: lexFilterablePreferenceToProtobuf(p.follow),
     like: lexFilterablePreferenceToProtobuf(p.like),
     likeViaRepost: lexFilterablePreferenceToProtobuf(p.likeViaRepost),

package/src/data-plane/server/routes/search.ts

@@ -139,8 +139,8 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => {
 
     async searchActorsTypeahead(req) {
       const { dids } = await searchActorsImpl({
-        term: req.query,
-        limit: req.limit || 10,
+        term: req.params?.query ?? '',
+        limit: req.params?.limit || 10,
       })
       return {
         actors: dids.map((did) => ({ did, score: 0 })),

package/src/data-plane/server/subscription.ts

@@ -7,52 +7,95 @@ import { Database } from './db/index.js'
 import { IndexingService } from './indexing/index.js'
 
 export class RepoSubscription {
-  firehose: Firehose
-  runner: MemoryRunner
-  background: BackgroundQueue
-  indexingSvc: IndexingService
+  private readonly abortController = new AbortController()
+  private current?: ReturnType<typeof createFirehose>
+  readonly background: BackgroundQueue<Database>
+  readonly indexingSvc: IndexingService
+  private readonly service: string
+  private readonly idResolver: IdResolver
 
-  constructor(
-    public opts: { service: string; db: Database; idResolver: IdResolver },
-  ) {
-    const { service, db, idResolver } = opts
-    this.background = new BackgroundQueue(db)
+  constructor({
+    service,
+    db,
+    idResolver,
+  }: {
+    service: string
+    db: Database
+    idResolver: IdResolver
+  }) {
+    this.service = service
+    this.idResolver = idResolver
+    this.background = new BackgroundQueue(db, {
+      // @TODO This has historically been using the default concurrency
+      // (Infinity) but we may want to limit this in the future to avoid
+      // overloading the database with indexing tasks.
+      concurrency: Number.POSITIVE_INFINITY,
+    })
     this.indexingSvc = new IndexingService(db, idResolver, this.background)
+  }
 
-    const { runner, firehose } = createFirehose({
-      idResolver,
-      service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
+  get destroyed() {
+    return this.abortController.signal.aborted
   }
 
-  start() {
-    this.firehose.start()
+  get running() {
+    return this.current != null
+  }
+
+  getCursor() {
+    return this.current?.runner.getCursor()
+  }
+
+  async start() {
+    this.abortController.signal.throwIfAborted()
+
+    if (!this.current) {
+      this.current = createFirehose({
+        idResolver: this.idResolver,
+        service: this.service,
+        indexingSvc: this.indexingSvc,
+      })
+      void this.current.firehose.start()
+    }
+  }
+
+  async stop() {
+    try {
+      if (this.current) {
+        const { firehose, runner } = this.current
+        this.current = undefined
+        try {
+          await firehose.destroy()
+        } finally {
+          await runner.destroy()
+        }
+      }
+    } finally {
+      await this.background.processAll()
+    }
   }
 
   async restart() {
-    await this.destroy()
-    const { runner, firehose } = createFirehose({
-      idResolver: this.opts.idResolver,
-      service: this.opts.service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
-    this.start()
+    await this.stop()
+    void this.start()
   }
 
   async processAll() {
-    await this.runner.processAll()
+    await this.current?.runner.processAll()
     await this.background.processAll()
   }
 
   async destroy() {
-    await this.firehose.destroy()
-    await this.runner.destroy()
-    await this.background.processAll()
+    this.abortController.abort()
+    try {
+      await this.stop()
+    } finally {
+      await this.background.destroy()
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/feature-gates/index.ts

@@ -72,10 +72,10 @@ export class FeatureGatesClient {
       this.client = new GrowthBookClient({
         apiHost: this.config.growthBookApiHost,
         clientKey: this.config.growthBookClientKey,
-        onFeatureUsage: (feature, result, userContext) => {
+        onFeatureUsage: async (feature, result, userContext) => {
           if (IGNORE_METRICS_FOR_GATES.has(feature as Gate)) return
 
-          this.metrics.track(
+          await this.metrics.track(
             'feature:viewed',
             {
               featureId: feature,
@@ -88,7 +88,7 @@ export class FeatureGatesClient {
             ),
           )
         },
-        trackingCallback: (experiment, result, userContext) => {
+        trackingCallback: async (experiment, result, userContext) => {
           /**
            * Experiments are only fired in a feature gate has an Experiment
            * attached in Growthbook. Howerver, we want to be extra sure that a
@@ -101,7 +101,7 @@ export class FeatureGatesClient {
           )
             return
 
-          this.metrics.track(
+          await this.metrics.track(
             'experiment:viewed',
             {
               experimentId: experiment.key,
@@ -154,14 +154,18 @@ export class FeatureGatesClient {
     }
   }
 
-  destroy() {
+  async destroy() {
     if (this.ready) {
       this.ready = false
       if (this.refreshInterval) {
         clearInterval(this.refreshInterval)
       }
     }
-    this.metrics.stop()
+    await this.metrics.stop()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 
   /**

package/src/feature-gates/metrics.test.ts

@@ -32,8 +32,8 @@ describe('MetricsClient', () => {
     global.fetch = fetchMock
   })
 
-  afterEach(() => {
-    client?.stop()
+  afterEach(async () => {
+    await client?.stop()
     vi.useRealTimers()
     vi.clearAllMocks()
   })
@@ -42,8 +42,8 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
-    client.track('view', { screen: 'home' })
+    await client.track('click', { button: 'submit' })
+    await client.track('view', { screen: 'home' })
 
     expect(fetchRequests).toHaveLength(0)
 
@@ -65,13 +65,13 @@ describe('MetricsClient', () => {
 
     // Add events up to maxBatchSize (should not flush yet)
     for (let i = 0; i < 5; i++) {
-      client.track('click', { button: `btn-${i}` })
+      await client.track('click', { button: `btn-${i}` })
     }
 
     expect(fetchRequests).toHaveLength(0)
 
     // One more event should trigger flush (> maxBatchSize)
-    client.track('click', { button: 'btn-trigger' })
+    await client.track('click', { button: 'btn-trigger' })
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -90,7 +90,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -119,7 +119,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush - should not throw
     vi.advanceTimersByTime(10_000)
@@ -140,12 +140,12 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     expect(fetchRequests).toHaveLength(0)
 
     // Stop should flush remaining events
-    client.stop()
+    await client.stop()
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -155,7 +155,7 @@ describe('MetricsClient', () => {
 
   it('does not send if trackingEndpoint is not configured', async () => {
     client = new MetricsClient<TestEvents>({})
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -170,7 +170,7 @@ describe('MetricsClient', () => {
     })
 
     // track() calls start() internally
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
     client.start()
     client.start()
 

package/src/feature-gates/metrics.ts

@@ -38,24 +38,24 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     if (this.started) return
     this.started = true
     this.flushInterval = setInterval(() => {
-      this.flush()
+      void this.flush()
     }, 10_000)
   }
 
-  stop() {
+  async stop() {
     if (this.flushInterval) {
       clearInterval(this.flushInterval)
       this.flushInterval = null
     }
-    this.flush()
+    await this.flush()
   }
 
-  track<E extends keyof M>(
+  async track<E extends keyof M>(
     event: E,
     payload: M[E],
     metadata: Record<string, any> = {},
   ) {
-    this.start()
+    await this.start()
 
     const e = {
       source: 'appview',
@@ -67,14 +67,14 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     this.queue.push(e)
 
     if (this.queue.length > this.maxBatchSize) {
-      this.flush()
+      await this.flush()
     }
   }
 
-  flush() {
+  async flush() {
     if (!this.queue.length) return
     const events = this.queue.splice(0, this.queue.length)
-    this.sendBatch(events)
+    await this.sendBatch(events)
   }
 
   private async sendBatch(events: Event<M>[]) {

package/src/index.ts

@@ -5,11 +5,8 @@ import compression from 'compression'
 import cors from 'cors'
 import { Etcd3 } from 'etcd3'
 import express from 'express'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
+// eslint-disable-next-line import/default
 import httpTerminator from 'http-terminator'
-// eslint-disable-next-line import/no-named-as-default-member
-const { createHttpTerminator } = httpTerminator
-type HttpTerminator = ReturnType<typeof createHttpTerminator>
 import { DAY, SECOND } from '@atproto/common'
 import { Keypair } from '@atproto/crypto'
 import { IdResolver } from '@atproto/identity'
@@ -63,7 +60,7 @@ export class BskyAppView {
   public ctx: AppContext
   public app: express.Application
   public server?: http.Server
-  private terminator?: HttpTerminator
+  private terminator?: httpTerminator.HttpTerminator
 
   constructor(opts: { ctx: AppContext; app: express.Application }) {
     this.ctx = opts.ctx
@@ -288,7 +285,7 @@ export class BskyAppView {
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
-    this.terminator = createHttpTerminator({ server })
+    this.terminator = httpTerminator.createHttpTerminator({ server })
     await events.once(server, 'listening')
     const { port } = server.address() as AddressInfo
     this.ctx.cfg.assignPort(port)
@@ -296,9 +293,19 @@ export class BskyAppView {
   }
 
   async destroy(): Promise<void> {
-    this.ctx.featureGatesClient.destroy()
-    await this.terminator?.terminate()
-    await this.ctx.etcd?.close()
+    try {
+      await this.ctx.featureGatesClient.destroy()
+    } finally {
+      try {
+        await this.terminator?.terminate()
+      } finally {
+        await this.ctx.etcd?.close()
+      }
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/tests/_util.ts

@@ -1,6 +1,7 @@
-import { Server } from 'node:http'
+import { RequestListener, createServer } from 'node:http'
 import { AddressInfo } from 'node:net'
-import { type Express } from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { expect } from 'vitest'
 import {
   $Typed,
@@ -265,19 +266,11 @@ export const stripViewerFromLabeler = (
   return stripViewer(labeler)
 }
 
-export async function startServer(app: Express) {
-  return new Promise<{
-    origin: string
-    server: Server
-    stop: () => Promise<void>
-  }>((resolve, reject) => {
+export async function startServer(listener: RequestListener) {
+  return new Promise<AsyncDisposable & { port: number }>((resolve, reject) => {
     const onListen = () => {
       const port = (server.address() as AddressInfo).port
-      resolve({
-        server,
-        origin: `http://localhost:${port}`,
-        stop: () => stopServer(server),
-      })
+      resolve({ port, [Symbol.asyncDispose]: () => terminator.terminate() })
       cleanup()
     }
     const onError = (err: Error) => {
@@ -289,21 +282,11 @@ export async function startServer(app: Express) {
       server.removeListener('error', onError)
     }
 
-    const server = app
+    const server = createServer(listener)
       .listen(0)
       .once('listening', onListen)
       .once('error', onError)
-  })
-}
 
-export async function stopServer(server: Server) {
-  return new Promise<void>((resolve, reject) => {
-    server.close((err) => {
-      if (err) {
-        reject(err)
-      } else {
-        resolve()
-      }
-    })
+    const terminator = httpTerminator.createHttpTerminator({ server })
   })
 }

package/tests/admin/admin-auth.test.ts

@@ -62,10 +62,10 @@ describe('admin auth', () => {
       $type: 'com.atproto.admin.defs#repoRef',
       did: sc.dids.bob,
     }
-  }, 20_000) // @NOTE seeding can take a while
+  })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('allows service auth requests from the configured appview did', async () => {

package/tests/admin/moderation.test.ts

@@ -46,7 +46,7 @@ describe('moderation', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('takes down accounts', async () => {

package/tests/auth.test.ts

@@ -1,4 +1,4 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
 import { AtpAgent, ids } from '@atproto/api'
 import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
 import { SeedClient, TestNetwork, usersSeed } from '@atproto/dev-env'
@@ -20,15 +20,13 @@ describe('auth', () => {
     agent = network.bsky.getAgent()
     sc = network.getSeedClient()
     await usersSeed(sc)
-    await network.processAll()
     alice = sc.dids.alice
     bob = sc.dids.bob
-  }, 20_000) // @NOTE seeding can take a while
-
-  afterAll(async () => {
-    await network.close()
   })
 
+  beforeEach(async () => network.processAll())
+  afterAll(async () => network?.close())
+
   // @TODO invalidations do not originate from appview frontends: requires identity event on the repo stream.
   it.skip('handles signing key change for service auth.', async () => {
     const issuer = sc.dids.alice

package/tests/blob-resolver.test.ts

@@ -1,5 +1,5 @@
 import { request } from 'undici'
-import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
 import { cidForCbor, verifyCidForBytes } from '@atproto/common'
 import { randomBytes } from '@atproto/crypto'
 import { TestNetwork, basicSeed } from '@atproto/dev-env'
@@ -17,15 +17,14 @@ describe('blob resolver', () => {
     })
     const sc = network.getSeedClient()
     await basicSeed(sc)
-    await network.processAll()
+
     fileDid = sc.dids.carol
     fileCid = sc.posts[fileDid][0].images[0].image.ref
     fileSize = sc.posts[fileDid][0].images[0].image.size
   })
 
-  afterAll(async () => {
-    await network.close()
-  })
+  beforeEach(async () => network.processAll())
+  afterAll(async () => network?.close())
 
   it('resolves blob with good signature check.', async () => {
     const response = await request(

package/tests/data-plane/db.test.ts

@@ -16,7 +16,7 @@ describe('db', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('handles client errors without crashing.', async () => {

package/tests/data-plane/duplicate-records.test.ts

@@ -21,7 +21,7 @@ describe('duplicate record', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const countRecords = async (db: Database, table: string) => {

package/tests/data-plane/handle-invalidation.test.ts

@@ -38,10 +38,10 @@ describe('handle invalidation', () => {
       }
       return origResolve(handle)
     }
-  }, 20_000) // @NOTE seeding can take a while
+  })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const backdateIndexedAt = async (did: string) => {