@atproto/bsky 0.0.242 → 0.0.244

package/src/data-plane/server/routes/labels.ts

@@ -19,8 +19,8 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
       .selectFrom('label')
       .where('uri', 'in', subjects)
       .where('src', 'in', issuers)
-      .where((qb) =>
-        qb.where('exp', 'is', null).orWhere(sql`exp::timestamp > now()`),
+      .where((eb) =>
+        eb.or([eb('exp', 'is', null), sql<boolean>`exp::timestamp > now()`]),
       )
       .selectAll()
       .execute()

package/src/data-plane/server/routes/mutes.ts

@@ -88,12 +88,14 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
     const { ref } = db.db.dynamic
     let builder = db.db
       .selectFrom('list')
-      .whereExists(
-        db.db
-          .selectFrom('list_mute')
-          .where('list_mute.mutedByDid', '=', actorDid)
-          .whereRef('list_mute.listUri', '=', ref('list.uri'))
-          .selectAll(),
+      .where(({ exists }) =>
+        exists(
+          db.db
+            .selectFrom('list_mute')
+            .where('list_mute.mutedByDid', '=', actorDid)
+            .whereRef('list_mute.listUri', '=', ref('list.uri'))
+            .selectAll(),
+        ),
       )
       .selectAll('list')
 

package/src/data-plane/server/routes/notifs.ts

@@ -6,8 +6,6 @@ import { lexParse } from '@atproto/lex'
 import { app } from '../../../lexicons/index.js'
 import { Service } from '../../../proto/bsky_connect.js'
 import {
-  ChatNotificationInclude,
-  ChatNotificationPreference,
   FilterableNotificationPreference,
   NotificationInclude,
   NotificationPreference,
@@ -32,17 +30,18 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
     let builder = db.db
       .selectFrom('notification as notif')
       .where('notif.did', '=', actorDid)
-      .where((clause) =>
-        clause
-          .where('reasonSubject', 'is', null)
-          .orWhereExists(
+      .where((eb) =>
+        eb.or([
+          eb('reasonSubject', 'is', null),
+          eb.exists(
             db.db
               .selectFrom('record as subject')
               .selectAll()
               .whereRef('subject.uri', '=', ref('notif.reasonSubject')),
           ),
+        ]),
       )
-      .if(priority, (qb) => qb.whereExists(priorityFollowQb))
+      .$if(priority, (qb) => qb.where(({ exists }) => exists(priorityFollowQb)))
       .select([
         'notif.author as authorDid',
         'notif.recordUri as uri',
@@ -117,13 +116,15 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
       // Ensure to hit notification_did_sortat_idx, handling case where lastSeenNotifs is null.
       .where('notification.did', '=', actorDid)
       .where('notification.sortAt', '>', lastSeen ?? '')
-      .if(priority, (qb) =>
-        qb.whereExists(
-          db.db
-            .selectFrom('follow')
-            .select(sql<boolean>`${true}`.as('val'))
-            .where('creator', '=', actorDid)
-            .whereRef('subjectDid', '=', ref('notification.author')),
+      .$if(priority, (qb) =>
+        qb.where(({ exists }) =>
+          exists(
+            db.db
+              .selectFrom('follow')
+              .select(sql<boolean>`${true}`.as('val'))
+              .where('creator', '=', actorDid)
+              .whereRef('subjectDid', '=', ref('notification.author')),
+          ),
         ),
       )
       .executeTakeFirst()
@@ -200,17 +201,6 @@ export const notificationPreferencesLexToProtobuf = (
   p: app.bsky.notification.defs.Preferences,
   json: string,
 ): NotificationPreferences => {
-  const lexChatPreferenceToProtobuf = (
-    p: app.bsky.notification.defs.ChatPreference,
-  ): ChatNotificationPreference =>
-    new ChatNotificationPreference({
-      include:
-        p.include === 'accepted'
-          ? ChatNotificationInclude.ACCEPTED
-          : ChatNotificationInclude.ALL,
-      push: { enabled: p.push ?? true },
-    })
-
   const lexFilterablePreferenceToProtobuf = (
     p: app.bsky.notification.defs.FilterablePreference,
   ): FilterableNotificationPreference =>
@@ -233,7 +223,6 @@ export const notificationPreferencesLexToProtobuf = (
 
   return new NotificationPreferences({
     entry: Buffer.from(json),
-    chat: lexChatPreferenceToProtobuf(p.chat),
     follow: lexFilterablePreferenceToProtobuf(p.follow),
     like: lexFilterablePreferenceToProtobuf(p.like),
     likeViaRepost: lexFilterablePreferenceToProtobuf(p.likeViaRepost),

package/src/data-plane/server/routes/relationships.ts

@@ -95,9 +95,11 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
     const { ref } = db.db.dynamic
     const sourceRef = ref('pair.source')
     const targetRef = ref('pair.target')
-    const values = valuesList(pairs.map((p) => sql`${p.a}, ${p.b}`))
+    const values = valuesList<{ source: string; target: string }>(
+      pairs.map((p) => sql`${p.a}, ${p.b}`),
+    )
     const res = await db.db
-      .selectFrom(values.as(sql`pair (source, target)`))
+      .selectFrom(values.as<'pair'>(sql`pair (source, target)`))
       .select([
         sql<string>`${sourceRef}`.as('source'),
         sql<string>`${targetRef}`.as('target'),

package/src/data-plane/server/routes/search.ts

@@ -139,8 +139,8 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => {
 
     async searchActorsTypeahead(req) {
       const { dids } = await searchActorsImpl({
-        term: req.query,
-        limit: req.limit || 10,
+        term: req.params?.query ?? '',
+        limit: req.params?.limit || 10,
       })
       return {
         actors: dids.map((did) => ({ did, score: 0 })),

package/src/data-plane/server/routes/suggestions.ts

@@ -41,7 +41,7 @@ const getFollowSuggestionsGlobal = async (
   const suggestions = await db.db
     .selectFrom('suggested_follow')
     .innerJoin('actor', 'actor.did', 'suggested_follow.did')
-    .if(alreadyIncluded.length > 0, (qb) =>
+    .$if(alreadyIncluded.length > 0, (qb) =>
       qb.where('suggested_follow.order', 'not in', alreadyIncluded),
     )
     .selectAll()
@@ -116,7 +116,7 @@ const getFollowSuggestionsRelativeTo = async (
       .where('follow.creator', '=', input.actorDid)
       .where('follow.subjectDid', '!=', input.relativeToDid)
       .where('follow.subjectDid', 'not in', actorsViewerFollows)
-      .if(resultDids.length > 0, (qb) =>
+      .$if(resultDids.length > 0, (qb) =>
         qb.where(
           'subjectDid',
           'not in',

package/src/data-plane/server/subscription.ts

@@ -7,52 +7,95 @@ import { Database } from './db/index.js'
 import { IndexingService } from './indexing/index.js'
 
 export class RepoSubscription {
-  firehose: Firehose
-  runner: MemoryRunner
-  background: BackgroundQueue
-  indexingSvc: IndexingService
+  private readonly abortController = new AbortController()
+  private current?: ReturnType<typeof createFirehose>
+  readonly background: BackgroundQueue<Database>
+  readonly indexingSvc: IndexingService
+  private readonly service: string
+  private readonly idResolver: IdResolver
 
-  constructor(
-    public opts: { service: string; db: Database; idResolver: IdResolver },
-  ) {
-    const { service, db, idResolver } = opts
-    this.background = new BackgroundQueue(db)
+  constructor({
+    service,
+    db,
+    idResolver,
+  }: {
+    service: string
+    db: Database
+    idResolver: IdResolver
+  }) {
+    this.service = service
+    this.idResolver = idResolver
+    this.background = new BackgroundQueue(db, {
+      // @TODO This has historically been using the default concurrency
+      // (Infinity) but we may want to limit this in the future to avoid
+      // overloading the database with indexing tasks.
+      concurrency: Number.POSITIVE_INFINITY,
+    })
     this.indexingSvc = new IndexingService(db, idResolver, this.background)
+  }
 
-    const { runner, firehose } = createFirehose({
-      idResolver,
-      service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
+  get destroyed() {
+    return this.abortController.signal.aborted
   }
 
-  start() {
-    this.firehose.start()
+  get running() {
+    return this.current != null
+  }
+
+  getCursor() {
+    return this.current?.runner.getCursor()
+  }
+
+  async start() {
+    this.abortController.signal.throwIfAborted()
+
+    if (!this.current) {
+      this.current = createFirehose({
+        idResolver: this.idResolver,
+        service: this.service,
+        indexingSvc: this.indexingSvc,
+      })
+      void this.current.firehose.start()
+    }
+  }
+
+  async stop() {
+    try {
+      if (this.current) {
+        const { firehose, runner } = this.current
+        this.current = undefined
+        try {
+          await firehose.destroy()
+        } finally {
+          await runner.destroy()
+        }
+      }
+    } finally {
+      await this.background.processAll()
+    }
   }
 
   async restart() {
-    await this.destroy()
-    const { runner, firehose } = createFirehose({
-      idResolver: this.opts.idResolver,
-      service: this.opts.service,
-      indexingSvc: this.indexingSvc,
-    })
-    this.runner = runner
-    this.firehose = firehose
-    this.start()
+    await this.stop()
+    void this.start()
   }
 
   async processAll() {
-    await this.runner.processAll()
+    await this.current?.runner.processAll()
     await this.background.processAll()
   }
 
   async destroy() {
-    await this.firehose.destroy()
-    await this.runner.destroy()
-    await this.background.processAll()
+    this.abortController.abort()
+    try {
+      await this.stop()
+    } finally {
+      await this.background.destroy()
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/feature-gates/index.ts

@@ -72,10 +72,10 @@ export class FeatureGatesClient {
       this.client = new GrowthBookClient({
         apiHost: this.config.growthBookApiHost,
         clientKey: this.config.growthBookClientKey,
-        onFeatureUsage: (feature, result, userContext) => {
+        onFeatureUsage: async (feature, result, userContext) => {
           if (IGNORE_METRICS_FOR_GATES.has(feature as Gate)) return
 
-          this.metrics.track(
+          await this.metrics.track(
             'feature:viewed',
             {
               featureId: feature,
@@ -88,7 +88,7 @@ export class FeatureGatesClient {
             ),
           )
         },
-        trackingCallback: (experiment, result, userContext) => {
+        trackingCallback: async (experiment, result, userContext) => {
           /**
            * Experiments are only fired in a feature gate has an Experiment
            * attached in Growthbook. Howerver, we want to be extra sure that a
@@ -101,7 +101,7 @@ export class FeatureGatesClient {
           )
             return
 
-          this.metrics.track(
+          await this.metrics.track(
             'experiment:viewed',
             {
               experimentId: experiment.key,
@@ -154,14 +154,18 @@ export class FeatureGatesClient {
     }
   }
 
-  destroy() {
+  async destroy() {
     if (this.ready) {
       this.ready = false
       if (this.refreshInterval) {
         clearInterval(this.refreshInterval)
       }
     }
-    this.metrics.stop()
+    await this.metrics.stop()
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 
   /**

package/src/feature-gates/metrics.test.ts

@@ -32,8 +32,8 @@ describe('MetricsClient', () => {
     global.fetch = fetchMock
   })
 
-  afterEach(() => {
-    client?.stop()
+  afterEach(async () => {
+    await client?.stop()
     vi.useRealTimers()
     vi.clearAllMocks()
   })
@@ -42,8 +42,8 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
-    client.track('view', { screen: 'home' })
+    await client.track('click', { button: 'submit' })
+    await client.track('view', { screen: 'home' })
 
     expect(fetchRequests).toHaveLength(0)
 
@@ -65,13 +65,13 @@ describe('MetricsClient', () => {
 
     // Add events up to maxBatchSize (should not flush yet)
     for (let i = 0; i < 5; i++) {
-      client.track('click', { button: `btn-${i}` })
+      await client.track('click', { button: `btn-${i}` })
     }
 
     expect(fetchRequests).toHaveLength(0)
 
     // One more event should trigger flush (> maxBatchSize)
-    client.track('click', { button: 'btn-trigger' })
+    await client.track('click', { button: 'btn-trigger' })
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -90,7 +90,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -119,7 +119,7 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush - should not throw
     vi.advanceTimersByTime(10_000)
@@ -140,12 +140,12 @@ describe('MetricsClient', () => {
     client = new MetricsClient<TestEvents>({
       trackingEndpoint: 'https://test.metrics.api',
     })
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     expect(fetchRequests).toHaveLength(0)
 
     // Stop should flush remaining events
-    client.stop()
+    await client.stop()
     await flushPromises()
 
     expect(fetchRequests).toHaveLength(1)
@@ -155,7 +155,7 @@ describe('MetricsClient', () => {
 
   it('does not send if trackingEndpoint is not configured', async () => {
     client = new MetricsClient<TestEvents>({})
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
 
     // Trigger flush via interval
     vi.advanceTimersByTime(10_000)
@@ -170,7 +170,7 @@ describe('MetricsClient', () => {
     })
 
     // track() calls start() internally
-    client.track('click', { button: 'submit' })
+    await client.track('click', { button: 'submit' })
     client.start()
     client.start()
 

package/src/feature-gates/metrics.ts

@@ -38,24 +38,24 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     if (this.started) return
     this.started = true
     this.flushInterval = setInterval(() => {
-      this.flush()
+      void this.flush()
     }, 10_000)
   }
 
-  stop() {
+  async stop() {
     if (this.flushInterval) {
       clearInterval(this.flushInterval)
       this.flushInterval = null
     }
-    this.flush()
+    await this.flush()
   }
 
-  track<E extends keyof M>(
+  async track<E extends keyof M>(
     event: E,
     payload: M[E],
     metadata: Record<string, any> = {},
   ) {
-    this.start()
+    await this.start()
 
     const e = {
       source: 'appview',
@@ -67,14 +67,14 @@ export class MetricsClient<M extends Record<string, any> = Events> {
     this.queue.push(e)
 
     if (this.queue.length > this.maxBatchSize) {
-      this.flush()
+      await this.flush()
     }
   }
 
-  flush() {
+  async flush() {
     if (!this.queue.length) return
     const events = this.queue.splice(0, this.queue.length)
-    this.sendBatch(events)
+    await this.sendBatch(events)
   }
 
   private async sendBatch(events: Event<M>[]) {

package/src/index.ts

@@ -5,11 +5,8 @@ import compression from 'compression'
 import cors from 'cors'
 import { Etcd3 } from 'etcd3'
 import express from 'express'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
+// eslint-disable-next-line import/default
 import httpTerminator from 'http-terminator'
-// eslint-disable-next-line import/no-named-as-default-member
-const { createHttpTerminator } = httpTerminator
-type HttpTerminator = ReturnType<typeof createHttpTerminator>
 import { DAY, SECOND } from '@atproto/common'
 import { Keypair } from '@atproto/crypto'
 import { IdResolver } from '@atproto/identity'
@@ -63,7 +60,7 @@ export class BskyAppView {
   public ctx: AppContext
   public app: express.Application
   public server?: http.Server
-  private terminator?: HttpTerminator
+  private terminator?: httpTerminator.HttpTerminator
 
   constructor(opts: { ctx: AppContext; app: express.Application }) {
     this.ctx = opts.ctx
@@ -288,7 +285,7 @@ export class BskyAppView {
     const server = this.app.listen(this.ctx.cfg.port)
     this.server = server
     server.keepAliveTimeout = 90000
-    this.terminator = createHttpTerminator({ server })
+    this.terminator = httpTerminator.createHttpTerminator({ server })
     await events.once(server, 'listening')
     const { port } = server.address() as AddressInfo
     this.ctx.cfg.assignPort(port)
@@ -296,9 +293,19 @@ export class BskyAppView {
   }
 
   async destroy(): Promise<void> {
-    this.ctx.featureGatesClient.destroy()
-    await this.terminator?.terminate()
-    await this.ctx.etcd?.close()
+    try {
+      await this.ctx.featureGatesClient.destroy()
+    } finally {
+      try {
+        await this.terminator?.terminate()
+      } finally {
+        await this.ctx.etcd?.close()
+      }
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/tests/_util.ts

@@ -1,6 +1,7 @@
-import { Server } from 'node:http'
+import { RequestListener, createServer } from 'node:http'
 import { AddressInfo } from 'node:net'
-import { type Express } from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { expect } from 'vitest'
 import {
   $Typed,
@@ -265,19 +266,11 @@ export const stripViewerFromLabeler = (
   return stripViewer(labeler)
 }
 
-export async function startServer(app: Express) {
-  return new Promise<{
-    origin: string
-    server: Server
-    stop: () => Promise<void>
-  }>((resolve, reject) => {
+export async function startServer(listener: RequestListener) {
+  return new Promise<AsyncDisposable & { port: number }>((resolve, reject) => {
     const onListen = () => {
       const port = (server.address() as AddressInfo).port
-      resolve({
-        server,
-        origin: `http://localhost:${port}`,
-        stop: () => stopServer(server),
-      })
+      resolve({ port, [Symbol.asyncDispose]: () => terminator.terminate() })
       cleanup()
     }
     const onError = (err: Error) => {
@@ -289,21 +282,11 @@ export async function startServer(app: Express) {
       server.removeListener('error', onError)
     }
 
-    const server = app
+    const server = createServer(listener)
       .listen(0)
       .once('listening', onListen)
       .once('error', onError)
-  })
-}
 
-export async function stopServer(server: Server) {
-  return new Promise<void>((resolve, reject) => {
-    server.close((err) => {
-      if (err) {
-        reject(err)
-      } else {
-        resolve()
-      }
-    })
+    const terminator = httpTerminator.createHttpTerminator({ server })
   })
 }

package/tests/admin/admin-auth.test.ts

@@ -62,10 +62,10 @@ describe('admin auth', () => {
       $type: 'com.atproto.admin.defs#repoRef',
       did: sc.dids.bob,
     }
-  }, 20_000) // @NOTE seeding can take a while
+  })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('allows service auth requests from the configured appview did', async () => {

package/tests/admin/moderation.test.ts

@@ -46,7 +46,7 @@ describe('moderation', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('takes down accounts', async () => {

package/tests/auth.test.ts

@@ -1,4 +1,4 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
 import { AtpAgent, ids } from '@atproto/api'
 import { Keypair, Secp256k1Keypair } from '@atproto/crypto'
 import { SeedClient, TestNetwork, usersSeed } from '@atproto/dev-env'
@@ -20,15 +20,13 @@ describe('auth', () => {
     agent = network.bsky.getAgent()
     sc = network.getSeedClient()
     await usersSeed(sc)
-    await network.processAll()
     alice = sc.dids.alice
     bob = sc.dids.bob
-  }, 20_000) // @NOTE seeding can take a while
-
-  afterAll(async () => {
-    await network.close()
   })
 
+  beforeEach(async () => network.processAll())
+  afterAll(async () => network?.close())
+
   // @TODO invalidations do not originate from appview frontends: requires identity event on the repo stream.
   it.skip('handles signing key change for service auth.', async () => {
     const issuer = sc.dids.alice

package/tests/blob-resolver.test.ts

@@ -1,5 +1,5 @@
 import { request } from 'undici'
-import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
 import { cidForCbor, verifyCidForBytes } from '@atproto/common'
 import { randomBytes } from '@atproto/crypto'
 import { TestNetwork, basicSeed } from '@atproto/dev-env'
@@ -17,15 +17,14 @@ describe('blob resolver', () => {
     })
     const sc = network.getSeedClient()
     await basicSeed(sc)
-    await network.processAll()
+
     fileDid = sc.dids.carol
     fileCid = sc.posts[fileDid][0].images[0].image.ref
     fileSize = sc.posts[fileDid][0].images[0].image.size
   })
 
-  afterAll(async () => {
-    await network.close()
-  })
+  beforeEach(async () => network.processAll())
+  afterAll(async () => network?.close())
 
   it('resolves blob with good signature check.', async () => {
     const response = await request(

package/tests/data-plane/db.test.ts

@@ -16,7 +16,7 @@ describe('db', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('handles client errors without crashing.', async () => {