@atproto/bsky 0.0.124 → 0.0.126

package/src/config.ts

@@ -10,7 +10,9 @@ export interface ServerConfigValues {
   alternateAudienceDids: string[]
   entrywayJwtPublicKeyHex?: string
   // external services
+  etcdHosts: string[]
   dataplaneUrls: string[]
+  dataplaneUrlsEtcdKeyPrefix?: string
   dataplaneHttpVersion?: '1.1' | '2'
   dataplaneIgnoreBadTls?: boolean
   bsyncUrl: string
@@ -47,6 +49,8 @@ export interface ServerConfigValues {
   bigThreadUris: Set<string>
   bigThreadDepth?: number
   maxThreadDepth?: number
+  // notifications
+  notificationsDelayMs?: number
   // client config
   clientCheckEmailConfirmed?: boolean
   topicsEnabled?: boolean
@@ -74,15 +78,15 @@ export class ServerConfig {
     const envPort = parseInt(process.env.BSKY_PORT || '', 10)
     const port = isNaN(envPort) ? 2584 : envPort
     const didPlcUrl = process.env.BSKY_DID_PLC_URL || 'http://localhost:2582'
-    const alternateAudienceDids = process.env.BSKY_ALT_AUDIENCE_DIDS
-      ? process.env.BSKY_ALT_AUDIENCE_DIDS.split(',')
-      : []
+    const alternateAudienceDids = envList(process.env.BSKY_ALT_AUDIENCE_DIDS)
     const entrywayJwtPublicKeyHex =
       process.env.BSKY_ENTRYWAY_JWT_PUBLIC_KEY_HEX || undefined
-    const handleResolveNameservers = process.env.BSKY_HANDLE_RESOLVE_NAMESERVERS
-      ? process.env.BSKY_HANDLE_RESOLVE_NAMESERVERS.split(',')
-      : []
+    const handleResolveNameservers = envList(
+      process.env.BSKY_HANDLE_RESOLVE_NAMESERVERS,
+    )
     const cdnUrl = process.env.BSKY_CDN_URL || process.env.BSKY_IMG_URI_ENDPOINT
+    const etcdHosts =
+      overrides?.etcdHosts ?? envList(process.env.BSKY_ETCD_HOSTS)
     // e.g. https://video.invalid/watch/%s/%s/playlist.m3u8
     const videoPlaylistUrlPattern = process.env.BSKY_VIDEO_PLAYLIST_URL_PATTERN
     // e.g. https://video.invalid/watch/%s/%s/thumbnail.jpg
@@ -97,16 +101,25 @@ export class ServerConfig {
     const suggestionsApiKey = process.env.BSKY_SUGGESTIONS_API_KEY || undefined
     const topicsUrl = process.env.BSKY_TOPICS_URL || undefined
     const topicsApiKey = process.env.BSKY_TOPICS_API_KEY
-    let dataplaneUrls = overrides?.dataplaneUrls
-    dataplaneUrls ??= process.env.BSKY_DATAPLANE_URLS
-      ? process.env.BSKY_DATAPLANE_URLS.split(',')
-      : []
+    const dataplaneUrls =
+      overrides?.dataplaneUrls ?? envList(process.env.BSKY_DATAPLANE_URLS)
+    const dataplaneUrlsEtcdKeyPrefix =
+      process.env.BSKY_DATAPLANE_URLS_ETCD_KEY_PREFIX || undefined
     const dataplaneHttpVersion = process.env.BSKY_DATAPLANE_HTTP_VERSION || '2'
     const dataplaneIgnoreBadTls =
       process.env.BSKY_DATAPLANE_IGNORE_BAD_TLS === 'true'
-    const labelsFromIssuerDids = process.env.BSKY_LABELS_FROM_ISSUER_DIDS
-      ? process.env.BSKY_LABELS_FROM_ISSUER_DIDS.split(',')
-      : []
+    assert(
+      !dataplaneUrlsEtcdKeyPrefix || etcdHosts.length,
+      'etcd prefix for dataplane urls may only be configured when there are etcd hosts',
+    )
+    assert(
+      dataplaneUrls.length || dataplaneUrlsEtcdKeyPrefix,
+      'dataplane urls are not configured directly nor with etcd',
+    )
+    assert(dataplaneHttpVersion === '1.1' || dataplaneHttpVersion === '2')
+    const labelsFromIssuerDids = envList(
+      process.env.BSKY_LABELS_FROM_ISSUER_DIDS,
+    )
     const bsyncUrl = process.env.BSKY_BSYNC_URL || undefined
     assert(bsyncUrl)
     const bsyncApiKey = process.env.BSKY_BSYNC_API_KEY || undefined
@@ -133,8 +146,6 @@ export class ServerConfig {
     )
     const modServiceDid = process.env.MOD_SERVICE_DID
     assert(modServiceDid)
-    assert(dataplaneUrls.length)
-    assert(dataplaneHttpVersion === '1.1' || dataplaneHttpVersion === '2')
     const statsigKey =
       process.env.NODE_ENV === 'test'
         ? 'secret-key'
@@ -161,6 +172,10 @@ export class ServerConfig {
       ? parseInt(process.env.BSKY_MAX_THREAD_DEPTH || '', 10)
       : undefined
 
+    const notificationsDelayMs = process.env.BSKY_NOTIFICATIONS_DELAY_MS
+      ? parseInt(process.env.BSKY_NOTIFICATIONS_DELAY_MS || '', 10)
+      : 0
+
     const disableSsrfProtection = process.env.BSKY_DISABLE_SSRF_PROTECTION
       ? process.env.BSKY_DISABLE_SSRF_PROTECTION === 'true'
       : debugMode
@@ -185,7 +200,9 @@ export class ServerConfig {
       serverDid,
       alternateAudienceDids,
       entrywayJwtPublicKeyHex,
+      etcdHosts,
       dataplaneUrls,
+      dataplaneUrlsEtcdKeyPrefix,
       dataplaneHttpVersion,
       dataplaneIgnoreBadTls,
       searchUrl,
@@ -220,6 +237,7 @@ export class ServerConfig {
       bigThreadUris,
       bigThreadDepth,
       maxThreadDepth,
+      notificationsDelayMs,
       disableSsrfProtection,
       proxyAllowHTTP2,
       proxyHeadersTimeout,
@@ -267,6 +285,14 @@ export class ServerConfig {
     return this.cfg.entrywayJwtPublicKeyHex
   }
 
+  get etcdHosts() {
+    return this.cfg.etcdHosts
+  }
+
+  get dataplaneUrlsEtcdKeyPrefix() {
+    return this.cfg.dataplaneUrlsEtcdKeyPrefix
+  }
+
   get dataplaneUrls() {
     return this.cfg.dataplaneUrls
   }
@@ -407,6 +433,10 @@ export class ServerConfig {
     return this.cfg.maxThreadDepth
   }
 
+  get notificationsDelayMs() {
+    return this.cfg.notificationsDelayMs ?? 0
+  }
+
   get disableSsrfProtection(): boolean {
     return this.cfg.disableSsrfProtection ?? false
   }

package/src/context.ts

@@ -1,4 +1,5 @@
 import * as plc from '@did-plc/lib'
+import { Etcd3 } from 'etcd3'
 import express from 'express'
 import { Dispatcher } from 'undici'
 import { AtpAgent } from '@atproto/api'
@@ -8,7 +9,7 @@ import { AuthVerifier } from './auth-verifier'
 import { BsyncClient } from './bsync'
 import { ServerConfig } from './config'
 import { CourierClient } from './courier'
-import { DataPlaneClient } from './data-plane/client'
+import { DataPlaneClient, HostList } from './data-plane/client'
 import { FeatureGates } from './feature-gates'
 import { Hydrator } from './hydration/hydrator'
 import { httpLogger as log } from './logger'
@@ -23,7 +24,9 @@ export class AppContext {
   constructor(
     private opts: {
       cfg: ServerConfig
+      etcd: Etcd3 | undefined
       dataplane: DataPlaneClient
+      dataplaneHostList: HostList
       searchAgent: AtpAgent | undefined
       suggestionsAgent: AtpAgent | undefined
       topicsAgent: AtpAgent | undefined
@@ -43,10 +46,18 @@ export class AppContext {
     return this.opts.cfg
   }
 
+  get etcd() {
+    return this.opts.etcd
+  }
+
   get dataplane(): DataPlaneClient {
     return this.opts.dataplane
   }
 
+  get dataplaneHostList(): HostList {
+    return this.opts.dataplaneHostList
+  }
+
   get searchAgent(): AtpAgent | undefined {
     return this.opts.searchAgent
   }

package/src/data-plane/client/hosts.ts

@@ -0,0 +1,103 @@
+import { Etcd3 } from 'etcd3'
+import { EtcdMap } from '../../etcd'
+import { dataplaneLogger as logger } from '../../logger'
+
+/**
+ * Interface for a reactive list of hosts, i.e. for use with the dataplane client.
+ */
+export interface HostList {
+  get: () => Iterable<string>
+  onUpdate(handler: HostListHandler): void
+}
+
+type HostListHandler = (hosts: Iterable<string>) => void
+
+/**
+ * Maintains a reactive HostList based on a simple setter.
+ */
+export class BasicHostList implements HostList {
+  private hosts: Iterable<string>
+  private handlers: HostListHandler[] = []
+
+  constructor(hosts: Iterable<string>) {
+    this.hosts = hosts
+  }
+
+  get() {
+    return this.hosts
+  }
+
+  set(hosts: Iterable<string>) {
+    this.hosts = hosts
+    this.update()
+  }
+
+  private update() {
+    for (const handler of this.handlers) {
+      handler(this.hosts)
+    }
+  }
+
+  onUpdate(handler: HostListHandler) {
+    this.handlers.push(handler)
+  }
+}
+
+/**
+ * Maintains a reactive HostList based on etcd key values under a given key prefix.
+ * When fallback is provided, ensures that this fallback is used whenever no hosts are available.
+ */
+export class EtcdHostList implements HostList {
+  private kv: EtcdMap
+  private inner = new BasicHostList(new Set())
+  private fallback: Set<string>
+
+  constructor(etcd: Etcd3, prefix: string, fallback?: string[]) {
+    this.fallback = new Set(fallback)
+    this.kv = new EtcdMap(etcd, prefix)
+    this.update() // init fallback if necessary
+    this.kv.watcher.on('connected', (res) => {
+      logger.warn(
+        { watcherId: this.kv.watcher.id, header: res.header },
+        'EtcdHostList connected',
+      )
+    })
+    this.kv.watcher.on('disconnected', (err) => {
+      logger.warn(
+        { watcherId: this.kv.watcher.id, err },
+        'EtcdHostList disconnected',
+      )
+    })
+    this.kv.watcher.on('error', (err) => {
+      logger.error({ watcherId: this.kv.watcher.id, err }, 'EtcdHostList error')
+    })
+  }
+
+  async connect() {
+    await this.kv.connect()
+    this.update()
+    this.kv.onUpdate(() => this.update())
+  }
+
+  get() {
+    return this.inner.get()
+  }
+
+  private update() {
+    const hosts = new Set<string>()
+    for (const host of this.kv.values()) {
+      if (URL.canParse(host)) {
+        hosts.add(host)
+      }
+    }
+    if (hosts.size) {
+      this.inner.set(hosts)
+    } else if (this.fallback.size) {
+      this.inner.set(this.fallback)
+    }
+  }
+
+  onUpdate(handler: HostListHandler) {
+    this.inner.onUpdate(handler)
+  }
+}

package/src/data-plane/client/index.ts

@@ -0,0 +1,123 @@
+import assert from 'node:assert'
+import { randomInt } from 'node:crypto'
+import {
+  Code,
+  ConnectError,
+  PromiseClient,
+  createPromiseClient,
+  makeAnyClient,
+} from '@connectrpc/connect'
+import { createGrpcTransport } from '@connectrpc/connect-node'
+import { Service } from '../../proto/bsky_connect'
+import { HostList } from './hosts'
+
+export * from './hosts'
+export * from './util'
+
+export type DataPlaneClient = PromiseClient<typeof Service>
+type HttpVersion = '1.1' | '2'
+const MAX_RETRIES = 3
+
+export const createDataPlaneClient = (
+  hostList: HostList,
+  opts: { httpVersion?: HttpVersion; rejectUnauthorized?: boolean },
+) => {
+  const clients = new DataPlaneClients(hostList, opts)
+  return makeAnyClient(Service, (method) => {
+    return async (...args) => {
+      let tries = 0
+      let error: unknown
+      let remainingClients = clients.get()
+      while (tries < MAX_RETRIES) {
+        const client = randomElement(remainingClients)
+        assert(client, 'no clients available')
+        try {
+          return await client[method.localName](...args)
+        } catch (err) {
+          if (
+            err instanceof ConnectError &&
+            (err.code === Code.Unavailable || err.code === Code.Aborted)
+          ) {
+            tries++
+            error = err
+            remainingClients = getRemainingClients(remainingClients, client)
+          } else {
+            throw err
+          }
+        }
+      }
+      assert(error)
+      throw error
+    }
+  }) as DataPlaneClient
+}
+
+export { Code }
+
+/**
+ * Uses a reactive HostList in order to maintain a pool of DataPlaneClients.
+ * Each DataPlaneClient is cached per host so that it maintains connections
+ * and other internal state when the underlying HostList is updated.
+ */
+class DataPlaneClients {
+  private clients: DataPlaneClient[] = []
+  private clientsByHost = new Map<string, DataPlaneClient>()
+
+  constructor(
+    private hostList: HostList,
+    private clientOpts: {
+      httpVersion?: HttpVersion
+      rejectUnauthorized?: boolean
+    },
+  ) {
+    this.refresh()
+    this.hostList.onUpdate(() => this.refresh())
+  }
+
+  get(): readonly DataPlaneClient[] {
+    return this.clients
+  }
+
+  private refresh() {
+    this.clients = []
+    for (const host of this.hostList.get()) {
+      let client = this.clientsByHost.get(host)
+      if (!client) {
+        client = this.createClient(host)
+        this.clientsByHost.set(host, client)
+      }
+      this.clients.push(client)
+    }
+  }
+
+  private createClient(host: string) {
+    return createBaseClient(host, this.clientOpts)
+  }
+}
+
+const createBaseClient = (
+  baseUrl: string,
+  opts: { httpVersion?: HttpVersion; rejectUnauthorized?: boolean },
+): DataPlaneClient => {
+  const { httpVersion = '2', rejectUnauthorized = true } = opts
+  const transport = createGrpcTransport({
+    baseUrl,
+    httpVersion,
+    acceptCompression: [],
+    nodeOptions: { rejectUnauthorized },
+  })
+  return createPromiseClient(Service, transport)
+}
+
+const getRemainingClients = (
+  clients: readonly DataPlaneClient[],
+  lastClient: DataPlaneClient,
+) => {
+  if (clients.length < 2) return clients // no clients to choose from
+  return clients.filter((c) => c !== lastClient)
+}
+
+const randomElement = <T>(arr: readonly T[]): T | undefined => {
+  if (arr.length === 0) return
+  return arr[randomInt(arr.length)]
+}

package/src/data-plane/client/util.ts

@@ -0,0 +1,66 @@
+import { Code, ConnectError } from '@connectrpc/connect'
+import * as ui8 from 'uint8arrays'
+import { getDidKeyFromMultibase } from '@atproto/identity'
+
+export const isDataplaneError = (
+  err: unknown,
+  code?: Code,
+): err is ConnectError => {
+  if (err instanceof ConnectError) {
+    return !code || err.code === code
+  }
+  return false
+}
+
+export const unpackIdentityServices = (servicesBytes: Uint8Array) => {
+  const servicesStr = ui8.toString(servicesBytes, 'utf8')
+  if (!servicesStr) return {}
+  return JSON.parse(servicesStr) as UnpackedServices
+}
+
+export const unpackIdentityKeys = (keysBytes: Uint8Array) => {
+  const keysStr = ui8.toString(keysBytes, 'utf8')
+  if (!keysStr) return {}
+  return JSON.parse(keysStr) as UnpackedKeys
+}
+
+export const getServiceEndpoint = (
+  services: UnpackedServices,
+  opts: { id: string; type: string },
+) => {
+  const endpoint =
+    services[opts.id] &&
+    services[opts.id].Type === opts.type &&
+    validateUrl(services[opts.id].URL)
+  return endpoint || undefined
+}
+
+export const getKeyAsDidKey = (keys: UnpackedKeys, opts: { id: string }) => {
+  const key =
+    keys[opts.id] &&
+    getDidKeyFromMultibase({
+      type: keys[opts.id].Type,
+      publicKeyMultibase: keys[opts.id].PublicKeyMultibase,
+    })
+  return key || undefined
+}
+
+type UnpackedServices = Record<string, { Type: string; URL: string }>
+
+type UnpackedKeys = Record<string, { Type: string; PublicKeyMultibase: string }>
+
+const validateUrl = (urlStr: string): string | undefined => {
+  let url
+  try {
+    url = new URL(urlStr)
+  } catch {
+    return undefined
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) {
+    return undefined
+  } else if (!url.hostname) {
+    return undefined
+  } else {
+    return urlStr
+  }
+}