@atproto/bsky 0.0.251 → 0.0.254

package/src/context.ts

@@ -1,143 +0,0 @@
-import * as plc from '@did-plc/lib'
-import { Etcd3 } from 'etcd3'
-import express from 'express'
-import { Dispatcher } from 'undici'
-import { Keypair } from '@atproto/crypto'
-import { IdResolver } from '@atproto/identity'
-import { Client } from '@atproto/lex'
-import { AuthVerifier } from './auth-verifier.js'
-import { BsyncClient } from './bsync.js'
-import { ServerConfig } from './config.js'
-import { CourierClient } from './courier.js'
-import { DataPlaneClient, HostList } from './data-plane/client/index.js'
-import { FeatureGatesClient } from './feature-gates/index.js'
-import { Hydrator } from './hydration/hydrator.js'
-import { KwsClient } from './kws.js'
-import { httpLogger as log } from './logger.js'
-import { RolodexClient } from './rolodex.js'
-import { StashClient } from './stash.js'
-import {
-  ParsedLabelers,
-  defaultLabelerHeader,
-  parseLabelerHeader,
-} from './util.js'
-import { Views } from './views/index.js'
-
-export class AppContext {
-  constructor(
-    private opts: {
-      cfg: ServerConfig
-      etcd: Etcd3 | undefined
-      dataplane: DataPlaneClient
-      dataplaneHostList: HostList
-      searchClient: Client | undefined
-      suggestionsClient: Client | undefined
-      topicsClient: Client | undefined
-      hydrator: Hydrator
-      views: Views
-      signingKey: Keypair
-      idResolver: IdResolver
-      bsyncClient: BsyncClient
-      stashClient: StashClient
-      courierClient: CourierClient | undefined
-      rolodexClient: RolodexClient | undefined
-      authVerifier: AuthVerifier
-      featureGatesClient: FeatureGatesClient
-      blobDispatcher: Dispatcher
-      kwsClient: KwsClient | undefined
-    },
-  ) {}
-
-  get cfg(): ServerConfig {
-    return this.opts.cfg
-  }
-
-  get etcd() {
-    return this.opts.etcd
-  }
-
-  get dataplane(): DataPlaneClient {
-    return this.opts.dataplane
-  }
-
-  get dataplaneHostList(): HostList {
-    return this.opts.dataplaneHostList
-  }
-
-  get searchClient(): Client | undefined {
-    return this.opts.searchClient
-  }
-
-  get suggestionsClient(): Client | undefined {
-    return this.opts.suggestionsClient
-  }
-
-  get topicsClient(): Client | undefined {
-    return this.opts.topicsClient
-  }
-
-  get hydrator(): Hydrator {
-    return this.opts.hydrator
-  }
-
-  get views(): Views {
-    return this.opts.views
-  }
-
-  get signingKey(): Keypair {
-    return this.opts.signingKey
-  }
-
-  get plcClient(): plc.Client {
-    return new plc.Client(this.cfg.didPlcUrl)
-  }
-
-  get idResolver(): IdResolver {
-    return this.opts.idResolver
-  }
-
-  get bsyncClient(): BsyncClient {
-    return this.opts.bsyncClient
-  }
-
-  get stashClient(): StashClient {
-    return this.opts.stashClient
-  }
-
-  get courierClient(): CourierClient | undefined {
-    return this.opts.courierClient
-  }
-
-  get rolodexClient(): RolodexClient | undefined {
-    return this.opts.rolodexClient
-  }
-
-  get authVerifier(): AuthVerifier {
-    return this.opts.authVerifier
-  }
-
-  get featureGatesClient(): FeatureGatesClient {
-    return this.opts.featureGatesClient
-  }
-
-  get blobDispatcher(): Dispatcher {
-    return this.opts.blobDispatcher
-  }
-
-  get kwsClient(): KwsClient | undefined {
-    return this.opts.kwsClient
-  }
-
-  reqLabelers(req: express.Request): ParsedLabelers {
-    const val = req.header('atproto-accept-labelers')
-    let parsed: ParsedLabelers | null
-    try {
-      parsed = parseLabelerHeader(val)
-    } catch (err) {
-      parsed = null
-      log.info({ err, val }, 'failed to parse labeler header')
-    }
-    if (!parsed) return defaultLabelerHeader(this.cfg.labelsFromIssuerDids)
-    return parsed
-  }
-}

package/src/courier.ts

@@ -1,41 +0,0 @@
-import {
-  Code,
-  ConnectError,
-  Interceptor,
-  PromiseClient,
-  createPromiseClient,
-} from '@connectrpc/connect'
-import {
-  ConnectTransportOptions,
-  createConnectTransport,
-} from '@connectrpc/connect-node'
-import { Service } from './proto/courier_connect.js'
-
-export type CourierClient = PromiseClient<typeof Service>
-
-export const createCourierClient = (
-  opts: ConnectTransportOptions,
-): CourierClient => {
-  const transport = createConnectTransport(opts)
-  return createPromiseClient(Service, transport)
-}
-
-export { Code }
-
-export const isCourierError = (
-  err: unknown,
-  code?: Code,
-): err is ConnectError => {
-  if (err instanceof ConnectError) {
-    return !code || err.code === code
-  }
-  return false
-}
-
-export const authWithApiKey =
-  (apiKey: string): Interceptor =>
-  (next) =>
-  (req) => {
-    req.header.set('authorization', `Bearer ${apiKey}`)
-    return next(req)
-  }

package/src/data-plane/client/hosts.ts

@@ -1,103 +0,0 @@
-import { Etcd3 } from 'etcd3'
-import { EtcdMap } from '../../etcd.js'
-import { dataplaneLogger as logger } from '../../logger.js'
-
-/**
- * Interface for a reactive list of hosts, i.e. for use with the dataplane client.
- */
-export interface HostList {
-  get: () => Iterable<string>
-  onUpdate(handler: HostListHandler): void
-}
-
-type HostListHandler = (hosts: Iterable<string>) => void
-
-/**
- * Maintains a reactive HostList based on a simple setter.
- */
-export class BasicHostList implements HostList {
-  private hosts: Iterable<string>
-  private handlers: HostListHandler[] = []
-
-  constructor(hosts: Iterable<string>) {
-    this.hosts = hosts
-  }
-
-  get() {
-    return this.hosts
-  }
-
-  set(hosts: Iterable<string>) {
-    this.hosts = hosts
-    this.update()
-  }
-
-  private update() {
-    for (const handler of this.handlers) {
-      handler(this.hosts)
-    }
-  }
-
-  onUpdate(handler: HostListHandler) {
-    this.handlers.push(handler)
-  }
-}
-
-/**
- * Maintains a reactive HostList based on etcd key values under a given key prefix.
- * When fallback is provided, ensures that this fallback is used whenever no hosts are available.
- */
-export class EtcdHostList implements HostList {
-  private kv: EtcdMap
-  private inner = new BasicHostList(new Set())
-  private fallback: Set<string>
-
-  constructor(etcd: Etcd3, prefix: string, fallback?: string[]) {
-    this.fallback = new Set(fallback)
-    this.kv = new EtcdMap(etcd, prefix)
-    this.update() // init fallback if necessary
-    this.kv.watcher.on('connected', (res) => {
-      logger.warn(
-        { watcherId: this.kv.watcher.id, header: res.header },
-        'EtcdHostList connected',
-      )
-    })
-    this.kv.watcher.on('disconnected', (err) => {
-      logger.warn(
-        { watcherId: this.kv.watcher.id, err },
-        'EtcdHostList disconnected',
-      )
-    })
-    this.kv.watcher.on('error', (err) => {
-      logger.error({ watcherId: this.kv.watcher.id, err }, 'EtcdHostList error')
-    })
-  }
-
-  async connect() {
-    await this.kv.connect()
-    this.update()
-    this.kv.onUpdate(() => this.update())
-  }
-
-  get() {
-    return this.inner.get()
-  }
-
-  private update() {
-    const hosts = new Set<string>()
-    for (const host of this.kv.values()) {
-      if (URL.canParse(host)) {
-        hosts.add(host)
-      }
-    }
-    if (hosts.size) {
-      this.inner.set(hosts)
-    } else if (this.fallback.size) {
-      this.inner.set(this.fallback)
-    }
-  }
-
-  onUpdate(handler: HostListHandler) {
-    this.inner.onUpdate(handler)
-  }
-}

package/src/data-plane/client/index.ts

@@ -1,125 +0,0 @@
-import assert from 'node:assert'
-import { randomInt } from 'node:crypto'
-import {
-  Code,
-  ConnectError,
-  PromiseClient,
-  createPromiseClient,
-  makeAnyClient,
-} from '@connectrpc/connect'
-import { createGrpcTransport } from '@connectrpc/connect-node'
-import { Service } from '../../proto/bsky_connect.js'
-import { HostList } from './hosts.js'
-import { callerInterceptor } from './util.js'
-
-export * from './hosts.js'
-export * from './util.js'
-
-export type DataPlaneClient = PromiseClient<typeof Service>
-type HttpVersion = '1.1' | '2'
-const MAX_RETRIES = 3
-
-export const createDataPlaneClient = (
-  hostList: HostList,
-  opts: { httpVersion?: HttpVersion; rejectUnauthorized?: boolean },
-) => {
-  const clients = new DataPlaneClients(hostList, opts)
-  return makeAnyClient(Service, (method) => {
-    return async (...args) => {
-      let tries = 0
-      let error: unknown
-      let remainingClients = clients.get()
-      while (tries < MAX_RETRIES) {
-        const client = randomElement(remainingClients)
-        assert(client, 'no clients available')
-        try {
-          return await client[method.localName](...args)
-        } catch (err) {
-          if (
-            err instanceof ConnectError &&
-            (err.code === Code.Unavailable || err.code === Code.Aborted)
-          ) {
-            tries++
-            error = err
-            remainingClients = getRemainingClients(remainingClients, client)
-          } else {
-            throw err
-          }
-        }
-      }
-      assert(error)
-      throw error
-    }
-  }) as DataPlaneClient
-}
-
-export { Code }
-
-/**
- * Uses a reactive HostList in order to maintain a pool of DataPlaneClients.
- * Each DataPlaneClient is cached per host so that it maintains connections
- * and other internal state when the underlying HostList is updated.
- */
-class DataPlaneClients {
-  private clients: DataPlaneClient[] = []
-  private clientsByHost = new Map<string, DataPlaneClient>()
-
-  constructor(
-    private hostList: HostList,
-    private clientOpts: {
-      httpVersion?: HttpVersion
-      rejectUnauthorized?: boolean
-    },
-  ) {
-    this.refresh()
-    this.hostList.onUpdate(() => this.refresh())
-  }
-
-  get(): readonly DataPlaneClient[] {
-    return this.clients
-  }
-
-  private refresh() {
-    this.clients = []
-    for (const host of this.hostList.get()) {
-      let client = this.clientsByHost.get(host)
-      if (!client) {
-        client = this.createClient(host)
-        this.clientsByHost.set(host, client)
-      }
-      this.clients.push(client)
-    }
-  }
-
-  private createClient(host: string) {
-    return createBaseClient(host, this.clientOpts)
-  }
-}
-
-const createBaseClient = (
-  baseUrl: string,
-  opts: { httpVersion?: HttpVersion; rejectUnauthorized?: boolean },
-): DataPlaneClient => {
-  const { httpVersion = '2', rejectUnauthorized = true } = opts
-  const transport = createGrpcTransport({
-    baseUrl,
-    httpVersion,
-    acceptCompression: [],
-    nodeOptions: { rejectUnauthorized },
-    interceptors: [callerInterceptor('appview')],
-  })
-  return createPromiseClient(Service, transport)
-}
-
-const getRemainingClients = (
-  clients: readonly DataPlaneClient[],
-  lastClient: DataPlaneClient,
-) => {
-  if (clients.length < 2) return clients // no clients to choose from
-  return clients.filter((c) => c !== lastClient)
-}
-
-const randomElement = <T>(arr: readonly T[]): T | undefined => {
-  if (arr.length === 0) return
-  return arr[randomInt(arr.length)]
-}

package/src/data-plane/client/util.test.ts

@@ -1,29 +0,0 @@
-import {
-  StreamRequest,
-  StreamResponse,
-  UnaryRequest,
-  UnaryResponse,
-} from '@connectrpc/connect'
-import { describe, expect, it, vi } from 'vitest'
-import { callerInterceptor } from './util.js'
-
-describe('callerInterceptor', () => {
-  it('sets x-atlantis-caller header on the request', async () => {
-    const fakeRequest = { header: new Headers({ 'x-other': 'value' }) } as
-      | UnaryRequest
-      | StreamRequest
-    const fakeResponse = {} as UnaryResponse | StreamResponse
-    const fakeHandler = vi.fn(async (_req: UnaryRequest | StreamRequest) => {
-      return fakeResponse
-    })
-
-    const interceptor = callerInterceptor('appview')
-    const next = interceptor(fakeHandler)
-    const res = await next(fakeRequest)
-
-    expect(fakeRequest.header.get('x-atlantis-caller')).toBe('appview')
-    expect(fakeRequest.header.get('x-other')).toBe('value')
-    expect(fakeHandler).toHaveBeenCalledWith(fakeRequest)
-    expect(res).toBe(fakeResponse)
-  })
-})

package/src/data-plane/client/util.ts

@@ -1,74 +0,0 @@
-import { Code, ConnectError, Interceptor } from '@connectrpc/connect'
-import * as ui8 from 'uint8arrays'
-import { getDidKeyFromMultibase } from '@atproto/identity'
-
-export const callerInterceptor =
-  (caller: string): Interceptor =>
-  (next) =>
-  (req) => {
-    req.header.set('x-atlantis-caller', caller)
-    return next(req)
-  }
-
-export const isDataplaneError = (
-  err: unknown,
-  code?: Code,
-): err is ConnectError => {
-  if (err instanceof ConnectError) {
-    return !code || err.code === code
-  }
-  return false
-}
-
-export const unpackIdentityServices = (servicesBytes: Uint8Array) => {
-  const servicesStr = ui8.toString(servicesBytes, 'utf8')
-  if (!servicesStr) return {}
-  return JSON.parse(servicesStr) as UnpackedServices
-}
-
-export const unpackIdentityKeys = (keysBytes: Uint8Array) => {
-  const keysStr = ui8.toString(keysBytes, 'utf8')
-  if (!keysStr) return {}
-  return JSON.parse(keysStr) as UnpackedKeys
-}
-
-export const getServiceEndpoint = (
-  services: UnpackedServices,
-  opts: { id: string; type: string },
-) => {
-  const endpoint =
-    services[opts.id] &&
-    services[opts.id].Type === opts.type &&
-    validateUrl(services[opts.id].URL)
-  return endpoint || undefined
-}
-
-export const getKeyAsDidKey = (keys: UnpackedKeys, opts: { id: string }) => {
-  const key =
-    keys[opts.id] &&
-    getDidKeyFromMultibase({
-      type: keys[opts.id].Type,
-      publicKeyMultibase: keys[opts.id].PublicKeyMultibase,
-    })
-  return key || undefined
-}
-
-type UnpackedServices = Record<string, { Type: string; URL: string }>
-
-type UnpackedKeys = Record<string, { Type: string; PublicKeyMultibase: string }>
-
-const validateUrl = (urlStr: string): string | undefined => {
-  let url
-  try {
-    url = new URL(urlStr)
-  } catch {
-    return undefined
-  }
-  if (!['http:', 'https:'].includes(url.protocol)) {
-    return undefined
-  } else if (!url.hostname) {
-    return undefined
-  } else {
-    return urlStr
-  }
-}

package/src/data-plane/index.ts

@@ -1,2 +0,0 @@
-export * from './server/index.js'
-export * from './client/index.js'

package/src/data-plane/server/background.ts

@@ -1,66 +0,0 @@
-import PQueue from 'p-queue'
-import { dbLogger } from '../../logger.js'
-
-// A simple queue for in-process, out-of-band/backgrounded work
-
-type Task<TContext> = (ctx: TContext, signal: AbortSignal) => Promise<void>
-
-export type BackgroundQueueOptions = NonNullable<
-  ConstructorParameters<typeof PQueue>[0]
-> & {
-  concurrency: number
-}
-
-// @NOTE Keep this in sync with the BackgroundQueue in
-// - packages/bsky/src/data-plane/server/background.ts
-// - packages/ozone/src/background.ts
-// - packages/pds/src/background.ts
-export class BackgroundQueue<TContext = unknown> {
-  private abortController = new AbortController()
-  private queue: PQueue
-
-  public get signal(): AbortSignal {
-    return this.abortController.signal
-  }
-
-  public get destroyed() {
-    return this.signal.aborted
-  }
-
-  constructor(
-    private readonly context: TContext,
-    options?: BackgroundQueueOptions,
-  ) {
-    this.queue = new PQueue(options)
-  }
-
-  add(task: Task<TContext>) {
-    if (this.destroyed) return
-
-    this.queue.add<void>(async () => {
-      try {
-        // The task will receive a signal allowing it to abort if the
-        // backgroundQueue is destroyed.
-        await task(this.context, this.signal)
-      } catch (err) {
-        dbLogger.error({ err }, 'background queue task failed')
-      }
-    })
-  }
-
-  async processAll() {
-    const { queue } = this
-    while (queue.size || queue.pending) await queue.onIdle()
-  }
-
-  // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks.
-  // The application calls this only once http connections have drained (tasks no longer being added).
-  async destroy() {
-    if (this.destroyed) {
-      dbLogger.warn('BackgroundQueue.destroy() called multiple times')
-    }
-
-    this.abortController.abort()
-    return this.processAll()
-  }
-}