@atproto/pds 0.5.11 → 0.5.14

package/src/background.ts

@@ -1,65 +0,0 @@
-import PQueue from 'p-queue'
-import { dbLogger } from './logger.js'
-// A simple queue for in-process, out-of-band/backgrounded work
-
-type Task<TContext> = (ctx: TContext, signal: AbortSignal) => Promise<void>
-
-export type BackgroundQueueOptions = NonNullable<
-  ConstructorParameters<typeof PQueue>[0]
-> & {
-  concurrency: number
-}
-
-// @NOTE Keep this in sync with the BackgroundQueue in
-// - packages/bsky/src/data-plane/server/background.ts
-// - packages/ozone/src/background.ts
-// - packages/pds/src/background.ts
-export class BackgroundQueue<TContext = unknown> {
-  private abortController = new AbortController()
-  private queue: PQueue
-
-  public get signal(): AbortSignal {
-    return this.abortController.signal
-  }
-
-  public get destroyed() {
-    return this.signal.aborted
-  }
-
-  constructor(
-    private readonly context: TContext,
-    options?: BackgroundQueueOptions,
-  ) {
-    this.queue = new PQueue(options)
-  }
-
-  add(task: Task<TContext>) {
-    if (this.destroyed) return
-
-    this.queue.add<void>(async () => {
-      try {
-        // The task will receive a signal allowing it to abort if the
-        // backgroundQueue is destroyed.
-        await task(this.context, this.signal)
-      } catch (err) {
-        dbLogger.error({ err }, 'background queue task failed')
-      }
-    })
-  }
-
-  async processAll() {
-    const { queue } = this
-    while (queue.size || queue.pending) await queue.onIdle()
-  }
-
-  // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks.
-  // The application calls this only once http connections have drained (tasks no longer being added).
-  async destroy() {
-    if (this.destroyed) {
-      dbLogger.warn('BackgroundQueue.destroy() called multiple times')
-    }
-
-    this.abortController.abort()
-    return this.processAll()
-  }
-}

package/src/basic-routes.ts

@@ -1,52 +0,0 @@
-import { Router } from 'express'
-import { sql } from 'kysely'
-import { AppContext } from './context.js'
-
-export const createRouter = (ctx: AppContext): Router => {
-  const router = Router()
-
-  router.get('/', function (req, res) {
-    res.type('text/plain')
-    res.send(`
-         __                         __
-        /\\ \\__                     /\\ \\__
-    __  \\ \\ ,_\\  _____   _ __   ___\\ \\ ,_\\   ___
-  /'__'\\ \\ \\ \\/ /\\ '__'\\/\\''__\\/ __'\\ \\ \\/  / __'\\
- /\\ \\L\\.\\_\\ \\ \\_\\ \\ \\L\\ \\ \\ \\//\\ \\L\\ \\ \\ \\_/\\ \\L\\ \\
- \\ \\__/.\\_\\\\ \\__\\\\ \\ ,__/\\ \\_\\\\ \\____/\\ \\__\\ \\____/
-  \\/__/\\/_/ \\/__/ \\ \\ \\/  \\/_/ \\/___/  \\/__/\\/___/
-                   \\ \\_\\
-                    \\/_/
-
-
-This is an AT Protocol Personal Data Server (aka, an atproto PDS)
-
-Most API routes are under /xrpc/
-
-      Code: https://github.com/bluesky-social/atproto
- Self-Host: https://github.com/bluesky-social/pds
-  Protocol: https://atproto.com
-`)
-  })
-
-  router.get('/robots.txt', function (req, res) {
-    res.type('text/plain')
-    res.send(
-      '# Hello!\n\n# Crawling the public API is allowed\nUser-agent: *\nAllow: /',
-    )
-  })
-
-  router.get('/xrpc/_health', async function (req, res) {
-    const { version } = ctx.cfg.service
-    try {
-      await sql`select 1`.execute(ctx.accountManager.db.db)
-    } catch (err) {
-      req.log.error({ err }, 'failed health check')
-      res.status(503).send({ version, error: 'Service Unavailable' })
-      return
-    }
-    res.send({ version })
-  })
-
-  return router
-}

package/src/bsky-app-view.ts

@@ -1,33 +0,0 @@
-import { format } from 'node:util'
-import { Client } from '@atproto/lex'
-
-export type AppViewOptions = {
-  url: string
-  did: string
-  cdnUrlPattern?: string
-  validateResponse?: boolean
-}
-
-export class BskyAppView {
-  public did: string
-  public url: string
-  public client: Client
-  private cdnUrlPattern?: string
-
-  constructor(options: AppViewOptions) {
-    this.did = options.did
-    this.url = options.url
-    this.client = new Client(
-      { service: options.url },
-      {
-        strictResponseProcessing: false,
-        validateResponse: options.validateResponse ?? false,
-      },
-    )
-    this.cdnUrlPattern = options.cdnUrlPattern
-  }
-
-  getImageUrl(pattern: string, did: string, cid: string): string | undefined {
-    if (this.cdnUrlPattern) return format(this.cdnUrlPattern, pattern, did, cid)
-  }
-}

package/src/config/config.ts

@@ -1,553 +0,0 @@
-import assert from 'node:assert'
-import path from 'node:path'
-import { DAY, HOUR, SECOND } from '@atproto/common'
-import {
-  BrandingInput as BrandingConfig,
-  HcaptchaConfig,
-} from '@atproto/oauth-provider'
-import { DidString, ensureValidDid, isValidDid } from '@atproto/syntax'
-import { ServerEnvironment } from './env.js'
-
-export type { BrandingConfig }
-
-// off-config but still from env:
-// logging: LOG_LEVEL, LOG_SYSTEMS, LOG_ENABLED, LOG_DESTINATION
-
-export const envToCfg = (env: ServerEnvironment): ServerConfig => {
-  const port = env.port ?? 2583
-  const hostname = env.hostname ?? 'localhost'
-  const publicUrl =
-    hostname === 'localhost'
-      ? `http://localhost:${port}`
-      : `https://${hostname}`
-  const did = env.serviceDid ?? `did:web:${hostname}`
-
-  if (!isValidDid(did)) {
-    throw new Error(`Invalid service DID: ${did}`)
-  }
-
-  const serviceCfg: ServerConfig['service'] = {
-    port,
-    hostname,
-    publicUrl,
-    did,
-    version: env.version, // default?
-    privacyPolicyUrl: env.privacyPolicyUrl,
-    termsOfServiceUrl: env.termsOfServiceUrl,
-    contactEmailAddress: env.contactEmailAddress,
-    acceptingImports: env.acceptingImports ?? true,
-    maxImportSize: env.maxImportSize,
-    blobUploadLimit: env.blobUploadLimit ?? 5 * 1024 * 1024, // 5mb
-    devMode: env.devMode ?? false,
-  }
-
-  const dbLoc = (name: string) => {
-    return env.dataDirectory ? path.join(env.dataDirectory, name) : name
-  }
-
-  const disableWalAutoCheckpoint = env.disableWalAutoCheckpoint ?? false
-
-  const dbCfg: ServerConfig['db'] = {
-    accountDbLoc: env.accountDbLocation ?? dbLoc('account.sqlite'),
-    sequencerDbLoc: env.sequencerDbLocation ?? dbLoc('sequencer.sqlite'),
-    didCacheDbLoc: env.didCacheDbLocation ?? dbLoc('did_cache.sqlite'),
-    disableWalAutoCheckpoint,
-  }
-
-  const actorStoreCfg: ServerConfig['actorStore'] = {
-    directory: env.actorStoreDirectory ?? dbLoc('actors'),
-    cacheSize: env.actorStoreCacheSize ?? 100,
-    disableWalAutoCheckpoint,
-  }
-
-  let blobstoreCfg: ServerConfig['blobstore']
-  if (env.blobstoreS3Bucket && env.blobstoreDiskLocation) {
-    throw new Error('Cannot set both S3 and disk blobstore env vars')
-  }
-  if (env.blobstoreS3Bucket) {
-    blobstoreCfg = {
-      provider: 's3',
-      bucket: env.blobstoreS3Bucket,
-      uploadTimeoutMs: env.blobstoreS3UploadTimeoutMs || 20000,
-      region: env.blobstoreS3Region,
-      endpoint: env.blobstoreS3Endpoint,
-      forcePathStyle: env.blobstoreS3ForcePathStyle,
-    }
-    if (env.blobstoreS3AccessKeyId || env.blobstoreS3SecretAccessKey) {
-      if (!env.blobstoreS3AccessKeyId || !env.blobstoreS3SecretAccessKey) {
-        throw new Error(
-          'Must specify both S3 access key id and secret access key blobstore env vars',
-        )
-      }
-      blobstoreCfg.credentials = {
-        accessKeyId: env.blobstoreS3AccessKeyId,
-        secretAccessKey: env.blobstoreS3SecretAccessKey,
-      }
-    }
-  } else if (env.blobstoreDiskLocation) {
-    blobstoreCfg = {
-      provider: 'disk',
-      location: env.blobstoreDiskLocation,
-      tempLocation: env.blobstoreDiskTmpLocation,
-    }
-  } else {
-    throw new Error('Must configure either S3 or disk blobstore')
-  }
-
-  let serviceHandleDomains: string[]
-  if (env.serviceHandleDomains && env.serviceHandleDomains.length > 0) {
-    serviceHandleDomains = env.serviceHandleDomains
-  } else {
-    if (hostname === 'localhost') {
-      serviceHandleDomains = ['.test']
-    } else {
-      serviceHandleDomains = [`.${hostname}`]
-    }
-  }
-  const invalidDomain = serviceHandleDomains.find(
-    (domain) => domain.length < 1 || !domain.startsWith('.'),
-  )
-  if (invalidDomain) {
-    throw new Error(`Invalid handle domain: ${invalidDomain}`)
-  }
-
-  const identityCfg: ServerConfig['identity'] = {
-    plcUrl: env.didPlcUrl ?? 'https://plc.directory',
-    cacheMaxTTL: env.didCacheMaxTTL ?? DAY,
-    cacheStaleTTL: env.didCacheStaleTTL ?? HOUR,
-    resolverTimeout: env.resolverTimeout ?? 3 * SECOND,
-    recoveryDidKey: env.recoveryDidKey ?? null,
-    serviceHandleDomains,
-    handleBackupNameservers: env.handleBackupNameservers,
-    enableDidDocWithSession: !!env.enableDidDocWithSession,
-  }
-
-  let entrywayCfg: ServerConfig['entryway'] = null
-  if (env.entrywayUrl) {
-    assert(
-      env.entrywayJwtVerifyKeyK256PublicKeyHex &&
-        env.entrywayPlcRotationKey &&
-        env.entrywayDid,
-      'if entryway url is configured, must include all required entryway configuration',
-    )
-    entrywayCfg = {
-      url: env.entrywayUrl,
-      did: env.entrywayDid,
-      jwtPublicKeyHex: env.entrywayJwtVerifyKeyK256PublicKeyHex,
-      plcRotationKey: env.entrywayPlcRotationKey,
-    }
-  }
-
-  // default to being required if left undefined
-  const invitesCfg: ServerConfig['invites'] =
-    env.inviteRequired === false
-      ? {
-          required: false,
-        }
-      : {
-          required: true,
-          interval: env.inviteInterval ?? null,
-          epoch: env.inviteEpoch ?? 0,
-        }
-
-  let emailCfg: ServerConfig['email']
-  if (!env.emailFromAddress && !env.emailSmtpUrl) {
-    emailCfg = null
-  } else {
-    if (!env.emailFromAddress || !env.emailSmtpUrl) {
-      throw new Error(
-        'Partial email config, must set both emailFromAddress and emailSmtpUrl',
-      )
-    }
-    emailCfg = {
-      smtpUrl: env.emailSmtpUrl,
-      fromAddress: env.emailFromAddress,
-      disableConfirmationLink: env.emailDisableConfirmationLink ?? false,
-    }
-  }
-
-  let moderationEmailCfg: ServerConfig['moderationEmail']
-  if (!env.moderationEmailAddress && !env.moderationEmailSmtpUrl) {
-    moderationEmailCfg = null
-  } else {
-    if (!env.moderationEmailAddress || !env.moderationEmailSmtpUrl) {
-      throw new Error(
-        'Partial moderation email config, must set both emailFromAddress and emailSmtpUrl',
-      )
-    }
-    moderationEmailCfg = {
-      smtpUrl: env.moderationEmailSmtpUrl,
-      fromAddress: env.moderationEmailAddress,
-      disableConfirmationLink: false,
-    }
-  }
-
-  const subscriptionCfg: ServerConfig['subscription'] = {
-    maxBuffer: env.maxSubscriptionBuffer ?? 500,
-    repoBackfillLimitMs: env.repoBackfillLimitMs ?? DAY,
-  }
-
-  let bskyAppViewCfg: ServerConfig['bskyAppView'] = null
-  if (env.bskyAppViewUrl) {
-    assert(
-      env.bskyAppViewDid,
-      'if bsky appview service url is configured, must configure its did as well.',
-    )
-    bskyAppViewCfg = {
-      url: env.bskyAppViewUrl,
-      did: env.bskyAppViewDid,
-      cdnUrlPattern: env.bskyAppViewCdnUrlPattern,
-    }
-  }
-
-  let modServiceCfg: ServerConfig['modService'] = null
-  if (env.modServiceUrl) {
-    assert(
-      env.modServiceDid,
-      'if mod service url is configured, must configure its did as well.',
-    )
-    modServiceCfg = {
-      url: env.modServiceUrl,
-      did: env.modServiceDid,
-    }
-  }
-
-  let reportServiceCfg: ServerConfig['reportService'] = null
-  if (env.reportServiceUrl) {
-    assert(
-      env.reportServiceDid,
-      'if report service url is configured, must configure its did as well.',
-    )
-    reportServiceCfg = {
-      url: env.reportServiceUrl,
-      did: env.reportServiceDid,
-    }
-  }
-
-  // if there's a mod service, default report service into it
-  if (modServiceCfg && !reportServiceCfg) {
-    reportServiceCfg = modServiceCfg
-  }
-
-  const redisCfg: ServerConfig['redis'] = env.redisScratchAddress
-    ? {
-        address: env.redisScratchAddress,
-        password: env.redisScratchPassword,
-      }
-    : null
-
-  const rateLimitsCfg: ServerConfig['rateLimits'] = env.rateLimitsEnabled
-    ? {
-        enabled: true,
-        bypassKey: env.rateLimitBypassKey,
-        bypassIps: env.rateLimitBypassIps?.map((ipOrCidr) =>
-          ipOrCidr.split('/')[0]?.trim(),
-        ),
-      }
-    : { enabled: false }
-
-  const crawlersCfg: ServerConfig['crawlers'] = env.crawlers ?? []
-
-  const fetchCfg: ServerConfig['fetch'] = {
-    disableSsrfProtection: env.disableSsrfProtection ?? env.devMode ?? false,
-    maxResponseSize: env.fetchMaxResponseSize ?? 512 * 1024, // 512kb
-  }
-
-  const proxyCfg: ServerConfig['proxy'] = {
-    disableSsrfProtection: env.disableSsrfProtection ?? env.devMode ?? false,
-    allowHTTP2: env.proxyAllowHTTP2 ?? false,
-    headersTimeout: env.proxyHeadersTimeout ?? 10e3,
-    bodyTimeout: env.proxyBodyTimeout ?? 30e3,
-    maxResponseSize: env.proxyMaxResponseSize ?? 10 * 1024 * 1024, // 10mb
-    maxRetries:
-      env.proxyMaxRetries != null && env.proxyMaxRetries > 0
-        ? env.proxyMaxRetries
-        : 0,
-    preferCompressed: env.proxyPreferCompressed ?? false,
-  }
-
-  const brandingCfg = {
-    name: env.serviceName ?? `${hostname} PDS`,
-    logo: env.logoUrl,
-    colors: {
-      light: env.lightColor,
-      dark: env.darkColor,
-
-      contrastSaturation: env.contrastSaturation,
-
-      primary: env.primaryColor,
-      primaryContrast: env.primaryColorContrast,
-      primaryHue: env.primaryColorHue,
-
-      error: env.errorColor,
-      errorContrast: env.errorColorContrast,
-      errorHue: env.errorColorHue,
-
-      warning: env.warningColor,
-      warningContrast: env.warningColorContrast,
-      warningHue: env.warningColorHue,
-
-      info: env.infoColor,
-      infoContrast: env.infoColorContrast,
-      infoHue: env.infoColorHue,
-
-      success: env.successColor,
-      successContrast: env.successColorContrast,
-      successHue: env.successColorHue,
-    },
-    links: [
-      {
-        title: { en: 'Home', fr: 'Accueil' },
-        href: env.homeUrl,
-        rel: 'canonical' as const, // Prevents login page from being indexed
-      },
-      {
-        title: { en: 'Terms of Service' },
-        href: env.termsOfServiceUrl,
-        rel: 'terms-of-service' as const,
-      },
-      {
-        title: { en: 'Privacy Policy' },
-        href: env.privacyPolicyUrl,
-        rel: 'privacy-policy' as const,
-      },
-      {
-        title: { en: 'Support' },
-        href: env.supportUrl,
-        rel: 'help' as const,
-      },
-    ].filter(
-      <T extends { href?: string }>(f: T): f is T & { href: string } =>
-        f.href != null && f.href !== '',
-    ),
-  }
-
-  const oauthCfg: ServerConfig['oauth'] = entrywayCfg
-    ? {
-        issuer: entrywayCfg.url,
-        provider: undefined,
-      }
-    : {
-        issuer: serviceCfg.publicUrl,
-        provider: {
-          hcaptcha:
-            env.hcaptchaSiteKey &&
-            env.hcaptchaSecretKey &&
-            env.hcaptchaTokenSalt
-              ? {
-                  siteKey: env.hcaptchaSiteKey,
-                  secretKey: env.hcaptchaSecretKey,
-                  tokenSalt: env.hcaptchaTokenSalt,
-                }
-              : undefined,
-          branding: brandingCfg,
-          trustedClients: env.trustedOAuthClients,
-        },
-      }
-
-  const lexiconCfg: LexiconResolverConfig = {}
-
-  if (env.lexiconDidAuthority != null) {
-    ensureValidDid(env.lexiconDidAuthority)
-    lexiconCfg.didAuthority = env.lexiconDidAuthority
-  }
-
-  return {
-    service: serviceCfg,
-    db: dbCfg,
-    actorStore: actorStoreCfg,
-    blobstore: blobstoreCfg,
-    identity: identityCfg,
-    entryway: entrywayCfg,
-    invites: invitesCfg,
-    email: emailCfg,
-    moderationEmail: moderationEmailCfg,
-    subscription: subscriptionCfg,
-    bskyAppView: bskyAppViewCfg,
-    modService: modServiceCfg,
-    reportService: reportServiceCfg,
-    redis: redisCfg,
-    rateLimits: rateLimitsCfg,
-    crawlers: crawlersCfg,
-    fetch: fetchCfg,
-    lexicon: lexiconCfg,
-    proxy: proxyCfg,
-    branding: brandingCfg,
-    oauth: oauthCfg,
-  }
-}
-
-export type ServerConfig = {
-  service: ServiceConfig
-  db: DatabaseConfig
-  actorStore: ActorStoreConfig
-  blobstore: S3BlobstoreConfig | DiskBlobstoreConfig
-  identity: IdentityConfig
-  entryway: EntrywayConfig | null
-  invites: InvitesConfig
-  email: EmailConfig | null
-  moderationEmail: EmailConfig | null
-  subscription: SubscriptionConfig
-  bskyAppView: BksyAppViewConfig | null
-  modService: ModServiceConfig | null
-  reportService: ReportServiceConfig | null
-  redis: RedisScratchConfig | null
-  rateLimits: RateLimitsConfig
-  crawlers: string[]
-  fetch: FetchConfig
-  proxy: ProxyConfig
-  branding: BrandingConfig
-  oauth: OAuthConfig
-  lexicon: LexiconResolverConfig
-}
-
-export type ServiceConfig = {
-  port: number
-  hostname: string
-  publicUrl: string
-  did: DidString
-  version?: string
-  privacyPolicyUrl?: string
-  termsOfServiceUrl?: string
-  acceptingImports: boolean
-  maxImportSize?: number
-  blobUploadLimit: number
-  contactEmailAddress?: string
-  devMode: boolean
-}
-
-export type DatabaseConfig = {
-  accountDbLoc: string
-  sequencerDbLoc: string
-  didCacheDbLoc: string
-  disableWalAutoCheckpoint: boolean
-}
-
-export type ActorStoreConfig = {
-  directory: string
-  cacheSize: number
-  disableWalAutoCheckpoint: boolean
-}
-
-export type S3BlobstoreConfig = {
-  provider: 's3'
-  bucket: string
-  region?: string
-  endpoint?: string
-  forcePathStyle?: boolean
-  uploadTimeoutMs?: number
-  credentials?: {
-    accessKeyId: string
-    secretAccessKey: string
-  }
-}
-
-export type DiskBlobstoreConfig = {
-  provider: 'disk'
-  location: string
-  tempLocation?: string
-}
-
-export type IdentityConfig = {
-  plcUrl: string
-  resolverTimeout: number
-  cacheStaleTTL: number
-  cacheMaxTTL: number
-  recoveryDidKey: string | null
-  serviceHandleDomains: string[]
-  handleBackupNameservers?: string[]
-  enableDidDocWithSession: boolean
-}
-
-export type EntrywayConfig = {
-  url: string
-  did: string
-  jwtPublicKeyHex: string
-  plcRotationKey: string
-}
-
-export type FetchConfig = {
-  disableSsrfProtection: boolean
-  maxResponseSize: number
-}
-
-export type ProxyConfig = {
-  disableSsrfProtection: boolean
-  allowHTTP2: boolean
-  headersTimeout: number
-  bodyTimeout: number
-  maxResponseSize: number
-  maxRetries: number
-
-  /**
-   * When proxying requests that might get intercepted (for read-after-write) we
-   * negotiate the encoding based on the client's preferences. We will however
-   * use or own weights in order to be able to better control if the PDS will
-   * need to perform content decoding. This settings allows to prefer compressed
-   * content over uncompressed one.
-   */
-  preferCompressed: boolean
-}
-
-export type OAuthConfig = {
-  issuer: string
-  provider?: {
-    hcaptcha?: HcaptchaConfig
-    branding: BrandingConfig
-    trustedClients?: string[]
-  }
-}
-
-export type LexiconResolverConfig = {
-  didAuthority?: `did:${string}:${string}`
-}
-
-export type InvitesConfig =
-  | {
-      required: true
-      interval: number | null
-      epoch: number
-    }
-  | {
-      required: false
-    }
-
-export type EmailConfig = {
-  smtpUrl: string
-  fromAddress: string
-  disableConfirmationLink: boolean
-}
-
-export type SubscriptionConfig = {
-  maxBuffer: number
-  repoBackfillLimitMs: number
-}
-
-export type RedisScratchConfig = {
-  address: string
-  password?: string
-}
-
-export type RateLimitsConfig =
-  | {
-      enabled: true
-      bypassKey?: string
-      bypassIps?: string[]
-    }
-  | { enabled: false }
-
-export type BksyAppViewConfig = {
-  url: string
-  did: string
-  cdnUrlPattern?: string
-}
-
-export type ModServiceConfig = {
-  url: string
-  did: string
-}
-
-export type ReportServiceConfig = {
-  url: string
-  did: string
-}