@atproto/pds 0.5.5 → 0.5.7

package/src/api/com/atproto/server/deleteAccount.ts

@@ -2,6 +2,7 @@ import { MINUTE } from '@atproto/common'
 import {
   AuthRequiredError,
   InvalidRequestError,
+  MethodRateLimit,
   Server,
 } from '@atproto/xrpc-server'
 import { OLD_PASSWORD_MAX_LENGTH } from '../../../../account-manager/helpers/scrypt.js'
@@ -11,58 +12,74 @@ import { com } from '../../../../lexicons/index.js'
 export default function (server: Server, ctx: AppContext) {
   const { entrywayClient } = ctx
 
-  server.add(com.atproto.server.deleteAccount, {
-    rateLimit: {
-      durationMs: 5 * MINUTE,
-      points: 50,
-    },
-    handler: async ({ input: { body }, req }) => {
-      const { did, password, token } = body
+  const rateLimit: MethodRateLimit = {
+    durationMs: 5 * MINUTE,
+    points: 50,
+  }
 
-      const account = await ctx.accountManager.getAccount(did, {
-        includeDeactivated: true,
-        includeTakenDown: true,
-      })
-      if (!account) {
-        throw new InvalidRequestError('account not found')
-      }
+  if (entrywayClient) {
+    server.add(com.atproto.server.deleteAccount, {
+      rateLimit,
+      handler: async ({ input: { body }, req }) => {
+        const account = await ctx.accountManager.getAccount(body.did, {
+          includeDeactivated: true,
+          includeTakenDown: true,
+        })
+        if (!account) {
+          throw new InvalidRequestError('account not found')
+        }
 
-      if (entrywayClient) {
         const { headers } = ctx.entrywayPassthruHeaders(req)
         await entrywayClient.xrpc(com.atproto.server.deleteAccount, {
           body,
           headers,
         })
-        return
-      }
+      },
+    })
+  } else {
+    server.add(com.atproto.server.deleteAccount, {
+      rateLimit,
+      handler: async ({ input: { body } }) => {
+        const { did, password, token } = body
+
+        if (password.length > OLD_PASSWORD_MAX_LENGTH) {
+          throw new AuthRequiredError(
+            'Password too long. Consider resetting your password.',
+          )
+        }
 
-      if (password.length > OLD_PASSWORD_MAX_LENGTH) {
-        throw new InvalidRequestError('Invalid password length.')
-      }
+        const account = await ctx.accountManager.getAccount(did, {
+          includeDeactivated: true,
+          includeTakenDown: true,
+        })
+        if (!account) {
+          throw new InvalidRequestError('account not found')
+        }
 
-      const validPass = await ctx.accountManager.verifyAccountPassword(
-        did,
-        password,
-      )
-      if (!validPass) {
-        throw new AuthRequiredError('Invalid did or password')
-      }
+        const validPass = await ctx.accountManager.verifyAccountPassword(
+          did,
+          password,
+        )
+        if (!validPass) {
+          throw new AuthRequiredError('Invalid did or password')
+        }
 
-      await ctx.accountManager.assertValidEmailToken(
-        did,
-        'delete_account',
-        token,
-      )
+        await ctx.accountManager.assertValidEmailToken(
+          did,
+          'delete_account',
+          token,
+        )
 
-      // @NOTE Order matters here: first "unlink" the account by removing it
-      // from the account manager database ("source of truth"), then notify the
-      // sequencer, and finally cleanup files from the file system.
-      await ctx.accountManager.deleteAccount(did)
-      try {
-        await ctx.sequencer.sequenceAccountDeletion(did)
-      } finally {
-        await ctx.actorStore.destroy(did)
-      }
-    },
-  })
+        // @NOTE Order matters here: first "unlink" the account by removing it
+        // from the account manager database ("source of truth"), then notify the
+        // sequencer, and finally cleanup files from the file system.
+        await ctx.accountManager.deleteAccount(did)
+        try {
+          await ctx.sequencer.sequenceAccountDeletion(did)
+        } finally {
+          await ctx.actorStore.destroy(did)
+        }
+      },
+    })
+  }
 }

package/src/api/com/atproto/server/util.ts

@@ -1,9 +1,10 @@
+import * as plc from '@did-plc/lib'
 import { getPdsEndpoint, getSigningDidKey } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
-import { DidDocument } from '@atproto/identity'
+import { DidDocument, IdResolver } from '@atproto/identity'
 import { InvalidRequestError } from '@atproto/xrpc-server'
+import { ActorStore } from '../../../../actor-store/actor-store.js'
 import { ServerConfig } from '../../../../config/index.js'
-import { AppContext } from '../../../../context.js'
 import { httpLogger } from '../../../../logger.js'
 
 // generate an invite code preceded by the hostname
@@ -29,7 +30,7 @@ export const getRandomToken = () => {
 }
 
 export const safeResolveDidDoc = async (
-  ctx: AppContext,
+  ctx: { idResolver: IdResolver },
   did: string,
   forceRefresh?: boolean,
 ): Promise<DidDocument | undefined> => {
@@ -42,7 +43,7 @@ export const safeResolveDidDoc = async (
 }
 
 export const didDocForSession = async (
-  ctx: AppContext,
+  ctx: { idResolver: IdResolver; cfg: ServerConfig },
   did: string,
   forceRefresh?: boolean,
 ): Promise<DidDocument | undefined> => {
@@ -51,7 +52,13 @@ export const didDocForSession = async (
 }
 
 export const isValidDidDocForService = async (
-  ctx: AppContext,
+  ctx: {
+    plcClient: plc.Client
+    idResolver: IdResolver
+    cfg: ServerConfig
+    plcRotationKey: crypto.Keypair
+    actorStore: ActorStore
+  },
   did: string,
 ): Promise<boolean> => {
   try {
@@ -63,7 +70,13 @@ export const isValidDidDocForService = async (
 }
 
 export const assertValidDidDocumentForService = async (
-  ctx: AppContext,
+  ctx: {
+    plcClient: plc.Client
+    idResolver: IdResolver
+    cfg: ServerConfig
+    plcRotationKey: crypto.Keypair
+    actorStore: ActorStore
+  },
   did: string,
 ) => {
   if (did.startsWith('did:plc')) {
@@ -86,7 +99,11 @@ export const assertValidDidDocumentForService = async (
 }
 
 const assertValidDocContents = async (
-  ctx: AppContext,
+  ctx: {
+    cfg: ServerConfig
+    plcRotationKey: crypto.Keypair
+    actorStore: ActorStore
+  },
   did: string,
   contents: {
     signingKey?: string

package/src/background.ts

@@ -1,34 +1,65 @@
 import PQueue from 'p-queue'
 import { dbLogger } from './logger.js'
-
 // A simple queue for in-process, out-of-band/backgrounded work
 
-export class BackgroundQueue {
-  queue: InstanceType<typeof PQueue> = new PQueue({ concurrency: 5 })
-  destroyed = false
-  constructor() {}
+type Task<TContext> = (ctx: TContext, signal: AbortSignal) => Promise<void>
 
-  add(task: Task) {
-    if (this.destroyed) {
-      return
-    }
-    this.queue
-      .add(() => task())
-      .catch((err) => {
+export type BackgroundQueueOptions = NonNullable<
+  ConstructorParameters<typeof PQueue>[0]
+> & {
+  concurrency: number
+}
+
+// @NOTE Keep this in sync with the BackgroundQueue in
+// - packages/bsky/src/data-plane/server/background.ts
+// - packages/ozone/src/background.ts
+// - packages/pds/src/background.ts
+export class BackgroundQueue<TContext = unknown> {
+  private abortController = new AbortController()
+  private queue: PQueue
+
+  public get signal(): AbortSignal {
+    return this.abortController.signal
+  }
+
+  public get destroyed() {
+    return this.signal.aborted
+  }
+
+  constructor(
+    private readonly context: TContext,
+    options?: BackgroundQueueOptions,
+  ) {
+    this.queue = new PQueue(options)
+  }
+
+  add(task: Task<TContext>) {
+    if (this.destroyed) return
+
+    this.queue.add<void>(async () => {
+      try {
+        // The task will receive a signal allowing it to abort if the
+        // backgroundQueue is destroyed.
+        await task(this.context, this.signal)
+      } catch (err) {
         dbLogger.error({ err }, 'background queue task failed')
-      })
+      }
+    })
   }
 
   async processAll() {
-    await this.queue.onIdle()
+    const { queue } = this
+    while (queue.size || queue.pending) await queue.onIdle()
   }
 
   // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks.
   // The application calls this only once http connections have drained (tasks no longer being added).
   async destroy() {
-    this.destroyed = true
-    await this.queue.onIdle()
+    if (this.destroyed) {
+      dbLogger.warn('BackgroundQueue.destroy() called multiple times')
+    }
+
+    this.abortController.abort()
+    return this.processAll()
   }
 }
-
-type Task = () => Promise<void>

package/src/config/config.ts

@@ -5,7 +5,7 @@ import {
   BrandingInput as BrandingConfig,
   HcaptchaConfig,
 } from '@atproto/oauth-provider'
-import { ensureValidDid } from '@atproto/syntax'
+import { DidString, ensureValidDid, isValidDid } from '@atproto/syntax'
 import { ServerEnvironment } from './env.js'
 
 export type { BrandingConfig }
@@ -21,6 +21,11 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => {
       ? `http://localhost:${port}`
       : `https://${hostname}`
   const did = env.serviceDid ?? `did:web:${hostname}`
+
+  if (!isValidDid(did)) {
+    throw new Error(`Invalid service DID: ${did}`)
+  }
+
   const serviceCfg: ServerConfig['service'] = {
     port,
     hostname,
@@ -400,7 +405,7 @@ export type ServiceConfig = {
   port: number
   hostname: string
   publicUrl: string
-  did: string
+  did: DidString
   version?: string
   privacyPolicyUrl?: string
   termsOfServiceUrl?: string

package/src/context.ts

@@ -21,12 +21,7 @@ import {
   createServiceAuthHeaders,
   createServiceJwt,
 } from '@atproto/xrpc-server'
-import {
-  Fetch,
-  isUnicastIp,
-  safeFetchWrap,
-  unicastLookup,
-} from '@atproto-labs/fetch-node'
+import { Fetch, safeFetchWrap } from '@atproto-labs/fetch-node'
 import { AccountManager } from './account-manager/account-manager.js'
 import { OAuthStore } from './account-manager/oauth-store.js'
 import { ScopeReferenceGetter } from './account-manager/scope-reference-getter.js'
@@ -47,6 +42,7 @@ import { ImageUrlBuilder } from './image/image-url-builder.js'
 import { fetchLogger, lexiconResolverLogger, oauthLogger } from './logger.js'
 import { ServerMailer } from './mailer/index.js'
 import { ModerationMailer } from './mailer/moderation.js'
+import { buildProxyAgent } from './pipethrough.js'
 import { LocalViewer, LocalViewerCreator } from './read-after-write/viewer.js'
 import { getRedisClient } from './redis.js'
 import { Sequencer } from './sequencer/index.js'
@@ -181,7 +177,7 @@ export class AppContext {
     })
     const plcClient = new plc.Client(cfg.identity.plcUrl)
 
-    const backgroundQueue = new BackgroundQueue()
+    const backgroundQueue = new BackgroundQueue(undefined, { concurrency: 5 })
     const crawlers = new Crawlers(
       backgroundQueue,
       cfg.service.hostname,
@@ -277,15 +273,14 @@ export class AppContext {
           )
 
     const accountManager = new AccountManager(
+      cfg,
+      actorStore,
       idResolver,
       jwtSecretKey,
       mailer,
       sequencer,
       plcClient,
       plcRotationKey,
-      cfg.service.did,
-      cfg.identity.serviceHandleDomains,
-      cfg.db,
     )
     await accountManager.migrateOrThrow()
 
@@ -296,36 +291,7 @@ export class AppContext {
     )
 
     // An agent for performing HTTP requests based on user provided URLs.
-    const proxyAgentBase = new undici.Agent({
-      allowH2: cfg.proxy.allowHTTP2, // This is experimental
-      headersTimeout: cfg.proxy.headersTimeout,
-      maxResponseSize: cfg.proxy.maxResponseSize,
-      bodyTimeout: cfg.proxy.bodyTimeout,
-      factory: cfg.proxy.disableSsrfProtection
-        ? undefined
-        : (origin, opts) => {
-            const { protocol, hostname } =
-              origin instanceof URL ? origin : new URL(origin)
-            if (protocol !== 'https:') {
-              throw new Error(`Forbidden protocol "${protocol}"`)
-            }
-            if (isUnicastIp(hostname) === false) {
-              throw new Error('Hostname resolved to non-unicast address')
-            }
-            return new undici.Pool(origin, opts)
-          },
-      connect: {
-        lookup: cfg.proxy.disableSsrfProtection ? undefined : unicastLookup,
-      },
-    })
-    const proxyAgent =
-      cfg.proxy.maxRetries > 0
-        ? new undici.RetryAgent(proxyAgentBase, {
-            statusCodes: [], // Only retry on socket errors
-            methods: ['GET', 'HEAD'],
-            maxRetries: cfg.proxy.maxRetries,
-          })
-        : proxyAgentBase
+    const proxyAgent = buildProxyAgent(cfg.proxy)
 
     /**
      * A fetch() function that protects against SSRF attacks, large responses &

package/src/db/migrator.ts

@@ -1,4 +1,5 @@
-import { Kysely, Migration, Migrator as KyselyMigrator } from 'kysely'
+import { Kysely } from 'kysely'
+import { Migration, Migrator as KyselyMigrator } from 'kysely/migration'
 
 export class Migrator<T> extends KyselyMigrator {
   constructor(

package/src/db/pagination.ts

@@ -1,4 +1,4 @@
-import { sql } from 'kysely'
+import { SqlBool, sql } from 'kysely'
 import { InvalidRequestError } from '@atproto/xrpc-server'
 import { AnyQb, DbRef } from './util.js'
 
@@ -66,16 +66,16 @@ export abstract class GenericKeyset<R, LR extends LabeledResult> {
     if (tryIndex) {
       // The tryIndex param will likely disappear and become the default implementation: here for now for gradual rollout query-by-query.
       if (direction === 'asc') {
-        return sql`((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}))`
+        return sql<SqlBool>`((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}))`
       } else {
-        return sql`((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}))`
+        return sql<SqlBool>`((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}))`
       }
     } else {
       // @NOTE this implementation can struggle to use an index on (primary, secondary) for pagination due to the "or" usage.
       if (direction === 'asc') {
-        return sql`((${this.primary} > ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} > ${labeled.secondary}))`
+        return sql<SqlBool>`((${this.primary} > ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} > ${labeled.secondary}))`
       } else {
-        return sql`((${this.primary} < ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} < ${labeled.secondary}))`
+        return sql<SqlBool>`((${this.primary} < ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} < ${labeled.secondary}))`
       }
     }
   }
@@ -125,8 +125,8 @@ export const paginate = <
   const { limit, cursor, keyset, direction = 'desc', tryIndex } = opts
   const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex)
   return qb
-    .if(!!limit, (q) => q.limit(limit as number))
+    .$if(!!limit, (q) => q.limit(limit as number))
     .orderBy(keyset.primary, direction)
     .orderBy(keyset.secondary, direction)
-    .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
+    .$if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb)) as QB
 }

package/src/db/util.ts

@@ -5,6 +5,7 @@ import {
   RawBuilder,
   ReferenceExpression,
   SelectQueryBuilder,
+  SqlBool,
   SqliteAdapter,
   SqliteIntrospector,
   SqliteQueryCompiler,
@@ -14,7 +15,7 @@ import { retry } from '@atproto/common'
 
 // Applies to repo_root or record table
 export const notSoftDeletedClause = (alias: DbRef) => {
-  return sql`${alias}."takedownRef" is null`
+  return sql<SqlBool>`${alias}."takedownRef" is null`
 }
 
 export const softDeleted = (repoOrRecord: { takedownRef: string | null }) => {
@@ -92,7 +93,9 @@ const RETRY_ERRORS = new Set([
 
 export type Ref = ReferenceExpression<any, any>
 
-export type DbRef = RawBuilder | ReturnType<DynamicModule['ref']>
+export type DbRef =
+  | RawBuilder<unknown>
+  | ReturnType<DynamicModule<unknown>['ref']>
 
 export type AnyQb = SelectQueryBuilder<any, any, any>
 

package/src/index.ts

@@ -9,11 +9,8 @@ import http from 'node:http'
 import { PlcClientError } from '@did-plc/lib'
 import cors from 'cors'
 import express from 'express'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
+// eslint-disable-next-line import/default
 import httpTerminator from 'http-terminator'
-// eslint-disable-next-line import/no-named-as-default-member
-const { createHttpTerminator } = httpTerminator
-type HttpTerminator = ReturnType<typeof createHttpTerminator>
 import { DAY, SECOND } from '@atproto/common'
 import {
   MethodHandler,
@@ -64,7 +61,7 @@ export class PDS {
   public ctx: AppContext
   public app: express.Application
   public server?: http.Server
-  private terminator?: HttpTerminator
+  private terminator?: httpTerminator.HttpTerminator
   private dbStatsInterval?: NodeJS.Timeout
   private sequencerStatsInterval?: NodeJS.Timeout
 
@@ -145,21 +142,42 @@ export class PDS {
     await this.ctx.sequencer.start()
     const server = this.app.listen(this.ctx.cfg.service.port)
     this.server = server
-    this.server.keepAliveTimeout = 90000
-    this.terminator = createHttpTerminator({ server })
+    this.server.keepAliveTimeout = 90_000
+    this.terminator = httpTerminator.createHttpTerminator({ server })
     await events.once(server, 'listening')
     return server
   }
 
   async destroy(): Promise<void> {
-    await this.ctx.sequencer.destroy()
-    await this.terminator?.terminate()
-    await this.ctx.backgroundQueue.destroy()
-    await this.ctx.accountManager.close()
-    await this.ctx.redisScratch?.quit()
-    await this.ctx.proxyAgent.destroy()
     clearInterval(this.dbStatsInterval)
     clearInterval(this.sequencerStatsInterval)
+
+    // @TODO Use disposable stack when it becomes available (Node24+)
+    try {
+      await this.terminator?.terminate()
+    } finally {
+      try {
+        await this.ctx.backgroundQueue.destroy()
+      } finally {
+        try {
+          await this.ctx.sequencer.destroy()
+        } finally {
+          try {
+            await this.ctx.accountManager.close()
+          } finally {
+            try {
+              await this.ctx.redisScratch?.quit()
+            } finally {
+              await this.ctx.proxyAgent.destroy()
+            }
+          }
+        }
+      }
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.destroy()
   }
 }
 

package/src/mailer/index.ts

@@ -36,9 +36,10 @@ export class ServerMailer {
   }
 
   async sendResetPassword(
-    params: { handle: string; token: string },
+    params: { handle: string; token: string; locale?: string },
     mailOpts: SendMailOptions,
   ) {
+    // @TODO (later) handle locale in the template
     await this.sendTemplate('resetPassword', params, {
       subject: 'Password Reset Requested',
       ...mailOpts,
@@ -46,9 +47,10 @@ export class ServerMailer {
   }
 
   async sendAccountDelete(
-    params: { token: string },
+    params: { token: string; locale?: string },
     mailOpts: SendMailOptions,
   ) {
+    // @TODO (later) handle locale in the template
     await this.sendTemplate('deleteAccount', params, {
       subject: 'Account Deletion Requested',
       ...mailOpts,

package/src/pipethrough.ts

@@ -1,7 +1,7 @@
 import { IncomingHttpHeaders, ServerResponse } from 'node:http'
 import { PassThrough, Readable, finished } from 'node:stream'
 import { Request } from 'express'
-import { Dispatcher } from 'undici'
+import { Agent, Dispatcher, Pool, interceptors } from 'undici'
 import {
   decodeStream,
   getServiceEndpoint,
@@ -20,12 +20,49 @@ import {
   excludeErrorResult,
   parseReqNsid,
 } from '@atproto/xrpc-server'
+import { isUnicastIp, unicastLookup } from '@atproto-labs/fetch-node'
 import { buildProxiedContentEncoding } from '@atproto-labs/xrpc-utils'
 import { isAccessPrivileged } from './auth-scope.js'
+import { ProxyConfig } from './config/config.js'
 import { AppContext } from './context.js'
 import { chat, com, tools } from './lexicons/index.js'
 import { httpLogger } from './logger.js'
 
+export function buildProxyAgent(cfg: ProxyConfig): Dispatcher {
+  const agent = new Agent({
+    allowH2: cfg.allowHTTP2,
+    headersTimeout: cfg.headersTimeout,
+    maxResponseSize: cfg.maxResponseSize,
+    bodyTimeout: cfg.bodyTimeout,
+    factory: cfg.disableSsrfProtection
+      ? undefined
+      : (origin, opts) => {
+          const { protocol, hostname } =
+            origin instanceof URL ? origin : new URL(origin)
+          if (protocol !== 'https:') {
+            throw new Error(`Forbidden protocol "${protocol}"`)
+          }
+          if (isUnicastIp(hostname) === false) {
+            throw new Error('Hostname resolved to non-unicast address')
+          }
+          return new Pool(origin, opts)
+        },
+    connect: {
+      lookup: cfg.disableSsrfProtection ? undefined : unicastLookup,
+    },
+  })
+
+  return agent.compose(
+    cfg.maxRetries > 0
+      ? interceptors.retry({
+          statusCodes: [], // Only retry on socket errors
+          methods: ['GET', 'HEAD'],
+          maxRetries: cfg.maxRetries,
+        })
+      : (dispatch) => dispatch,
+  )
+}
+
 export const proxyHandler = (ctx: AppContext): CatchallHandler => {
   const performAuth = ctx.authVerifier.authorization<RpcPermissionMatch>({
     authorize: (permissions, { params }) => permissions.assertRpc(params),

package/src/scripts/sequencer-recovery/recovery-db.ts

@@ -35,7 +35,7 @@ export const getAndMigrateRecoveryDb = async (
   const pragmas: Record<string, string> = disableWalAutoCheckpoint
     ? { wal_autocheckpoint: '0' }
     : {}
-  const db = Database.sqlite(location, pragmas)
+  const db = Database.sqlite<RecoveryDbSchema>(location, pragmas)
   const migrator = new Migrator(db.db, migrations)
   await migrator.migrateToLatestOrThrow()
   return db

package/src/sequencer/events.ts

@@ -83,12 +83,10 @@ export const formatSeqIdentityEvt = async (
   did: DidString,
   handle?: HandleString,
 ): Promise<RepoSeqInsert> => {
-  const evt: IdentityEvt = {
-    did,
-  }
-  if (handle) {
-    evt.handle = handle
-  }
+  const evt: IdentityEvt = handle //
+    ? { did, handle }
+    : { did }
+
   return {
     did,
     eventType: 'identity',
@@ -101,13 +99,10 @@ export const formatSeqAccountEvt = async (
   did: DidString,
   status: AccountStatus,
 ): Promise<RepoSeqInsert> => {
-  const evt: AccountEvt = {
-    did,
-    active: status === 'active',
-  }
-  if (status !== AccountStatus.Active) {
-    evt.status = status
-  }
+  const evt: AccountEvt =
+    status === AccountStatus.Active
+      ? { did, active: true }
+      : { did, active: false, status }
 
   return {
     did,