@atproto/pds 0.4.34 → 0.4.36

package/src/config/env.ts

@@ -6,14 +6,22 @@ export const readEnv = (): ServerEnvironment => {
     port: envInt('PDS_PORT'),
     hostname: envStr('PDS_HOSTNAME'),
     serviceDid: envStr('PDS_SERVICE_DID'),
+    serviceName: envStr('PDS_SERVICE_NAME'),
     version: envStr('PDS_VERSION'),
+    homeUrl: envStr('PDS_HOME_URL'),
+    logoUrl: envStr('PDS_LOGO_URL'),
     privacyPolicyUrl: envStr('PDS_PRIVACY_POLICY_URL'),
+    supportUrl: envStr('PDS_SUPPORT_URL'),
     termsOfServiceUrl: envStr('PDS_TERMS_OF_SERVICE_URL'),
     contactEmailAddress: envStr('PDS_CONTACT_EMAIL_ADDRESS'),
     acceptingImports: envBool('PDS_ACCEPTING_REPO_IMPORTS'),
     blobUploadLimit: envInt('PDS_BLOB_UPLOAD_LIMIT'),
     devMode: envBool('PDS_DEV_MODE'),
 
+    // branding
+    primaryColor: envStr('PDS_PRIMARY_COLOR'),
+    errorColor: envStr('PDS_ERROR_COLOR'),
+
     // database
     dataDirectory: envStr('PDS_DATA_DIRECTORY'),
     disableWalAutoCheckpoint: envBool('PDS_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT'),
@@ -97,6 +105,7 @@ export const readEnv = (): ServerEnvironment => {
     crawlers: envList('PDS_CRAWLERS'),
 
     // secrets
+    dpopSecret: envStr('PDS_DPOP_SECRET'),
     jwtSecret: envStr('PDS_JWT_SECRET'),
     adminPassword: envStr('PDS_ADMIN_PASSWORD'),
 
@@ -106,6 +115,9 @@ export const readEnv = (): ServerEnvironment => {
     plcRotationKeyK256PrivateKeyHex: envStr(
       'PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX',
     ),
+
+    // fetch
+    fetchDisableSsrfProtection: envBool('PDS_DISABLE_SSRF_PROTECTION'),
   }
 }
 
@@ -114,14 +126,22 @@ export type ServerEnvironment = {
   port?: number
   hostname?: string
   serviceDid?: string
+  serviceName?: string
   version?: string
+  homeUrl?: string
+  logoUrl?: string
   privacyPolicyUrl?: string
+  supportUrl?: string
   termsOfServiceUrl?: string
   contactEmailAddress?: string
   acceptingImports?: boolean
   blobUploadLimit?: number
   devMode?: boolean
 
+  // branding
+  primaryColor?: string
+  errorColor?: string
+
   // database
   dataDirectory?: string
   disableWalAutoCheckpoint?: boolean
@@ -203,10 +223,14 @@ export type ServerEnvironment = {
   crawlers?: string[]
 
   // secrets
+  dpopSecret?: string
   jwtSecret?: string
   adminPassword?: string
 
   // keys
   plcRotationKeyKmsKeyId?: string
   plcRotationKeyK256PrivateKeyHex?: string
+
+  // fetch
+  fetchDisableSsrfProtection?: boolean
 }

package/src/config/secrets.ts

@@ -27,6 +27,7 @@ export const envToSecrets = (env: ServerEnvironment): ServerSecrets => {
   }
 
   return {
+    dpopSecret: env.dpopSecret,
     jwtSecret: env.jwtSecret,
     adminPassword: env.adminPassword,
     plcRotationKey,
@@ -34,6 +35,7 @@ export const envToSecrets = (env: ServerEnvironment): ServerSecrets => {
 }
 
 export type ServerSecrets = {
+  dpopSecret?: string
   jwtSecret: string
   adminPassword: string
   plcRotationKey: SigningKeyKms | SigningKeyMemory

package/src/context.ts

@@ -12,12 +12,21 @@ import {
   RateLimiterOpts,
   createServiceAuthHeaders,
 } from '@atproto/xrpc-server'
+import {
+  JoseKey,
+  Fetch,
+  safeFetchWrap,
+  OAuthVerifier,
+} from '@atproto/oauth-provider'
+
 import { ServerConfig, ServerSecrets } from './config'
+import { PdsOAuthProvider } from './oauth/provider'
 import {
   AuthVerifier,
   createPublicKeyObject,
   createSecretKeyObject,
 } from './auth-verifier'
+import { fetchLogger } from './logger'
 import { ServerMailer } from './mailer'
 import { ModerationMailer } from './mailer/moderation'
 import { BlobStore } from '@atproto/repo'
@@ -50,6 +59,8 @@ export type AppContextOptions = {
   moderationAgent?: AtpAgent
   reportingAgent?: AtpAgent
   entrywayAgent?: AtpAgent
+  safeFetch: Fetch
+  authProvider?: PdsOAuthProvider
   authVerifier: AuthVerifier
   plcRotationKey: crypto.Keypair
   cfg: ServerConfig
@@ -74,7 +85,9 @@ export class AppContext {
   public moderationAgent: AtpAgent | undefined
   public reportingAgent: AtpAgent | undefined
   public entrywayAgent: AtpAgent | undefined
+  public safeFetch: Fetch
   public authVerifier: AuthVerifier
+  public authProvider?: PdsOAuthProvider
   public plcRotationKey: crypto.Keypair
   public cfg: ServerConfig
 
@@ -97,7 +110,9 @@ export class AppContext {
     this.moderationAgent = opts.moderationAgent
     this.reportingAgent = opts.reportingAgent
     this.entrywayAgent = opts.entrywayAgent
+    this.safeFetch = opts.safeFetch
     this.authVerifier = opts.authVerifier
+    this.authProvider = opts.authProvider
     this.plcRotationKey = opts.plcRotationKey
     this.cfg = opts.cfg
   }
@@ -206,7 +221,12 @@ export class AppContext {
       : undefined
 
     const jwtSecretKey = createSecretKeyObject(secrets.jwtSecret)
+    const jwtPublicKey = cfg.entryway
+      ? createPublicKeyObject(cfg.entryway.jwtPublicKeyHex)
+      : null
+
     const accountManager = new AccountManager(
+      backgroundQueue,
       cfg.db.accountDbLoc,
       jwtSecretKey,
       cfg.service.did,
@@ -214,20 +234,6 @@ export class AppContext {
     )
     await accountManager.migrateOrThrow()
 
-    const jwtKey = cfg.entryway
-      ? createPublicKeyObject(cfg.entryway.jwtPublicKeyHex)
-      : jwtSecretKey
-
-    const authVerifier = new AuthVerifier(accountManager, idResolver, {
-      jwtKey, // @TODO support multiple keys?
-      adminPass: secrets.adminPassword,
-      dids: {
-        pds: cfg.service.did,
-        entryway: cfg.entryway?.did,
-        modService: cfg.modService?.did,
-      },
-    })
-
     const plcRotationKey =
       secrets.plcRotationKey.provider === 'kms'
         ? await KmsKeypair.load({
@@ -250,6 +256,64 @@ export class AppContext {
       appviewCdnUrlPattern: cfg.bskyAppView?.cdnUrlPattern,
     })
 
+    // A fetch() function that protects against SSRF attacks, large responses &
+    // known bad domains. This function can safely be used to fetch user
+    // provided URLs (unless "disableSsrfProtection" is true, of course).
+    const safeFetch = safeFetchWrap({
+      allowHttp: cfg.fetch.disableSsrfProtection,
+      responseMaxSize: 512 * 1024, // 512kB
+      ssrfProtection: !cfg.fetch.disableSsrfProtection,
+      fetch: async (input, init) => {
+        const request = input instanceof Request ? input : null
+        const method = init?.method ?? request?.method ?? 'GET'
+        const uri = request?.url ?? String(input)
+        fetchLogger.debug({ method, uri }, 'fetch')
+        return globalThis.fetch(input, init)
+      },
+    })
+
+    const authProvider = cfg.oauth.provider
+      ? new PdsOAuthProvider({
+          issuer: cfg.oauth.issuer,
+          keyset: [
+            // Note: OpenID compatibility would require an RS256 private key in this list
+            await JoseKey.fromKeyLike(jwtSecretKey, undefined, 'HS256'),
+          ],
+          accountManager,
+          actorStore,
+          localViewer,
+          redis: redisScratch,
+          dpopSecret: secrets.dpopSecret,
+          customization: cfg.oauth.provider.customization,
+          safeFetch,
+        })
+      : undefined
+
+    const oauthVerifier: OAuthVerifier =
+      authProvider ?? // OAuthProvider extends OAuthVerifier
+      new OAuthVerifier({
+        issuer: cfg.oauth.issuer,
+        keyset: [await JoseKey.fromKeyLike(jwtPublicKey!, undefined, 'ES256K')],
+        dpopSecret: secrets.dpopSecret,
+        redis: redisScratch,
+      })
+
+    const authVerifier = new AuthVerifier(
+      accountManager,
+      idResolver,
+      oauthVerifier,
+      {
+        publicUrl: cfg.service.publicUrl,
+        jwtKey: jwtPublicKey ?? jwtSecretKey,
+        adminPass: secrets.adminPassword,
+        dids: {
+          pds: cfg.service.did,
+          entryway: cfg.entryway?.did,
+          modService: cfg.modService?.did,
+        },
+      },
+    )
+
     return new AppContext({
       actorStore,
       blobstore,
@@ -269,7 +333,9 @@ export class AppContext {
       moderationAgent,
       reportingAgent,
       entrywayAgent,
+      safeFetch,
       authVerifier,
+      authProvider,
       plcRotationKey,
       cfg,
       ...(overrides ?? {}),

package/src/db/cast.ts

@@ -0,0 +1,59 @@
+export type DateISO = `${string}T${string}Z`
+export const toDateISO = (date: Date): DateISO => date.toISOString() as DateISO
+export const fromDateISO = (date: DateISO): Date => new Date(date)
+
+export type Json = string
+export const toJson = (obj: unknown): Json => {
+  const json = JSON.stringify(obj)
+  if (json === undefined) throw new TypeError('Input not JSONifyable')
+  return json as Json
+}
+export const fromJson = <T>(json: Json): T => {
+  try {
+    return JSON.parse(json) as T
+  } catch (cause) {
+    throw new TypeError('Database contains invalid JSON', { cause })
+  }
+}
+
+export type JsonArray = `[${string}]`
+export const isJsonArray = (json: string): json is JsonArray =>
+  // Although the JSON in the DB should have been encoded using toJson,
+  // there should not be any leading or trailing whitespace. We will still trim
+  // the string to protect against any manual editing of the DB.
+  json.trimStart().startsWith('[') && json.trimEnd().endsWith(']')
+export function assertJsonArray(json: string): asserts json is JsonArray {
+  if (!isJsonArray(json)) throw new TypeError('Not an Array')
+}
+export const toJsonArray = (obj: readonly unknown[]): JsonArray => {
+  const json = toJson(obj)
+  assertJsonArray(json)
+  return json as JsonArray
+}
+export const fromJsonArray = <T>(json: JsonArray): T[] => {
+  assertJsonArray(json)
+  return fromJson(json) as T[]
+}
+
+export type JsonObject = `{${string}}`
+const isJsonObject = (json: string): json is JsonObject =>
+  // Although the JSON in the DB should have been encoded using toJson,
+  // there should not be any leading or trailing whitespace. We will still trim
+  // the string to protect against any manual editing of the DB.
+  json.trimStart().startsWith('{') && json.trimEnd().endsWith('}')
+function assertJsonObject(json: string): asserts json is JsonObject {
+  if (!isJsonObject(json)) throw new TypeError('Not an Object')
+}
+export const toJsonObject = (
+  obj: Readonly<Record<string, unknown>>,
+): JsonObject => {
+  const json = toJson(obj)
+  assertJsonObject(json)
+  return json as JsonObject
+}
+export const fromJsonObject = <T extends Record<string, unknown>>(
+  json: JsonObject,
+): T => {
+  assertJsonObject(json)
+  return fromJson(json) as T
+}

package/src/db/db.ts

@@ -51,7 +51,7 @@ export class Database<Schema> {
   }
 
   async transactionNoRetry<T>(
-    fn: (db: Database<Schema>) => Promise<T>,
+    fn: (db: Database<Schema>) => T | Promise<T>,
   ): Promise<T> {
     this.assertNotTransaction()
     const leakyTxPlugin = new LeakyTxPlugin()
@@ -60,22 +60,25 @@ export class Database<Schema> {
       .transaction()
       .execute(async (txn) => {
         const dbTxn = new Database(txn)
-        const txRes = await fn(dbTxn)
-          .catch(async (err) => {
-            leakyTxPlugin.endTx()
-            // ensure that all in-flight queries are flushed & the connection is open
-            await dbTxn.db.getExecutor().provideConnection(async () => {})
-            throw err
-          })
-          .finally(() => leakyTxPlugin.endTx())
-        const hooks = dbTxn.commitHooks
-        return { hooks, txRes }
+        try {
+          const txRes = await fn(dbTxn)
+          leakyTxPlugin.endTx()
+          const hooks = dbTxn.commitHooks
+          return { hooks, txRes }
+        } catch (err) {
+          leakyTxPlugin.endTx()
+          // ensure that all in-flight queries are flushed & the connection is open
+          await txn.getExecutor().provideConnection(async () => {})
+          throw err
+        }
       })
     hooks.map((hook) => hook())
     return txRes
   }
 
-  async transaction<T>(fn: (db: Database<Schema>) => Promise<T>): Promise<T> {
+  async transaction<T>(
+    fn: (db: Database<Schema>) => T | Promise<T>,
+  ): Promise<T> {
     return retrySqlite(() => this.transactionNoRetry(fn))
   }
 

package/src/db/index.ts

@@ -1,3 +1,4 @@
 export * from './db'
+export * from './cast'
 export * from './migrator'
 export * from './util'

package/src/error.ts

@@ -1,12 +1,19 @@
 import { XRPCError } from '@atproto/xrpc-server'
 import { ErrorRequestHandler } from 'express'
 import { httpLogger as log } from './logger'
+import { OAuthError } from '@atproto/oauth-provider'
 
 export const handler: ErrorRequestHandler = (err, _req, res, next) => {
   log.error(err, 'unexpected internal server error')
   if (res.headersSent) {
     return next(err)
   }
+
+  if (err instanceof OAuthError) {
+    res.status(err.status).json(err.toJSON())
+    return
+  }
+
   const serverError = XRPCError.fromError(err)
   res.status(serverError.type).json(serverError.payload)
 }

package/src/index.ts

@@ -11,6 +11,7 @@ import events from 'events'
 import { Options as XrpcServerOptions } from '@atproto/xrpc-server'
 import { DAY, HOUR, MINUTE, SECOND } from '@atproto/common'
 import API from './api'
+import * as authRoutes from './auth-routes'
 import * as basicRoutes from './basic-routes'
 import * as wellKnown from './well-known'
 import * as error from './error'
@@ -99,6 +100,7 @@ export class PDS {
 
     server = API(server, ctx)
 
+    app.use(authRoutes.createRouter(ctx))
     app.use(basicRoutes.createRouter(ctx))
     app.use(wellKnown.createRouter(ctx))
     app.use(server.xrpc.router)

package/src/logger.ts

@@ -1,8 +1,6 @@
-import pino from 'pino'
+import { stdSerializers } from 'pino'
 import pinoHttp from 'pino-http'
 import { subsystemLogger } from '@atproto/common'
-import * as jose from 'jose'
-import { parseBasicAuth } from './auth-verifier'
 
 export const dbLogger = subsystemLogger('pds:db')
 export const didCacheLogger = subsystemLogger('pds:did-cache')
@@ -13,44 +11,91 @@ export const mailerLogger = subsystemLogger('pds:mailer')
 export const labelerLogger = subsystemLogger('pds:labeler')
 export const crawlerLogger = subsystemLogger('pds:crawler')
 export const httpLogger = subsystemLogger('pds')
+export const fetchLogger = subsystemLogger('pds:fetch')
+export const oauthLogger = subsystemLogger('pds:oauth')
 
 export const loggerMiddleware = pinoHttp({
   logger: httpLogger,
   serializers: {
-    err: (err) => {
-      return {
-        code: err?.code,
-        message: err?.message,
-      }
-    },
-    req: (req) => {
-      const serialized = pino.stdSerializers.req(req)
-      const authHeader = serialized.headers.authorization || ''
-      let auth: string | undefined = undefined
-      if (authHeader.startsWith('Bearer ')) {
-        const token = authHeader.slice('Bearer '.length)
-        const { sub } = jose.decodeJwt(token)
-        if (sub) {
-          auth = 'Bearer ' + sub
-        } else {
-          auth = 'Bearer Invalid'
-        }
-      }
-      if (authHeader.startsWith('Basic ')) {
-        const parsed = parseBasicAuth(authHeader)
-        if (!parsed) {
-          auth = 'Basic Invalid'
-        } else {
-          auth = 'Basic ' + parsed.username
-        }
-      }
-      return {
-        ...serialized,
-        headers: {
-          ...serialized.headers,
-          authorization: auth,
-        },
-      }
-    },
+    err: errSerializer,
+    req: reqSerializer,
   },
 })
+
+function errSerializer(err: any) {
+  return {
+    code: err?.code,
+    message: err?.message,
+  }
+}
+
+function reqSerializer(req: any) {
+  const serialized = stdSerializers.req(req)
+  serialized.headers = obfuscateHeaders(serialized.headers)
+  return serialized
+}
+
+function obfuscateHeaders(headers: Record<string, string>) {
+  const obfuscatedHeaders: Record<string, string> = {}
+  for (const key in headers) {
+    if (key.toLowerCase() === 'authorization') {
+      obfuscatedHeaders[key] = obfuscateAuthHeader(headers[key])
+    } else if (key.toLowerCase() === 'dpop') {
+      obfuscatedHeaders[key] = obfuscateJws(headers[key]) || 'Invalid'
+    } else {
+      obfuscatedHeaders[key] = headers[key]
+    }
+  }
+  return obfuscatedHeaders
+}
+
+function obfuscateAuthHeader(authHeader: string): string {
+  // This is a hot path (runs on every request). Avoid using split() or regex.
+
+  const spaceIdx = authHeader.indexOf(' ')
+  if (spaceIdx === -1) return 'Invalid'
+
+  const type = authHeader.slice(0, spaceIdx)
+  switch (type.toLowerCase()) {
+    case 'bearer':
+      return `${type} ${obfuscateBearer(authHeader.slice(spaceIdx + 1))}`
+    case 'dpop':
+      return `${type} ${obfuscateJws(authHeader.slice(spaceIdx + 1)) || 'Invalid'}`
+    case 'basic':
+      return `${type} ${obfuscateBasic(authHeader.slice(spaceIdx + 1)) || 'Invalid'}`
+    default:
+      return `Invalid`
+  }
+}
+
+function obfuscateBasic(token: string): null | string {
+  if (!token) return null
+  const buffer = Buffer.from(token, 'base64')
+  if (!buffer.length) return null // Buffer.from will silently ignore invalid base64 chars
+  const authHeader = buffer.toString('utf8')
+  const colIdx = authHeader.indexOf(':')
+  if (colIdx === -1) return null
+  const username = authHeader.slice(0, colIdx)
+  return `${username}:***`
+}
+
+function obfuscateBearer(token: string): string {
+  return obfuscateJws(token) || obfuscateToken(token)
+}
+
+function obfuscateToken(token: string): string {
+  return token ? '***' : ''
+}
+
+function obfuscateJws(token: string): null | string {
+  const firstDot = token.indexOf('.')
+  if (firstDot === -1) return null
+
+  const secondDot = token.indexOf('.', firstDot + 1)
+  if (secondDot === -1) return null
+
+  if (token.indexOf('.', secondDot + 1) !== -1) return null
+
+  // Strip the signature
+  return token.slice(0, secondDot) + '.obfuscated'
+}

package/src/oauth/detailed-account-store.ts

@@ -0,0 +1,96 @@
+import {
+  AccountInfo,
+  AccountStore,
+  DeviceId,
+  LoginCredentials,
+} from '@atproto/oauth-provider'
+
+import { AccountManager } from '../account-manager/index'
+import { ActorStore } from '../actor-store/index'
+import { ProfileViewBasic } from '../lexicon/types/app/bsky/actor/defs'
+import { LocalViewerCreator } from '../read-after-write/index'
+
+/**
+ * Although the {@link AccountManager} class implements the {@link AccountStore}
+ * interface, the accounts it returns do not contain any profile information
+ * (display name, avatar, etc). This is due to the fact that the account manager
+ * does not have access to the account's repos. The {@link DetailedAccountStore}
+ * is a wrapper around the {@link AccountManager} that enriches the accounts
+ * with profile information using the account's repos through the
+ * {@link ActorStore}.
+ */
+export class DetailedAccountStore implements AccountStore {
+  constructor(
+    private accountManager: AccountManager,
+    private actorStore: ActorStore,
+    private localViewer: LocalViewerCreator,
+  ) {}
+
+  private async getProfile(did: string): Promise<ProfileViewBasic | null> {
+    // TODO: Should we cache this?
+    return this.actorStore.read(did, async (actorStoreReader) => {
+      const localViewer = this.localViewer(actorStoreReader)
+      return localViewer.getProfileBasic()
+    })
+  }
+
+  private async enrichAccountInfo(
+    accountInfo: AccountInfo,
+  ): Promise<AccountInfo> {
+    const { account } = accountInfo
+    if (!account.picture || !account.name) {
+      const profile = await this.getProfile(account.sub)
+      if (profile) {
+        account.picture ||= profile.avatar
+        account.name ||= profile.displayName
+      }
+    }
+
+    return accountInfo
+  }
+
+  async authenticateAccount(
+    credentials: LoginCredentials,
+    deviceId: DeviceId,
+  ): Promise<AccountInfo | null> {
+    const accountInfo = await this.accountManager.authenticateAccount(
+      credentials,
+      deviceId,
+    )
+    if (!accountInfo) return null
+    return this.enrichAccountInfo(accountInfo)
+  }
+
+  async addAuthorizedClient(
+    deviceId: DeviceId,
+    sub: string,
+    clientId: string,
+  ): Promise<void> {
+    return this.accountManager.addAuthorizedClient(deviceId, sub, clientId)
+  }
+
+  async getDeviceAccount(
+    deviceId: DeviceId,
+    sub: string,
+  ): Promise<AccountInfo | null> {
+    const accountInfo = await this.accountManager.getDeviceAccount(
+      deviceId,
+      sub,
+    )
+    if (!accountInfo) return null
+    return this.enrichAccountInfo(accountInfo)
+  }
+
+  async listDeviceAccounts(deviceId: DeviceId): Promise<AccountInfo[]> {
+    const accountInfos = await this.accountManager.listDeviceAccounts(deviceId)
+    return Promise.all(
+      accountInfos.map(async (accountInfo) =>
+        this.enrichAccountInfo(accountInfo),
+      ),
+    )
+  }
+
+  async removeDeviceAccount(deviceId: DeviceId, sub: string): Promise<void> {
+    return this.accountManager.removeDeviceAccount(deviceId, sub)
+  }
+}