@atproto/pds 0.4.80 → 0.4.82

package/src/pipethrough.ts

@@ -3,6 +3,7 @@ import { IncomingHttpHeaders, ServerResponse } from 'node:http'
 import { PassThrough, Readable } from 'node:stream'
 import { Dispatcher } from 'undici'
 
+import { buildProxiedContentEncoding } from '@atproto-labs/xrpc-utils'
 import {
   decodeStream,
   getServiceEndpoint,
@@ -101,20 +102,6 @@ export const proxyHandler = (ctx: AppContext): CatchallHandler => {
   }
 }
 
-const ACCEPT_ENCODING_COMPRESSED = [
-  ['gzip', { q: 1.0 }],
-  ['deflate', { q: 0.9 }],
-  ['br', { q: 0.8 }],
-  ['identity', { q: 0.1 }],
-] as const satisfies Accept[]
-
-const ACCEPT_ENCODING_UNCOMPRESSED = [
-  ['identity', { q: 1.0 }],
-  ['gzip', { q: 0.3 }],
-  ['deflate', { q: 0.2 }],
-  ['br', { q: 0.1 }],
-] as const satisfies Accept[]
-
 export type PipethroughOptions = {
   /**
    * Specify the issuer (requester) for service auth. If not provided, no
@@ -176,11 +163,9 @@ export async function pipethrough(
       // upstream server for an encoding that both the requester and the PDS can
       // understand. Since we might have to do the decoding ourselves, we will
       // use our own preferences (and weight) to negotiate the encoding.
-      'accept-encoding': negotiateContentEncoding(
+      'accept-encoding': buildProxiedContentEncoding(
         req.headers['accept-encoding'],
-        ctx.cfg.proxy.preferCompressed
-          ? ACCEPT_ENCODING_COMPRESSED
-          : ACCEPT_ENCODING_UNCOMPRESSED,
+        ctx.cfg.proxy.preferCompressed,
       ),
 
       authorization: options?.iss
@@ -224,7 +209,7 @@ export async function parseProxyInfo(
 
 export const parseProxyHeader = async (
   // Using subset of AppContext for testing purposes
-  ctx: Pick<AppContext, 'idResolver'>,
+  ctx: Pick<AppContext, 'cfg' | 'idResolver'>,
   proxyTo: string,
 ): Promise<{ did: string; url: string }> => {
   // /!\ Hot path
@@ -261,6 +246,14 @@ export const parseProxyHeader = async (
     throw new InvalidRequestError('could not resolve proxy did service url')
   }
 
+  // Special case a configured appview, while still proxying correctly any other appview
+  if (
+    ctx.cfg.bskyAppView &&
+    proxyTo === `${ctx.cfg.bskyAppView.did}#bsky_appview`
+  ) {
+    return { did, url: ctx.cfg.bskyAppView.url }
+  }
+
   return { did, url }
 }
 
@@ -367,116 +360,6 @@ function handleUpstreamRequestError(
 // Request parsing/forwarding
 // -------------------
 
-type AcceptFlags = { q: number }
-type Accept = [name: string, flags: AcceptFlags]
-
-// accept-encoding defaults to "identity with lowest priority"
-const ACCEPT_ENC_DEFAULT = ['identity', { q: 0.001 }] as const satisfies Accept
-const ACCEPT_FORBID_STAR = ['*', { q: 0 }] as const satisfies Accept
-
-function negotiateContentEncoding(
-  acceptHeader: undefined | string | string[],
-  preferences: readonly Accept[],
-): string {
-  const acceptMap = Object.fromEntries<undefined | AcceptFlags>(
-    parseAcceptEncoding(acceptHeader),
-  )
-
-  // Make sure the default (identity) is covered by the preferences
-  if (!preferences.some(coversIdentityAccept)) {
-    preferences = [...preferences, ACCEPT_ENC_DEFAULT]
-  }
-
-  const common = preferences.filter(([name]) => {
-    const acceptQ = (acceptMap[name] ?? acceptMap['*'])?.q
-    // Per HTTP/1.1, "identity" is always acceptable unless explicitly rejected
-    if (name === 'identity') {
-      return acceptQ == null || acceptQ > 0
-    } else {
-      return acceptQ != null && acceptQ > 0
-    }
-  })
-
-  // Since "identity" was present in the preferences, a missing "identity" in
-  // the common array means that the client explicitly rejected it. Let's reflect
-  // this by adding it to the common array.
-  if (!common.some(coversIdentityAccept)) {
-    common.push(ACCEPT_FORBID_STAR)
-  }
-
-  // If no common encodings are acceptable, throw a 406 Not Acceptable error
-  if (!common.some(isAllowedAccept)) {
-    throw new XRPCServerError(
-      ResponseType.NotAcceptable,
-      'this service does not support any of the requested encodings',
-    )
-  }
-
-  return formatAcceptHeader(common as [Accept, ...Accept[]])
-}
-
-function coversIdentityAccept([name]: Accept): boolean {
-  return name === 'identity' || name === '*'
-}
-
-function isAllowedAccept([, flags]: Accept): boolean {
-  return flags.q > 0
-}
-
-/**
- * @see {@link https://developer.mozilla.org/en-US/docs/Glossary/Quality_values}
- */
-function formatAcceptHeader(accept: readonly [Accept, ...Accept[]]): string {
-  return accept.map(formatAcceptPart).join(',')
-}
-
-function formatAcceptPart([name, flags]: Accept): string {
-  return `${name};q=${flags.q}`
-}
-
-function parseAcceptEncoding(
-  acceptEncodings: undefined | string | string[],
-): Accept[] {
-  if (!acceptEncodings?.length) return []
-
-  return Array.isArray(acceptEncodings)
-    ? acceptEncodings.flatMap(parseAcceptEncoding)
-    : acceptEncodings.split(',').map(parseAcceptEncodingDefinition)
-}
-
-function parseAcceptEncodingDefinition(def: string): Accept {
-  const { length, 0: encoding, 1: params } = def.trim().split(';', 3)
-
-  if (length > 2) {
-    throw new InvalidRequestError(`Invalid accept-encoding: "${def}"`)
-  }
-
-  if (!encoding || encoding.includes('=')) {
-    throw new InvalidRequestError(`Invalid accept-encoding: "${def}"`)
-  }
-
-  const flags = { q: 1 }
-  if (length === 2) {
-    const { length, 0: key, 1: value } = params.split('=', 3)
-    if (length !== 2) {
-      throw new InvalidRequestError(`Invalid accept-encoding: "${def}"`)
-    }
-
-    if (key === 'q' || key === 'Q') {
-      const q = parseFloat(value)
-      if (q === 0 || (Number.isFinite(q) && q <= 1 && q >= 0.001)) {
-        flags.q = q
-      } else {
-        throw new InvalidRequestError(`Invalid accept-encoding: "${def}"`)
-      }
-    } else {
-      throw new InvalidRequestError(`Invalid accept-encoding: "${def}"`)
-    }
-  }
-
-  return [encoding.toLowerCase(), flags]
-}
-
 export function isJsonContentType(contentType?: string): boolean | undefined {
   if (!contentType) return undefined
   return /application\/(?:\w+\+)?json/i.test(contentType)

package/tests/_util.ts

@@ -1,7 +1,10 @@
+import { lexToJson } from '@atproto/lexicon'
 import { AtUri } from '@atproto/syntax'
+import { type Express } from 'express'
 import { CID } from 'multiformats/cid'
+import { Server } from 'node:http'
+import { AddressInfo } from 'node:net'
 import { FeedViewPost } from '../src/lexicon/types/app/bsky/feed/defs'
-import { lexToJson } from '@atproto/lexicon'
 
 // Swap out identifiers and dates with stable
 // values for the purpose of snapshot testing
@@ -156,3 +159,46 @@ export const paginateAll = async <T extends { cursor?: string }>(
   } while (cursor && results.length < limit)
   return results
 }
+
+export async function startServer(app: Express) {
+  return new Promise<{
+    origin: string
+    server: Server
+    stop: () => Promise<void>
+  }>((resolve, reject) => {
+    const onListen = () => {
+      const port = (server.address() as AddressInfo).port
+      resolve({
+        server,
+        origin: `http://localhost:${port}`,
+        stop: () => stopServer(server),
+      })
+      cleanup()
+    }
+    const onError = (err: Error) => {
+      reject(err)
+      cleanup()
+    }
+    const cleanup = () => {
+      server.removeListener('listening', onListen)
+      server.removeListener('error', onError)
+    }
+
+    const server = app
+      .listen(0)
+      .once('listening', onListen)
+      .once('error', onError)
+  })
+}
+
+export async function stopServer(server: Server) {
+  return new Promise<void>((resolve, reject) => {
+    server.close((err) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve()
+      }
+    })
+  })
+}

package/tests/proxied/proxy-header.test.ts

@@ -1,14 +1,13 @@
-import http from 'node:http'
-import assert from 'node:assert'
-import express from 'express'
-import axios from 'axios'
-import * as plc from '@did-plc/lib'
-import { SeedClient, TestNetworkNoAppView, usersSeed } from '@atproto/dev-env'
 import { Keypair } from '@atproto/crypto'
+import { SeedClient, TestNetworkNoAppView, usersSeed } from '@atproto/dev-env'
 import { verifyJwt } from '@atproto/xrpc-server'
-import { parseProxyHeader } from '../../src/pipethrough'
+import * as plc from '@did-plc/lib'
+import express from 'express'
+import assert from 'node:assert'
 import { once } from 'node:events'
+import http from 'node:http'
 import { AddressInfo } from 'node:net'
+import { parseProxyHeader } from '../../src/pipethrough'
 
 describe('proxy header', () => {
   let network: TestNetworkNoAppView
@@ -40,19 +39,6 @@ describe('proxy header', () => {
     await network.close()
   })
 
-  const assertAxiosErr = async (promise: Promise<unknown>, msg: string) => {
-    try {
-      await promise
-    } catch (err) {
-      if (!axios.isAxiosError(err)) {
-        throw err
-      }
-      expect(err.response?.data?.['message']).toEqual(msg)
-      return
-    }
-    throw new Error('no error thrown')
-  }
-
   it('parses proxy header', async () => {
     expect(parseProxyHeader(network.pds.ctx, `#atproto_test`)).rejects.toThrow(
       'no did specified in proxy header',
@@ -84,7 +70,7 @@ describe('proxy header', () => {
 
   it('proxies requests based on header', async () => {
     const path = `/xrpc/app.bsky.actor.getProfile?actor=${alice}`
-    await axios.get(`${network.pds.url}${path}`, {
+    await fetch(`${network.pds.url}${path}`, {
       headers: {
         ...sc.getHeaders(alice),
         'atproto-proxy': `${proxyServer.did}#atproto_test`,
@@ -106,37 +92,49 @@ describe('proxy header', () => {
 
   it('fails on a non-existant did', async () => {
     const path = `/xrpc/app.bsky.actor.getProfile?actor=${alice}`
-    const attempt = axios.get(`${network.pds.url}${path}`, {
+    const response = await fetch(`${network.pds.url}${path}`, {
       headers: {
         ...sc.getHeaders(alice),
         'atproto-proxy': `did:plc:12345678123456781234578#atproto_test`,
       },
     })
-    await assertAxiosErr(attempt, 'could not resolve proxy did')
+
+    await expect(response.json()).resolves.toMatchObject({
+      message: 'could not resolve proxy did',
+    })
+
     expect(proxyServer.requests.length).toBe(1)
   })
 
   it('fails when a service is not specified', async () => {
     const path = `/xrpc/app.bsky.actor.getProfile?actor=${alice}`
-    const attempt = axios.get(`${network.pds.url}${path}`, {
+    const response = await fetch(`${network.pds.url}${path}`, {
       headers: {
         ...sc.getHeaders(alice),
         'atproto-proxy': proxyServer.did,
       },
     })
-    await assertAxiosErr(attempt, 'no service id specified in proxy header')
+
+    await expect(response.json()).resolves.toMatchObject({
+      message: 'no service id specified in proxy header',
+    })
+
     expect(proxyServer.requests.length).toBe(1)
   })
 
   it('fails on a non-existant service', async () => {
     const path = `/xrpc/app.bsky.actor.getProfile?actor=${alice}`
-    const attempt = axios.get(`${network.pds.url}${path}`, {
+    const response = await fetch(`${network.pds.url}${path}`, {
       headers: {
         ...sc.getHeaders(alice),
         'atproto-proxy': `${proxyServer.did}#atproto_bad`,
       },
     })
-    await assertAxiosErr(attempt, 'could not resolve proxy did service url')
+
+    await expect(response.json()).resolves.toMatchObject({
+      message: 'could not resolve proxy did service url',
+    })
+
     expect(proxyServer.requests.length).toBe(1)
   })
 })

package/tests/server.test.ts

@@ -1,11 +1,12 @@
-import { AddressInfo } from 'net'
-import express from 'express'
-import axios, { AxiosError } from 'axios'
-import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env'
 import { AtpAgent, AtUri } from '@atproto/api'
+import { randomStr } from '@atproto/crypto'
+import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env'
+import express from 'express'
+import { finished } from 'node:stream/promises'
+import { request } from 'undici'
 import { handler as errorHandler } from '../src/error'
+import { startServer } from './_util'
 import basicSeed from './seeds/basic'
-import { randomStr } from '@atproto/crypto'
 
 describe('server', () => {
   let network: TestNetworkNoAppView
@@ -31,53 +32,42 @@ describe('server', () => {
   })
 
   it('preserves 404s.', async () => {
-    const promise = axios.get(`${network.pds.url}/unknown`)
-    await expect(promise).rejects.toThrow('failed with status code 404')
+    const res = await fetch(`${network.pds.url}/unknown`)
+    expect(res.status).toEqual(404)
   })
 
   it('error handler turns unknown errors into 500s.', async () => {
     const app = express()
-    app.get('/oops', () => {
-      throw new Error('Oops!')
-    })
-    app.use(errorHandler)
-    const srv = app.listen()
-    const port = (srv.address() as AddressInfo).port
-    const promise = axios.get(`http://localhost:${port}/oops`)
-    await expect(promise).rejects.toThrow('failed with status code 500')
-    srv.close()
+      .get('/oops', () => {
+        throw new Error('Oops!')
+      })
+      .use(errorHandler)
+
+    const { origin, stop } = await startServer(app)
     try {
-      await promise
-    } catch (err: unknown) {
-      const axiosError = err as AxiosError
-      expect(axiosError.response?.status).toEqual(500)
-      expect(axiosError.response?.data).toEqual({
+      const res = await fetch(new URL(`/oops`, origin))
+      expect(res.status).toEqual(500)
+      await expect(res.json()).resolves.toEqual({
         error: 'InternalServerError',
         message: 'Internal Server Error',
       })
+    } finally {
+      await stop()
     }
   })
 
   it('limits size of json input.', async () => {
-    let error: AxiosError
-    try {
-      await axios.post(
-        `${network.pds.url}/xrpc/com.atproto.repo.createRecord`,
-        {
-          data: 'x'.repeat(150 * 1024), // 150kb
-        },
-        { headers: sc.getHeaders(alice) },
-      )
-      throw new Error('Request should have failed')
-    } catch (err) {
-      if (axios.isAxiosError(err)) {
-        error = err
-      } else {
-        throw err
-      }
-    }
-    expect(error.response?.status).toEqual(413)
-    expect(error.response?.data).toEqual({
+    const res = await fetch(
+      `${network.pds.url}/xrpc/com.atproto.repo.createRecord`,
+      {
+        method: 'POST',
+        body: 'x'.repeat(150 * 1024), // 150kb
+        headers: sc.getHeaders(alice),
+      },
+    )
+
+    expect(res.status).toEqual(413)
+    await expect(res.json()).resolves.toEqual({
       error: 'PayloadTooLargeError',
       message: 'request entity too large',
     })
@@ -102,56 +92,53 @@ describe('server', () => {
     )
     const uri = new AtUri(createRes.data.uri)
 
-    const res = await axios.get(
+    const res = await request(
       `${network.pds.url}/xrpc/com.atproto.repo.getRecord?repo=${uri.host}&collection=${uri.collection}&rkey=${uri.rkey}`,
       {
-        decompress: false,
         headers: { ...sc.getHeaders(alice), 'accept-encoding': 'gzip' },
       },
     )
 
+    await finished(res.body.resume())
+
     expect(res.headers['content-encoding']).toEqual('gzip')
   })
 
   it('compresses large car file responses', async () => {
-    const res = await axios.get(
+    const res = await request(
       `${network.pds.url}/xrpc/com.atproto.sync.getRepo?did=${alice}`,
-      { decompress: false, headers: { 'accept-encoding': 'gzip' } },
+      { headers: { 'accept-encoding': 'gzip' } },
     )
+
+    await finished(res.body.resume())
+
     expect(res.headers['content-encoding']).toEqual('gzip')
   })
 
   it('does not compress small payloads', async () => {
-    const res = await axios.get(`${network.pds.url}/xrpc/_health`, {
-      decompress: false,
+    const res = await request(`${network.pds.url}/xrpc/_health`, {
       headers: { 'accept-encoding': 'gzip' },
     })
+
+    await finished(res.body.resume())
+
     expect(res.headers['content-encoding']).toBeUndefined()
   })
 
   it('healthcheck succeeds when database is available.', async () => {
-    const { data, status } = await axios.get(`${network.pds.url}/xrpc/_health`)
-    expect(status).toEqual(200)
-    expect(data).toEqual({ version: '0.0.0' })
+    const res = await fetch(`${network.pds.url}/xrpc/_health`)
+    expect(res.status).toEqual(200)
+    await expect(res.json()).resolves.toEqual({ version: '0.0.0' })
   })
 
   // @TODO this is hanging for some unknown reason
   it.skip('healthcheck fails when database is unavailable.', async () => {
     await network.pds.ctx.accountManager.db.close()
-    let error: AxiosError
-    try {
-      await axios.get(`${network.pds.url}/xrpc/_health`)
-      throw new Error('Healthcheck should have failed')
-    } catch (err) {
-      if (axios.isAxiosError(err)) {
-        error = err
-      } else {
-        throw err
-      }
-    }
-    expect(error.response?.status).toEqual(503)
-    expect(error.response?.data).toEqual({
-      version: '0.0.0',
+
+    const response = await fetch(`${network.pds.url}/xrpc/_health`)
+    expect(response.status).toEqual(503)
+    await expect(response.json()).resolves.toEqual({
+      version: 'unknown',
       error: 'Service Unavailable',
     })
   })