@atproto/pds 0.5.6 → 0.5.8

package/src/pipethrough.ts

@@ -1,7 +1,7 @@
 import { IncomingHttpHeaders, ServerResponse } from 'node:http'
 import { PassThrough, Readable, finished } from 'node:stream'
 import { Request } from 'express'
-import { Dispatcher } from 'undici'
+import { Agent, Dispatcher, Pool, interceptors } from 'undici'
 import {
   decodeStream,
   getServiceEndpoint,
@@ -20,12 +20,49 @@ import {
   excludeErrorResult,
   parseReqNsid,
 } from '@atproto/xrpc-server'
+import { isUnicastIp, unicastLookup } from '@atproto-labs/fetch-node'
 import { buildProxiedContentEncoding } from '@atproto-labs/xrpc-utils'
 import { isAccessPrivileged } from './auth-scope.js'
+import { ProxyConfig } from './config/config.js'
 import { AppContext } from './context.js'
 import { chat, com, tools } from './lexicons/index.js'
 import { httpLogger } from './logger.js'
 
+export function buildProxyAgent(cfg: ProxyConfig): Dispatcher {
+  const agent = new Agent({
+    allowH2: cfg.allowHTTP2,
+    headersTimeout: cfg.headersTimeout,
+    maxResponseSize: cfg.maxResponseSize,
+    bodyTimeout: cfg.bodyTimeout,
+    factory: cfg.disableSsrfProtection
+      ? undefined
+      : (origin, opts) => {
+          const { protocol, hostname } =
+            origin instanceof URL ? origin : new URL(origin)
+          if (protocol !== 'https:') {
+            throw new Error(`Forbidden protocol "${protocol}"`)
+          }
+          if (isUnicastIp(hostname) === false) {
+            throw new Error('Hostname resolved to non-unicast address')
+          }
+          return new Pool(origin, opts)
+        },
+    connect: {
+      lookup: cfg.disableSsrfProtection ? undefined : unicastLookup,
+    },
+  })
+
+  return agent.compose(
+    cfg.maxRetries > 0
+      ? interceptors.retry({
+          statusCodes: [], // Only retry on socket errors
+          methods: ['GET', 'HEAD'],
+          maxRetries: cfg.maxRetries,
+        })
+      : (dispatch) => dispatch,
+  )
+}
+
 export const proxyHandler = (ctx: AppContext): CatchallHandler => {
   const performAuth = ctx.authVerifier.authorization<RpcPermissionMatch>({
     authorize: (permissions, { params }) => permissions.assertRpc(params),

package/tests/_util.ts

@@ -1,6 +1,7 @@
-import { Server } from 'node:http'
+import { RequestListener, createServer } from 'node:http'
 import { AddressInfo } from 'node:net'
-import { type Express } from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { ToolsOzoneModerationDefs, lexToJson } from '@atproto/api'
 import { isCidString } from '@atproto/lex'
 import { isCid, isPlainObject } from '@atproto/lex-data'
@@ -146,19 +147,11 @@ export const paginateAll = async <T extends { cursor?: string }>(
   return results
 }
 
-export async function startServer(app: Express) {
-  return new Promise<{
-    origin: string
-    server: Server
-    stop: () => Promise<void>
-  }>((resolve, reject) => {
+export async function startServer(listener: RequestListener) {
+  return new Promise<AsyncDisposable & { port: number }>((resolve, reject) => {
     const onListen = () => {
       const port = (server.address() as AddressInfo).port
-      resolve({
-        server,
-        origin: `http://localhost:${port}`,
-        stop: () => stopServer(server),
-      })
+      resolve({ port, [Symbol.asyncDispose]: () => terminator.terminate() })
       cleanup()
     }
     const onError = (err: Error) => {
@@ -170,22 +163,12 @@ export async function startServer(app: Express) {
       server.removeListener('error', onError)
     }
 
-    const server = app
+    const server = createServer(listener)
       .listen(0)
       .once('listening', onListen)
       .once('error', onError)
-  })
-}
 
-export async function stopServer(server: Server) {
-  return new Promise<void>((resolve, reject) => {
-    server.close((err) => {
-      if (err) {
-        reject(err)
-      } else {
-        resolve()
-      }
-    })
+    const terminator = httpTerminator.createHttpTerminator({ server })
   })
 }
 

package/tests/account-deactivation.test.ts

@@ -5,6 +5,7 @@ import {
   TestNetworkNoAppView,
   basicSeed,
 } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 
 describe('account deactivation', () => {
   let network: TestNetworkNoAppView
@@ -12,7 +13,7 @@ describe('account deactivation', () => {
   let sc: SeedClient
   let agent: AtpAgent
 
-  let alice: string
+  let alice: DidString
   let aliceAvatar: ImageRef
 
   beforeAll(async () => {
@@ -39,7 +40,7 @@ describe('account deactivation', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('deactivates account', async () => {

package/tests/account-deletion.test.ts

@@ -57,7 +57,7 @@ describe('account deletion', () => {
 
   afterAll(async () => {
     mailer.transporter.sendMail = _origSendMail
-    await network.close()
+    await network?.close()
   })
 
   const getMailFrom = async (promise): Promise<Mail.Options> => {

package/tests/account-migration.test.ts

@@ -71,8 +71,8 @@ describe('account migration', () => {
   })
 
   afterAll(async () => {
-    await newPds.close()
-    await network.close()
+    await newPds?.close()
+    await network?.close()
   })
 
   it('migrates an account', async () => {

package/tests/account-status.test.ts

@@ -19,7 +19,7 @@ describe('account-status', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('takedown + activation triggers an error', async () => {

package/tests/account.test.ts

@@ -60,7 +60,7 @@ describe('account', () => {
 
   afterAll(async () => {
     mailer.transporter.sendMail = _origSendMail
-    await network.close()
+    await network?.close()
   })
 
   it('serves the accounts system config', async () => {

package/tests/app-passwords.test.ts

@@ -24,7 +24,7 @@ describe('app_passwords', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   let appPass: string

package/tests/auth.test.ts

@@ -16,7 +16,7 @@ describe('auth', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const createAccount = async (info) => {

package/tests/blob-deletes.test.ts

@@ -1,5 +1,6 @@
 import { AtpAgent, BlobRef } from '@atproto/api'
 import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 import { AppContext } from '../src/index.js'
 
 describe('blob deletes', () => {
@@ -9,8 +10,8 @@ describe('blob deletes', () => {
 
   let ctx: AppContext
 
-  let alice: string
-  let bob: string
+  let alice: DidString
+  let bob: DidString
 
   beforeAll(async () => {
     network = await TestNetworkNoAppView.create({
@@ -35,7 +36,7 @@ describe('blob deletes', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const getDbBlobsForDid = (did: string) => {
@@ -184,7 +185,7 @@ describe('blob deletes', () => {
 
 async function updateProfile(
   sc: SeedClient,
-  did: string,
+  did: DidString,
   avatar?: BlobRef,
   banner?: BlobRef,
 ) {

package/tests/create-post.test.ts

@@ -25,7 +25,7 @@ describe('pds posts record creation', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('allows for creating posts with tags', async () => {

package/tests/crud.test.ts

@@ -47,7 +47,7 @@ describe('crud operations', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('registers users', async () => {

package/tests/db.test.ts

@@ -14,7 +14,7 @@ describe('db', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('commits changes', async () => {

package/tests/email-confirmation.test.ts

@@ -42,7 +42,7 @@ describe('email confirmation', () => {
 
   afterAll(async () => {
     mailer.transporter.sendMail = _origSendMail
-    await network.close()
+    await network?.close()
   })
 
   const getMailFrom = async (promise): Promise<SendMailOptions> => {

package/tests/file-uploads.test.ts

@@ -41,8 +41,8 @@ describe('file uploads', () => {
   })
 
   afterAll(async () => {
-    aliceDb.close()
-    await network.close()
+    aliceDb?.close()
+    await network?.close()
   })
 
   let smallBlob: TypedBlobRef

package/tests/get-service-auth.test.ts

@@ -23,7 +23,7 @@ describe('com.atproto.server.getServiceAuth', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('issues a token whose aud matches a bare-DID input', async () => {

package/tests/handles.test.ts

@@ -60,7 +60,7 @@ describe('handles', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const getHandleFromDb = async (

package/tests/invite-codes.test.ts

@@ -26,7 +26,7 @@ describe('account', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('describes the fact that invites are required', async () => {

package/tests/invites-admin.test.ts

@@ -1,6 +1,7 @@
 import { AtpAgent } from '@atproto/api'
 import { randomStr } from '@atproto/crypto'
 import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 
 describe('pds admin invite views', () => {
   let network: TestNetworkNoAppView
@@ -20,12 +21,12 @@ describe('pds admin invite views', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
-  let alice: string
-  let bob: string
-  let carol: string
+  let alice: DidString
+  let bob: DidString
+  let carol: DidString
 
   beforeAll(async () => {
     const adminCode = await agent.api.com.atproto.server.createInviteCode(

package/tests/moderation.test.ts

@@ -47,7 +47,7 @@ describe('moderation', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('takes down accounts', async () => {

package/tests/moderator-auth.test.ts

@@ -71,7 +71,7 @@ describe('moderator auth', () => {
   }, 20_000) // @NOTE seeding can take a while
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('allows service auth requests from the configured appview did', async () => {

package/tests/plc-operations.test.ts

@@ -6,6 +6,7 @@ import { AtpAgent } from '@atproto/api'
 import { check } from '@atproto/common'
 import { Secp256k1Keypair } from '@atproto/crypto'
 import { SeedClient, TestNetworkNoAppView, basicSeed } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 import { AppContext } from '../src/index.js'
 
 describe('plc operations', () => {
@@ -17,7 +18,7 @@ describe('plc operations', () => {
   const mailCatcher = new EventEmitter()
   let _origSendMail
 
-  let alice: string
+  let alice: DidString
 
   let sampleKey: string
 
@@ -48,7 +49,7 @@ describe('plc operations', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   const getMailFrom = async (promise): Promise<SendMailOptions> => {

package/tests/preferences.test.ts

@@ -29,7 +29,7 @@ describe('user preferences', () => {
   }, 20_000) // @NOTE seeding can take a while
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('requires auth to set or put preferences.', async () => {

package/tests/proxied/admin.test.ts

@@ -1,5 +1,6 @@
 import { AtpAgent } from '@atproto/api'
 import { SeedClient, TestNetwork } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 import { forSnapshot } from '../_util.js'
 import basicSeed from '../seeds/basic.js'
 
@@ -8,7 +9,7 @@ describe('proxies admin requests', () => {
   let agent: AtpAgent
   let sc: SeedClient
 
-  let moderator: string
+  let moderator: DidString
 
   beforeAll(async () => {
     network = await TestNetwork.create({
@@ -69,7 +70,7 @@ describe('proxies admin requests', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('creates reports of a repo.', async () => {

package/tests/proxied/feedgen.test.ts

@@ -51,7 +51,7 @@ describe('feedgen proxy view', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('performs basic proxy of getFeed', async () => {

package/tests/proxied/notif.test.ts

@@ -1,16 +1,13 @@
-import { once } from 'node:events'
-import http from 'node:http'
-import { AddressInfo } from 'node:net'
-import express from 'express'
 import { AtpAgent } from '@atproto/api'
 import { SeedClient, TestNetworkNoAppView } from '@atproto/dev-env'
 import { createServer, verifyJwt } from '@atproto/xrpc-server'
 import { app } from '../../src/lexicons/index.js'
+import { startServer } from '../_util.js'
 import usersSeed from '../seeds/users.js'
 
 describe('notif service proxy', () => {
   let network: TestNetworkNoAppView
-  let notifServer: http.Server
+  let notifServer: AsyncDisposable & { port: number }
   let notifDid: string
   let agent: AtpAgent
   let sc: SeedClient
@@ -29,10 +26,9 @@ describe('notif service proxy', () => {
     notifServer = await createMockNotifService(spy)
     notifDid = sc.dids.dan
     await plc.updateData(notifDid, network.pds.ctx.plcRotationKey, (x) => {
-      const addr = notifServer.address() as AddressInfo
       x.services['bsky_notif'] = {
         type: 'BskyNotificationService',
-        endpoint: `http://localhost:${addr.port}`,
+        endpoint: `http://localhost:${notifServer.port}`,
       }
       return x
     })
@@ -40,9 +36,8 @@ describe('notif service proxy', () => {
   }, 20_000) // @NOTE seeding can take a while
 
   afterAll(async () => {
-    await network.close()
-    notifServer.close()
-    await once(notifServer, 'close')
+    await notifServer?.[Symbol.asyncDispose]()
+    await network?.close()
   })
 
   it('proxies to notif service.', async () => {
@@ -86,7 +81,5 @@ async function createMockNotifService(ref: { current: unknown }) {
       jwt: req.headers.authorization?.replace('Bearer ', ''),
     }
   })
-  const server = express().use(svc.router).listen()
-  await once(server, 'listening')
-  return server
+  return startServer(svc.router)
 }

package/tests/proxied/procedures.test.ts

@@ -1,5 +1,6 @@
 import { AtpAgent } from '@atproto/api'
 import { SeedClient, TestNetwork } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 import basicSeed from '../seeds/basic.js'
 
 describe('proxies appview procedures', () => {
@@ -7,9 +8,9 @@ describe('proxies appview procedures', () => {
   let agent: AtpAgent
   let sc: SeedClient
 
-  let alice: string
-  let bob: string
-  let carol: string
+  let alice: DidString
+  let bob: DidString
+  let carol: DidString
 
   beforeAll(async () => {
     network = await TestNetwork.create({
@@ -25,7 +26,7 @@ describe('proxies appview procedures', () => {
   })
 
   afterAll(async () => {
-    await network.close()
+    await network?.close()
   })
 
   it('maintains muted actors.', async () => {

package/tests/proxied/proxy-catchall.test.ts

@@ -4,6 +4,8 @@ import { AddressInfo } from 'node:net'
 import { setTimeout as sleep } from 'node:timers/promises'
 import * as plc from '@did-plc/lib'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import AtpAgent from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
 import { TestNetworkNoAppView } from '@atproto/dev-env'
@@ -158,10 +160,14 @@ describe('proxy header', () => {
 })
 
 class ProxyServer {
+  private terminator: httpTerminator.HttpTerminator
+
   constructor(
-    private server: http.Server,
+    server: http.Server,
     public did: string,
-  ) {}
+  ) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(
     plcClient: plc.Client,
@@ -245,11 +251,6 @@ class ProxyServer {
   }
 
   async close() {
-    await new Promise<void>((resolve, reject) => {
-      this.server.close((err) => {
-        if (err) reject(err)
-        else resolve()
-      })
-    })
+    await this.terminator.terminate()
   }
 }

package/tests/proxied/proxy-header.test.ts

@@ -4,8 +4,11 @@ import http from 'node:http'
 import { AddressInfo } from 'node:net'
 import * as plc from '@did-plc/lib'
 import express from 'express'
+// eslint-disable-next-line import/default
+import httpTerminator from 'http-terminator'
 import { Keypair } from '@atproto/crypto'
 import { SeedClient, TestNetworkNoAppView, usersSeed } from '@atproto/dev-env'
+import type { DidString } from '@atproto/syntax'
 import { verifyJwt } from '@atproto/xrpc-server'
 import { parseProxyHeader } from '../../src/pipethrough.js'
 
@@ -13,7 +16,7 @@ describe('proxy header', () => {
   let network: TestNetworkNoAppView
   let sc: SeedClient
 
-  let alice: string
+  let alice: DidString
 
   let proxyServer: ProxyServer
 
@@ -35,8 +38,8 @@ describe('proxy header', () => {
   }, 20_000) // @NOTE seeding can take a while
 
   afterAll(async () => {
-    await proxyServer.close()
-    await network.close()
+    await proxyServer?.close()
+    await network?.close()
   })
 
   it('parses proxy header', async () => {
@@ -171,12 +174,16 @@ type ProxyReq = {
 }
 
 class ProxyServer {
+  private terminator: httpTerminator.HttpTerminator
+
   constructor(
-    public server: http.Server,
+    server: http.Server,
     public url: string,
     public did: string,
     public requests: ProxyReq[],
-  ) {}
+  ) {
+    this.terminator = httpTerminator.createHttpTerminator({ server })
+  }
 
   static async create(
     plcClient: plc.Client,
@@ -226,9 +233,7 @@ class ProxyServer {
     return new ProxyServer(server, url, did, requests)
   }
 
-  close(): Promise<void> {
-    return new Promise<void>((resolve) => {
-      this.server.close(() => resolve())
-    })
+  async close(): Promise<void> {
+    await this.terminator.terminate()
   }
 }