@atproto/xrpc-server 0.7.19 → 0.9.0

package/src/util.ts

@@ -1,8 +1,8 @@
 import assert from 'node:assert'
-import { IncomingMessage } from 'node:http'
+import { IncomingMessage, OutgoingMessage } from 'node:http'
 import { Duplex, Readable, pipeline } from 'node:stream'
-import express from 'express'
-import mime from 'mime-types'
+import { Request, Response, json, text } from 'express'
+import { contentType } from 'mime-types'
 import { MaxSizeChecker, createDecoders } from '@atproto/common'
 import {
   LexXrpcProcedure,
@@ -12,18 +12,31 @@ import {
   jsonToLex,
 } from '@atproto/lexicon'
 import { ResponseType } from '@atproto/xrpc'
+import { InternalServerError, InvalidRequestError, XRPCError } from './errors'
 import {
-  HandlerInput,
+  Awaitable,
   HandlerSuccess,
-  InternalServerError,
-  InvalidRequestError,
+  Input,
   Params,
-  RouteOpts,
+  RouteOptions,
   UndecodedParams,
-  XRPCError,
   handlerSuccess,
 } from './types'
 
+export const asArray = <T>(arr: T | T[]): T[] =>
+  Array.isArray(arr) ? arr : [arr]
+
+export function setHeaders(
+  res: OutgoingMessage,
+  headers?: Record<string, string | number>,
+) {
+  if (headers) {
+    for (const [name, val] of Object.entries(headers)) {
+      if (val != null) res.setHeader(name, val)
+    }
+  }
+}
+
 export function decodeQueryParams(
   def: LexXrpcProcedure | LexXrpcQuery | LexXrpcSubscription,
   params: UndecodedParams,
@@ -67,93 +80,104 @@ export function decodeQueryParam(
   }
 }
 
-export function getQueryParams(url = ''): Record<string, string | string[]> {
-  const { searchParams } = new URL(url ?? '', 'http://x')
-  const result: Record<string, string | string[]> = {}
+export type QueryParams = Record<string, undefined | string | string[]>
+export function getQueryParams(url = ''): QueryParams {
+  const result: QueryParams = Object.create(null)
+
+  const queryStringIdx = url.indexOf('?')
+  if (queryStringIdx === -1) return result
+
+  const queryString = url.slice(queryStringIdx + 1)
+  if (queryString === '') return result
+
+  const searchParams = new URLSearchParams(queryString)
   for (const key of searchParams.keys()) {
-    result[key] = searchParams.getAll(key)
-    if (result[key].length === 1) {
-      result[key] = result[key][0]
+    if (key === '__proto__') {
+      // Prevent prototype pollution
+      throw new InvalidRequestError(
+        `Invalid query parameter: ${key}`,
+        'InvalidQueryParameter',
+      )
     }
+
+    const values = searchParams.getAll(key)
+    result[key] = values.length === 1 ? values[0] : values
   }
+
   return result
 }
 
-export function validateInput(
+export function createInputVerifier(
   nsid: string,
   def: LexXrpcProcedure | LexXrpcQuery,
-  req: express.Request,
-  opts: RouteOpts,
+  options: RouteOptions,
   lexicons: Lexicons,
-): HandlerInput | undefined {
-  // request expectation
+): (req: Request, res: Response) => Awaitable<Input> {
+  if (def.type === 'query' || !def.input) {
+    return (req) => {
+      // @NOTE We allow (and ignore) "empty" bodies
+      if (getBodyPresence(req) === 'present') {
+        throw new InvalidRequestError(
+          `A request body was provided when none was expected`,
+        )
+      }
 
-  const bodyPresence = getBodyPresence(req)
-  if (bodyPresence === 'present' && (def.type !== 'procedure' || !def.input)) {
-    throw new InvalidRequestError(
-      `A request body was provided when none was expected`,
-    )
-  }
-  if (def.type === 'query') {
-    return
-  }
-  if (bodyPresence === 'missing' && def.input) {
-    throw new InvalidRequestError(
-      `A request body is expected but none was provided`,
-    )
+      return undefined
+    }
   }
 
-  // mimetype
-  const inputEncoding = normalizeMime(req.headers['content-type'] || '')
-  if (
-    def.input?.encoding &&
-    (!inputEncoding || !isValidEncoding(def.input?.encoding, inputEncoding))
-  ) {
-    if (!inputEncoding) {
+  // Lexicon definition expects a request body
+
+  const { input } = def
+  const { blobLimit } = options
+
+  const bodyParser = createBodyParser(input.encoding, options)
+
+  return async (req, res) => {
+    if (getBodyPresence(req) === 'missing') {
+      throw new InvalidRequestError(
+        `A request body is expected but none was provided`,
+      )
+    }
+
+    const reqEncoding = normalizeMime(req.headers['content-type'])
+    if (!reqEncoding) {
       throw new InvalidRequestError(
         `Request encoding (Content-Type) required but not provided`,
       )
-    } else {
+    } else if (!isValidEncoding(input.encoding, reqEncoding)) {
       throw new InvalidRequestError(
-        `Wrong request encoding (Content-Type): ${inputEncoding}`,
+        `Wrong request encoding (Content-Type): ${reqEncoding}`,
       )
     }
-  }
 
-  if (!inputEncoding) {
-    // no input body
-    return undefined
-  }
+    if (bodyParser) {
+      await bodyParser(req, res)
+    }
 
-  // if input schema, validate
-  if (def.input?.schema) {
-    try {
-      const lexBody = req.body ? jsonToLex(req.body) : req.body
-      req.body = lexicons.assertValidXrpcInput(nsid, lexBody)
-    } catch (e) {
-      throw new InvalidRequestError(e instanceof Error ? e.message : String(e))
+    if (input.schema) {
+      try {
+        const lexBody = req.body ? jsonToLex(req.body) : req.body
+        req.body = lexicons.assertValidXrpcInput(nsid, lexBody)
+      } catch (e) {
+        throw new InvalidRequestError(
+          e instanceof Error ? e.message : String(e),
+        )
+      }
     }
-  }
 
-  // if middleware already got the body, we pass that along as input
-  // otherwise, we pass along a decoded readable stream
-  let body
-  if (req.readableEnded) {
-    body = req.body
-  } else {
-    body = decodeBodyStream(req, opts.blobLimit)
-  }
+    // if middleware already got the body, we pass that along as input
+    // otherwise, we pass along a decoded readable stream
+    const body = req.readableEnded ? req.body : decodeBodyStream(req, blobLimit)
 
-  return {
-    encoding: inputEncoding,
-    body,
+    return { encoding: reqEncoding, body }
   }
 }
 
 export function validateOutput(
   nsid: string,
   def: LexXrpcProcedure | LexXrpcQuery,
-  output: HandlerSuccess | undefined,
+  output: HandlerSuccess | void,
   lexicons: Lexicons,
 ): void {
   // initial validation
@@ -197,42 +221,58 @@ export function validateOutput(
   }
 }
 
-export function normalizeMime(v: string) {
+export function normalizeMime(v?: string): string | false {
   if (!v) return false
-  const fullType = mime.contentType(v)
+  const fullType = contentType(v)
   if (!fullType) return false
   const shortType = fullType.split(';')[0]
   if (!shortType) return false
   return shortType
 }
 
+const ENCODING_ANY = '*/*'
+
 function isValidEncoding(possibleStr: string, value: string) {
   const possible = possibleStr.split(',').map((v) => v.trim())
   const normalized = normalizeMime(value)
   if (!normalized) return false
-  if (possible.includes('*/*')) return true
+  if (possible.includes(ENCODING_ANY)) return true
   return possible.includes(normalized)
 }
 
 type BodyPresence = 'missing' | 'empty' | 'present'
 
-function getBodyPresence(req: express.Request): BodyPresence {
+function getBodyPresence(req: IncomingMessage): BodyPresence {
   if (req.headers['transfer-encoding'] != null) return 'present'
   if (req.headers['content-length'] === '0') return 'empty'
   if (req.headers['content-length'] != null) return 'present'
   return 'missing'
 }
 
-export function processBodyAsBytes(req: express.Request): Promise<Uint8Array> {
-  return new Promise((resolve) => {
-    const chunks: Buffer[] = []
-    req.on('data', (chunk) => chunks.push(chunk))
-    req.on('end', () => resolve(new Uint8Array(Buffer.concat(chunks))))
-  })
+function createBodyParser(inputEncoding: string, options: RouteOptions) {
+  if (inputEncoding === ENCODING_ANY) {
+    // When the lexicon's input encoding is */*, the handler will determine how to process it
+    return
+  }
+  const { jsonLimit, textLimit } = options
+  const jsonParser = json({ limit: jsonLimit })
+  const textParser = text({ limit: textLimit })
+  // Transform json and text parser middlewares into a single function
+  return (req: Request, res: Response) => {
+    return new Promise<void>((resolve, reject) => {
+      jsonParser(req, res, (err) => {
+        if (err) return reject(XRPCError.fromError(err))
+        textParser(req, res, (err) => {
+          if (err) return reject(XRPCError.fromError(err))
+          resolve()
+        })
+      })
+    })
+  }
 }
 
 function decodeBodyStream(
-  req: express.Request,
+  req: IncomingMessage,
   maxSize: number | undefined,
 ): Readable {
   const contentEncoding = req.headers['content-encoding']
@@ -318,13 +358,19 @@ export interface ServerTiming {
   description?: string
 }
 
-export const parseReqNsid = (req: express.Request | IncomingMessage) =>
+export const parseReqNsid = (req: Request | IncomingMessage) =>
   parseUrlNsid('originalUrl' in req ? req.originalUrl : req.url || '/')
 
 /**
  * Validates and extracts the nsid from an xrpc path
  */
 export const parseUrlNsid = (url: string): string => {
+  const nsid = extractUrlNsid(url)
+  if (nsid) return nsid
+  throw new InvalidRequestError('invalid xrpc path')
+}
+
+export const extractUrlNsid = (url: string): string | undefined => {
   // /!\ Hot path
 
   if (
@@ -337,7 +383,7 @@ export const parseUrlNsid = (url: string): string => {
     url[1] !== 'x' ||
     url[0] !== '/'
   ) {
-    throw new InvalidRequestError('invalid xrpc path')
+    return undefined
   }
 
   const startOfNsid = 6
@@ -355,7 +401,7 @@ export const parseUrlNsid = (url: string): string => {
       alphaNumRequired = false
     } else if (char === 45 /* "-" */ || char === 46 /* "." */) {
       if (alphaNumRequired) {
-        throw new InvalidRequestError('invalid xrpc path')
+        return undefined
       }
       alphaNumRequired = true
     } else if (char === 47 /* "/" */) {
@@ -363,25 +409,25 @@ export const parseUrlNsid = (url: string): string => {
       if (curr === url.length - 1 || url.charCodeAt(curr + 1) === 63) {
         break
       }
-      throw new InvalidRequestError('invalid xrpc path')
+      return undefined
     } else if (char === 63 /* "?"" */) {
       break
     } else {
-      throw new InvalidRequestError('invalid xrpc path')
+      return undefined
     }
   }
 
   // last char was one of: '-', '.', '/'
   if (alphaNumRequired) {
-    throw new InvalidRequestError('invalid xrpc path')
+    return undefined
   }
 
   // A domain name consists of minimum two characters
   if (curr - startOfNsid < 2) {
-    throw new InvalidRequestError('invalid xrpc path')
+    return undefined
   }
 
-  // @TODO is there a max ?
+  // @TODO check max length of nsid
 
   return url.slice(startOfNsid, curr)
 }

package/tests/auth.test.ts

@@ -56,8 +56,8 @@ describe('Auth', () => {
       return {
         encoding: 'application/json',
         body: {
-          username: auth?.credentials?.username,
-          original: auth?.artifacts?.original,
+          username: auth.credentials.username,
+          original: auth.artifacts.original,
         },
       }
     },

package/tests/bodies.test.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import * as http from 'node:http'
 import { AddressInfo } from 'node:net'
 import { Readable } from 'node:stream'
@@ -121,34 +122,27 @@ describe('Bodies', () => {
       blobLimit: BLOB_LIMIT,
     },
   })
-  server.method(
-    'io.example.validationTest',
-    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => {
-      if (ctx.input?.body instanceof Readable) {
-        throw new Error('Input is readable')
-      }
+  server.method('io.example.validationTest', (ctx) => {
+    assert(!(ctx.input?.body instanceof Readable), 'Input is readable')
 
-      return {
-        encoding: 'json',
-        body: ctx.input?.body ?? null,
-      }
-    },
-  )
+    return {
+      encoding: 'json',
+      body: ctx.input?.body ?? null,
+    }
+  })
   server.method('io.example.validationTestTwo', () => ({
     encoding: 'json',
     body: { wrong: 'data' },
   }))
-  server.method(
-    'io.example.blobTest',
-    async (ctx: { input?: xrpcServer.HandlerInput }) => {
-      const buffer = await consumeInput(ctx.input?.body)
-      const cid = await cidForCbor(buffer)
-      return {
-        encoding: 'json',
-        body: { cid: cid.toString() },
-      }
-    },
-  )
+  server.method('io.example.blobTest', async (ctx) => {
+    assert(ctx.input?.body != null, 'Input body is required')
+    const buffer = await consumeInput(ctx.input.body)
+    const cid = await cidForCbor(buffer)
+    return {
+      encoding: 'json',
+      body: { cid: cid.toString() },
+    }
+  })
 
   let client: XrpcClient
   let url: string
@@ -295,10 +289,7 @@ describe('Bodies', () => {
     expect(fileResponse.data.cid).toEqual(expectedCid.toString())
   })
 
-  // This does not work because the xrpc-server will add a json middleware
-  // regardless of the "input" definition. This is probably a behavior that
-  // should be fixed in the xrpc-server.
-  it.skip('supports upload of json data', async () => {
+  it('supports upload of json data', async () => {
     const jsonFile = new Blob([Buffer.from(`{"foo":"bar","baz":[3, null]}`)], {
       type: 'application/json',
     })

package/tests/errors.test.ts

@@ -144,7 +144,7 @@ describe('Errors', () => {
 
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS, { validateResponse: false }) // disable validateResponse to test client validation
-  server.method('io.example.error', (ctx: { params: xrpcServer.Params }) => {
+  server.method('io.example.error', (ctx) => {
     if (ctx.params.which === 'foo') {
       throw new xrpcServer.InvalidRequestError('It was this one!', 'Foo')
     } else if (ctx.params.which === 'bar') {

package/tests/ipld.test.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import * as http from 'node:http'
 import { AddressInfo } from 'node:net'
 import { CID } from 'multiformats/cid'
@@ -49,20 +50,20 @@ const LEXICONS: LexiconDoc[] = [
 describe('Ipld vals', () => {
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS)
-  server.method(
-    'io.example.ipld',
-    (ctx: { input?: xrpcServer.HandlerInput }) => {
-      const asCid = CID.asCID(ctx.input?.body.cid)
-      if (!(asCid instanceof CID)) {
-        throw new Error('expected cid')
-      }
-      const bytes = ctx.input?.body.bytes
-      if (!(bytes instanceof Uint8Array)) {
-        throw new Error('expected bytes')
-      }
-      return { encoding: 'application/json', body: ctx.input?.body }
-    },
-  )
+  server.method('io.example.ipld', (ctx) => {
+    assert(ctx.input?.body, 'expected input body')
+    assert(typeof ctx.input.body === 'object', 'expected input body')
+
+    const asCid = CID.asCID(ctx.input.body['cid'])
+    if (!(asCid instanceof CID)) {
+      throw new Error('expected cid')
+    }
+    const bytes = ctx.input.body['bytes']
+    if (!(bytes instanceof Uint8Array)) {
+      throw new Error('expected bytes')
+    }
+    return { encoding: 'application/json', body: ctx.input.body }
+  })
 
   let client: XrpcClient
   beforeAll(async () => {

package/tests/parameters.test.ts

@@ -34,13 +34,10 @@ const LEXICONS: LexiconDoc[] = [
 describe('Parameters', () => {
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS)
-  server.method(
-    'io.example.paramTest',
-    (ctx: { params: xrpcServer.Params }) => ({
-      encoding: 'json',
-      body: ctx.params,
-    }),
-  )
+  server.method('io.example.paramTest', (ctx) => ({
+    encoding: 'json',
+    body: ctx.params,
+  }))
 
   let client: XrpcClient
   beforeAll(async () => {

package/tests/procedures.test.ts

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import * as http from 'node:http'
 import { AddressInfo } from 'node:net'
 import { Readable } from 'node:stream'
@@ -85,42 +86,29 @@ const LEXICONS: LexiconDoc[] = [
 describe('Procedures', () => {
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS)
-  server.method('io.example.pingOne', (ctx: { params: xrpcServer.Params }) => {
+  server.method('io.example.pingOne', (ctx) => {
     return { encoding: 'text/plain', body: ctx.params.message }
   })
-  server.method(
-    'io.example.pingTwo',
-    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => {
-      return { encoding: 'text/plain', body: ctx.input?.body }
-    },
-  )
-  server.method(
-    'io.example.pingThree',
-    async (ctx: {
-      params: xrpcServer.Params
-      input?: xrpcServer.HandlerInput
-    }) => {
-      if (!(ctx.input?.body instanceof Readable))
-        throw new Error('Input not readable')
-      const buffers: Buffer[] = []
-      for await (const data of ctx.input.body) {
-        buffers.push(data)
-      }
-      return {
-        encoding: 'application/octet-stream',
-        body: Buffer.concat(buffers),
-      }
-    },
-  )
-  server.method(
-    'io.example.pingFour',
-    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => {
-      return {
-        encoding: 'application/json',
-        body: { message: ctx.input?.body?.message },
-      }
-    },
-  )
+  server.method('io.example.pingTwo', (ctx) => {
+    return { encoding: 'text/plain', body: ctx.input?.body }
+  })
+  server.method('io.example.pingThree', async (ctx) => {
+    assert(ctx.input?.body instanceof Readable, 'Input not readable')
+    const buffers: Buffer[] = []
+    for await (const data of ctx.input.body) {
+      buffers.push(data)
+    }
+    return {
+      encoding: 'application/octet-stream',
+      body: Buffer.concat(buffers),
+    }
+  })
+  server.method('io.example.pingFour', (ctx) => {
+    return {
+      encoding: 'application/json',
+      body: { message: ctx.input?.body?.['message'] },
+    }
+  })
 
   let client: XrpcClient
   beforeAll(async () => {

package/tests/queries.test.ts

@@ -70,25 +70,22 @@ const LEXICONS: LexiconDoc[] = [
 describe('Queries', () => {
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS)
-  server.method('io.example.pingOne', (ctx: { params: xrpcServer.Params }) => {
+  server.method('io.example.pingOne', (ctx) => {
     return { encoding: 'text/plain', body: ctx.params.message }
   })
-  server.method('io.example.pingTwo', (ctx: { params: xrpcServer.Params }) => {
+  server.method('io.example.pingTwo', (ctx) => {
     return {
       encoding: 'application/octet-stream',
       body: new TextEncoder().encode(String(ctx.params.message)),
     }
   })
-  server.method(
-    'io.example.pingThree',
-    (ctx: { params: xrpcServer.Params }) => {
-      return {
-        encoding: 'application/json',
-        body: { message: ctx.params.message },
-        headers: { 'x-test-header-name': 'test-value' },
-      }
-    },
-  )
+  server.method('io.example.pingThree', (ctx) => {
+    return {
+      encoding: 'application/json',
+      body: { message: ctx.params.message },
+      headers: { 'x-test-header-name': 'test-value' },
+    }
+  })
 
   let client: XrpcClient
   beforeAll(async () => {

package/tests/rate-limiter.test.ts

@@ -4,7 +4,7 @@ import { MINUTE } from '@atproto/common'
 import { LexiconDoc } from '@atproto/lexicon'
 import { XrpcClient } from '@atproto/xrpc'
 import * as xrpcServer from '../src'
-import { RateLimiter } from '../src'
+import { MemoryRateLimiter } from '../src'
 import { closeServer, createServer } from './_util'
 
 const LEXICONS: LexiconDoc[] = [
@@ -132,11 +132,8 @@ describe('Parameters', () => {
   let s: http.Server
   const server = xrpcServer.createServer(LEXICONS, {
     rateLimits: {
-      creator: (opts: xrpcServer.RateLimiterOpts) =>
-        RateLimiter.memory({
-          bypassSecret: 'bypass',
-          ...opts,
-        }),
+      creator: (opts) => new MemoryRateLimiter(opts),
+      bypass: ({ req }) => req.headers['x-ratelimit-bypass'] === 'bypass',
       shared: [
         {
           name: 'shared-limit',
@@ -159,7 +156,7 @@ describe('Parameters', () => {
       points: 5,
       calcKey: ({ params }) => params.str as string,
     },
-    handler: (ctx: { params: xrpcServer.Params }) => ({
+    handler: (ctx) => ({
       encoding: 'json',
       body: ctx.params,
     }),
@@ -169,7 +166,7 @@ describe('Parameters', () => {
       durationMs: 5 * MINUTE,
       points: 2,
     },
-    handler: (ctx: xrpcServer.XRPCReqContext) => {
+    handler: (ctx) => {
       if (ctx.params.count === 1) {
         ctx.resetRouteRateLimits()
       }
@@ -185,7 +182,7 @@ describe('Parameters', () => {
       name: 'shared-limit',
       calcPoints: ({ params }) => params.points as number,
     },
-    handler: (ctx: { params: xrpcServer.Params }) => ({
+    handler: (ctx) => ({
       encoding: 'json',
       body: ctx.params,
     }),
@@ -195,7 +192,7 @@ describe('Parameters', () => {
       name: 'shared-limit',
       calcPoints: ({ params }) => params.points as number,
     },
-    handler: (ctx: { params: xrpcServer.Params }) => ({
+    handler: (ctx) => ({
       encoding: 'json',
       body: ctx.params,
     }),
@@ -212,7 +209,7 @@ describe('Parameters', () => {
         points: 10,
       },
     ],
-    handler: (ctx: { params: xrpcServer.Params }) => ({
+    handler: (ctx) => ({
       encoding: 'json',
       body: ctx.params,
     }),