@atproto/xrpc-server 0.6.3 → 0.7.0

package/src/util.ts

@@ -1,7 +1,6 @@
-import assert from 'assert'
-import { Readable, Transform } from 'stream'
-import { IncomingMessage } from 'http'
-import { createDeflate, createGunzip } from 'zlib'
+import assert from 'node:assert'
+import { Duplex, pipeline, Readable } from 'node:stream'
+import { IncomingMessage } from 'node:http'
 import express from 'express'
 import mime from 'mime-types'
 import {
@@ -11,7 +10,9 @@ import {
   LexXrpcQuery,
   LexXrpcSubscription,
 } from '@atproto/lexicon'
-import { forwardStreamErrors, MaxSizeChecker } from '@atproto/common'
+import { createDecoders, MaxSizeChecker } from '@atproto/common'
+import { ResponseType } from '@atproto/xrpc'
+
 import {
   UndecodedParams,
   Params,
@@ -155,7 +156,7 @@ export function validateOutput(
   def: LexXrpcProcedure | LexXrpcQuery,
   output: HandlerSuccess | undefined,
   lexicons: Lexicons,
-): HandlerSuccess | undefined {
+): void {
   // initial validation
   if (output) {
     handlerSuccess.parse(output)
@@ -195,8 +196,6 @@ export function validateOutput(
       throw new InternalServerError(e instanceof Error ? e.message : String(e))
     }
   }
-
-  return output
 }
 
 export function normalizeMime(v: string) {
@@ -237,40 +236,52 @@ function decodeBodyStream(
   req: express.Request,
   maxSize: number | undefined,
 ): Readable {
-  let stream: Readable = req
   const contentEncoding = req.headers['content-encoding']
   const contentLength = req.headers['content-length']
 
+  const contentLengthParsed = contentLength
+    ? parseInt(contentLength, 10)
+    : undefined
+
+  if (Number.isNaN(contentLengthParsed)) {
+    throw new XRPCError(ResponseType.InvalidRequest, 'invalid content-length')
+  }
+
   if (
     maxSize !== undefined &&
-    contentLength &&
-    parseInt(contentLength, 10) > maxSize
+    contentLengthParsed !== undefined &&
+    contentLengthParsed > maxSize
   ) {
-    throw new XRPCError(413, 'request entity too large')
-  }
-
-  let decoder: Transform | undefined
-  if (contentEncoding === 'gzip') {
-    decoder = createGunzip()
-  } else if (contentEncoding === 'deflate') {
-    decoder = createDeflate()
+    throw new XRPCError(
+      ResponseType.PayloadTooLarge,
+      'request entity too large',
+    )
   }
 
-  if (decoder) {
-    forwardStreamErrors(stream, decoder)
-    stream = stream.pipe(decoder)
+  let transforms: Duplex[]
+  try {
+    transforms = createDecoders(contentEncoding)
+  } catch (cause) {
+    throw new XRPCError(
+      ResponseType.UnsupportedMediaType,
+      'unsupported content-encoding',
+      undefined,
+      { cause },
+    )
   }
 
   if (maxSize !== undefined) {
     const maxSizeChecker = new MaxSizeChecker(
       maxSize,
-      () => new XRPCError(413, 'request entity too large'),
+      () =>
+        new XRPCError(ResponseType.PayloadTooLarge, 'request entity too large'),
     )
-    forwardStreamErrors(stream, maxSizeChecker)
-    stream = stream.pipe(maxSizeChecker)
+    transforms.push(maxSizeChecker)
   }
 
-  return stream
+  return transforms.length > 0
+    ? (pipeline([req, ...transforms], () => {}) as Duplex)
+    : req
 }
 
 export function serverTimingHeader(timings: ServerTiming[]) {
@@ -308,11 +319,70 @@ export interface ServerTiming {
   description?: string
 }
 
-export const parseReqNsid = (
-  req: express.Request | IncomingMessage,
-): string => {
-  const originalUrl =
-    ('originalUrl' in req && req.originalUrl) || req.url || '/'
-  const nsid = originalUrl.split('?')[0].replace('/xrpc/', '')
-  return nsid.endsWith('/') ? nsid.slice(0, -1) : nsid // trim trailing slash
+export const parseReqNsid = (req: express.Request | IncomingMessage) =>
+  parseUrlNsid('originalUrl' in req ? req.originalUrl : req.url || '/')
+
+/**
+ * Validates and extracts the nsid from an xrpc path
+ */
+export const parseUrlNsid = (url: string): string => {
+  // /!\ Hot path
+
+  if (
+    // Ordered by likelihood of failure
+    url.length <= 6 ||
+    url[5] !== '/' ||
+    url[4] !== 'c' ||
+    url[3] !== 'p' ||
+    url[2] !== 'r' ||
+    url[1] !== 'x' ||
+    url[0] !== '/'
+  ) {
+    throw new InvalidRequestError('invalid xrpc path')
+  }
+
+  const startOfNsid = 6
+
+  let curr = startOfNsid
+  let char: number
+  let alphaNumRequired = true
+  for (; curr < url.length; curr++) {
+    char = url.charCodeAt(curr)
+    if (
+      (char >= 48 && char <= 57) || // 0-9
+      (char >= 65 && char <= 90) || // A-Z
+      (char >= 97 && char <= 122) // a-z
+    ) {
+      alphaNumRequired = false
+    } else if (char === 45 /* "-" */ || char === 46 /* "." */) {
+      if (alphaNumRequired) {
+        throw new InvalidRequestError('invalid xrpc path')
+      }
+      alphaNumRequired = true
+    } else if (char === 47 /* "/" */) {
+      // Allow trailing slash (next char is either EOS or "?")
+      if (curr === url.length - 1 || url.charCodeAt(curr + 1) === 63) {
+        break
+      }
+      throw new InvalidRequestError('invalid xrpc path')
+    } else if (char === 63 /* "?"" */) {
+      break
+    } else {
+      throw new InvalidRequestError('invalid xrpc path')
+    }
+  }
+
+  // last char was one of: '-', '.', '/'
+  if (alphaNumRequired) {
+    throw new InvalidRequestError('invalid xrpc path')
+  }
+
+  // A domain name consists of minimum two characters
+  if (curr - startOfNsid < 2) {
+    throw new InvalidRequestError('invalid xrpc path')
+  }
+
+  // @TODO is there a max ?
+
+  return url.slice(startOfNsid, curr)
 }

package/tests/bodies.test.ts

@@ -1,9 +1,9 @@
 import * as http from 'http'
 import { Readable } from 'stream'
-import { gzipSync } from 'zlib'
+import { brotliCompressSync, deflateSync, gzipSync } from 'zlib'
 import getPort from 'get-port'
 import { LexiconDoc } from '@atproto/lexicon'
-import { XrpcClient } from '@atproto/xrpc'
+import { ResponseType, XrpcClient } from '@atproto/xrpc'
 import { cidForCbor } from '@atproto/common'
 import { randomBytes } from '@atproto/crypto'
 import { createServer, closeServer } from './_util'
@@ -91,13 +91,25 @@ const BLOB_LIMIT = 5000
 async function consumeInput(
   input: Readable | string | object,
 ): Promise<Buffer> {
-  if (typeof input === 'string') return Buffer.from(input)
+  if (Buffer.isBuffer(input)) {
+    return input
+  }
+  if (typeof input === 'string') {
+    return Buffer.from(input)
+  }
   if (input instanceof Readable) {
-    const buffers: Buffer[] = []
-    for await (const data of input) {
-      buffers.push(data)
+    try {
+      return Buffer.concat(await input.toArray())
+    } catch (err) {
+      if (err instanceof xrpcServer.XRPCError) {
+        throw err
+      } else {
+        throw new xrpcServer.XRPCError(
+          ResponseType.InvalidRequest,
+          'unable to read input',
+        )
+      }
     }
-    return Buffer.concat(buffers)
   }
   throw new Error('Invalid input')
 }
@@ -111,10 +123,16 @@ describe('Bodies', () => {
   })
   server.method(
     'io.example.validationTest',
-    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => ({
-      encoding: 'json',
-      body: ctx.input?.body,
-    }),
+    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => {
+      if (ctx.input?.body instanceof Readable) {
+        throw new Error('Input is readable')
+      }
+
+      return {
+        encoding: 'json',
+        body: ctx.input?.body ?? null,
+      }
+    },
   )
   server.method('io.example.validationTestTwo', () => ({
     encoding: 'json',
@@ -338,24 +356,106 @@ describe('Bodies', () => {
     expect(streamResponse.data.cid).toEqual(expectedCid.toString())
   })
 
-  it('supports blobs and compression', async () => {
+  it('supports blob uploads', async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForCbor(bytes)
+
+    const { data } = await client.call('io.example.blobTest', {}, bytes, {
+      encoding: 'application/octet-stream',
+    })
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
+
+  it(`supports identity encoding`, async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForCbor(bytes)
+
+    const { data } = await client.call('io.example.blobTest', {}, bytes, {
+      encoding: 'application/octet-stream',
+      headers: { 'content-encoding': 'identity' },
+    })
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
+
+  it('supports gzip encoding', async () => {
     const bytes = randomBytes(1024)
     const expectedCid = await cidForCbor(bytes)
 
-    const { data: uncompressed } = await client.call(
+    const { data } = await client.call(
       'io.example.blobTest',
       {},
-      bytes,
+      gzipSync(bytes),
       {
         encoding: 'application/octet-stream',
+        headers: {
+          'content-encoding': 'gzip',
+        },
       },
     )
-    expect(uncompressed.cid).toEqual(expectedCid.toString())
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
 
-    const { data: compressed } = await client.call(
+  it('supports deflate encoding', async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForCbor(bytes)
+
+    const { data } = await client.call(
       'io.example.blobTest',
       {},
-      gzipSync(bytes),
+      deflateSync(bytes),
+      {
+        encoding: 'application/octet-stream',
+        headers: {
+          'content-encoding': 'deflate',
+        },
+      },
+    )
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
+
+  it('supports br encoding', async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForCbor(bytes)
+
+    const { data } = await client.call(
+      'io.example.blobTest',
+      {},
+      brotliCompressSync(bytes),
+      {
+        encoding: 'application/octet-stream',
+        headers: {
+          'content-encoding': 'br',
+        },
+      },
+    )
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
+
+  it('supports multiple encodings', async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForCbor(bytes)
+
+    const { data } = await client.call(
+      'io.example.blobTest',
+      {},
+      brotliCompressSync(deflateSync(gzipSync(bytes))),
+      {
+        encoding: 'application/octet-stream',
+        headers: {
+          'content-encoding': 'gzip, identity, deflate, identity, br, identity',
+        },
+      },
+    )
+    expect(data.cid).toEqual(expectedCid.toString())
+  })
+
+  it('fails gracefully on invalid encodings', async () => {
+    const bytes = randomBytes(1024)
+
+    const promise = client.call(
+      'io.example.blobTest',
+      {},
+      brotliCompressSync(bytes),
       {
         encoding: 'application/octet-stream',
         headers: {
@@ -363,7 +463,8 @@ describe('Bodies', () => {
         },
       },
     )
-    expect(compressed.cid).toEqual(expectedCid.toString())
+
+    await expect(promise).rejects.toThrow('unable to read input')
   })
 
   it('supports empty payload', async () => {
@@ -428,9 +529,7 @@ describe('Bodies', () => {
     })
   })
 
-  // @TODO: figure out why this is failing dependent on the prev test being run
-  // https://github.com/bluesky-social/atproto/pull/550/files#r1106400413
-  it.skip('errors on an empty Content-type on blob upload', async () => {
+  it('errors on an empty Content-type on blob upload', async () => {
     // empty mimetype, but correct syntax
     const res = await fetch(`${url}/xrpc/io.example.blobTest`, {
       method: 'post',

package/tests/parsing.test.ts

@@ -0,0 +1,89 @@
+import { parseUrlNsid } from '../src/util'
+
+const testValid = (url: string, expected: string) => {
+  expect(parseUrlNsid(url)).toBe(expected)
+}
+
+const testInvalid = (url: string, errorMessage = 'invalid xrpc path') => {
+  expect(() => parseUrlNsid(url)).toThrow(errorMessage)
+}
+
+describe('parseUrlNsid', () => {
+  it('should extract the NSID from the URL', () => {
+    testValid('/xrpc/blee.blah.bloo', 'blee.blah.bloo')
+    testValid('/xrpc/blee.blah.bloo?foo[]', 'blee.blah.bloo')
+    testValid('/xrpc/blee.blah.bloo?foo=bar', 'blee.blah.bloo')
+    testValid('/xrpc/com.example.nsid', 'com.example.nsid')
+    testValid('/xrpc/com.example.nsid?foo=bar', 'com.example.nsid')
+    testValid('/xrpc/com.example-domain.nsid', 'com.example-domain.nsid')
+  })
+
+  it('should allow a trailing slash', () => {
+    testValid('/xrpc/blee.blah.bloo/?', 'blee.blah.bloo')
+    testValid('/xrpc/blee.blah.bloo/?foo=', 'blee.blah.bloo')
+    testValid('/xrpc/blee.blah.bloo/?bool', 'blee.blah.bloo')
+    testValid('/xrpc/com.example.nsid/', 'com.example.nsid')
+  })
+
+  it('should throw an error if the URL is too short', () => {
+    testInvalid('/xrpc/a')
+  })
+
+  it('should throw an error if the URL is empty', () => {
+    testInvalid('')
+  })
+
+  it('should throw an error if the URL is missing the NSID', () => {
+    testInvalid('/xrpc/')
+    testInvalid('/xrpc/?')
+    testInvalid('/xrpc/?foo=bar')
+  })
+
+  it('should throw an error if the URL contains extra path segments', () => {
+    testInvalid('/xrpc/123/extra')
+    testInvalid('/xrpc/123/extra?foo=bar')
+  })
+
+  it('should throw an error if the URL is missing the XRPC path prefix', () => {
+    testInvalid('/foo/123')
+    testInvalid('/foo/com.example.nsid')
+  })
+
+  it('should throw an error if the NSID starts with a dot', () => {
+    testInvalid('/xrpc/.')
+    testInvalid('/xrpc/..')
+    testInvalid('/xrpc/....')
+    testInvalid('/xrpc/.com.example.nsid')
+    testInvalid('/xrpc/com..example.nsid')
+    testInvalid('/xrpc/com.example..nsid')
+    testInvalid('/xrpc/com.example.nsid.')
+    testInvalid('/xrpc/com.example.nsid./')
+    testInvalid('/xrpc/com.example.nsid.?foo=bar')
+    testInvalid('/xrpc/com.example.nsid./?foo=bar')
+  })
+
+  it('should throw an error if the NSID contains a misplaced dash', () => {
+    testInvalid('/xrpc/-')
+    testInvalid('/xrpc/com.example.-nsid')
+    testInvalid('/xrpc/com.example-.nsid')
+    testInvalid('/xrpc/com.-example.nsid')
+    testInvalid('/xrpc/com.-example-.nsid')
+    testInvalid('/xrpc/com.example.nsid-')
+    testInvalid('/xrpc/-com.example.nsid')
+    testInvalid('/xrpc/com.example--domain.nsid')
+  })
+
+  it('should throw an error if the URL starts with a space', () => {
+    testInvalid(' /xrpc/com.example.nsid')
+  })
+
+  it('should throw an error if the NSID contains invalid characters', () => {
+    testInvalid('/xrpc/com.example.nsid#')
+    testInvalid('/xrpc/com.example.nsid!')
+    testInvalid('/xrpc/com.example#?nsid')
+    testInvalid('/xrpc/!com.example.nsid')
+    testInvalid('/xrpc/com.example.nsid ')
+    testInvalid('/xrpc/ com.example.nsid')
+    testInvalid('/xrpc/com. example.nsid')
+  })
+})