@atproto/xrpc-server 0.0.1

package/src/util.ts

@@ -0,0 +1,237 @@
+import { Readable, Transform } from 'stream'
+import { createDeflate, createGunzip } from 'zlib'
+import express from 'express'
+import mime from 'mime-types'
+import { Lexicons, LexXrpcProcedure, LexXrpcQuery } from '@atproto/lexicon'
+import { forwardStreamErrors, MaxSizeChecker } from '@atproto/common'
+import {
+  UndecodedParams,
+  Params,
+  HandlerInput,
+  HandlerSuccess,
+  handlerSuccess,
+  InvalidRequestError,
+  InternalServerError,
+  Options,
+  XRPCError,
+} from './types'
+
+export function decodeQueryParams(
+  def: LexXrpcProcedure | LexXrpcQuery,
+  params: UndecodedParams,
+): Params {
+  const decoded: Params = {}
+  for (const k in params) {
+    const val = params[k]
+    const property = def.parameters?.properties?.[k]
+    if (property) {
+      if (property.type === 'array') {
+        const vals: typeof val[] = []
+        decoded[k] = val
+          ? vals
+              .concat(val) // Cast to array
+              .flatMap((v) => decodeQueryParam(property.items.type, v) ?? [])
+          : undefined
+      } else {
+        decoded[k] = decodeQueryParam(property.type, val)
+      }
+    }
+  }
+  return decoded
+}
+
+export function decodeQueryParam(
+  type: string,
+  value: unknown,
+): string | number | boolean | undefined {
+  if (!value) {
+    return undefined
+  }
+  if (type === 'string' || type === 'datetime') {
+    return String(value)
+  }
+  if (type === 'number') {
+    return Number(String(value))
+  } else if (type === 'integer') {
+    return Number(String(value)) | 0
+  } else if (type === 'boolean') {
+    return value === 'true'
+  }
+}
+
+export function validateInput(
+  nsid: string,
+  def: LexXrpcProcedure | LexXrpcQuery,
+  req: express.Request,
+  opts: Options,
+  lexicons: Lexicons,
+): HandlerInput | undefined {
+  // request expectation
+  const reqHasBody = hasBody(req)
+  if (reqHasBody && (def.type !== 'procedure' || !def.input)) {
+    throw new InvalidRequestError(
+      `A request body was provided when none was expected`,
+    )
+  }
+  if (def.type === 'query') {
+    return
+  }
+  if (!reqHasBody && def.input) {
+    throw new InvalidRequestError(
+      `A request body is expected but none was provided`,
+    )
+  }
+
+  // mimetype
+  const inputEncoding = normalizeMime(req.headers['content-type'] || '')
+  if (
+    def.input?.encoding &&
+    (!inputEncoding || !isValidEncoding(def.input?.encoding, inputEncoding))
+  ) {
+    throw new InvalidRequestError(`Invalid request encoding: ${inputEncoding}`)
+  }
+
+  if (!inputEncoding) {
+    // no input body
+    return undefined
+  }
+
+  // if input schema, validate
+  if (def.input?.schema) {
+    try {
+      lexicons.assertValidXrpcInput(nsid, req.body)
+    } catch (e) {
+      throw new InvalidRequestError(e instanceof Error ? e.message : String(e))
+    }
+  }
+
+  // if middleware already got the body, we pass that along as input
+  // otherwise, we pass along a decoded readable stream
+  let body
+  if (req.readableEnded) {
+    body = req.body
+  } else {
+    body = decodeBodyStream(req, opts.payload?.blobLimit)
+  }
+
+  return {
+    encoding: inputEncoding,
+    body,
+  }
+}
+
+export function validateOutput(
+  nsid: string,
+  def: LexXrpcProcedure | LexXrpcQuery,
+  output: HandlerSuccess | undefined,
+  lexicons: Lexicons,
+): HandlerSuccess | undefined {
+  // initial validation
+  if (output) {
+    handlerSuccess.parse(output)
+  }
+
+  // response expectation
+  if (output?.body && !def.output) {
+    throw new InternalServerError(
+      `A response body was provided when none was expected`,
+    )
+  }
+  if (!output?.body && def.output) {
+    throw new InternalServerError(
+      `A response body is expected but none was provided`,
+    )
+  }
+
+  // mimetype
+  if (
+    def.output?.encoding &&
+    (!output?.encoding ||
+      !isValidEncoding(def.output?.encoding, output?.encoding))
+  ) {
+    throw new InternalServerError(
+      `Invalid response encoding: ${output?.encoding}`,
+    )
+  }
+
+  // output schema
+  if (def.output?.schema) {
+    try {
+      lexicons.assertValidXrpcOutput(nsid, output?.body)
+    } catch (e) {
+      throw new InternalServerError(e instanceof Error ? e.message : String(e))
+    }
+  }
+
+  return output
+}
+
+export function normalizeMime(v: string) {
+  const fullType = mime.contentType(v)
+  if (!v) return false
+  const shortType = fullType.split(';')[0]
+  if (!shortType) return false
+  return shortType
+}
+
+function isValidEncoding(possibleStr: string, value: string) {
+  const possible = possibleStr.split(',').map((v) => v.trim())
+  const normalized = normalizeMime(value)
+  if (!normalized) return false
+  if (possible.includes('*/*')) return true
+  return possible.includes(normalized)
+}
+
+export function hasBody(req: express.Request) {
+  const contentLength = req.headers['content-length']
+  const transferEncoding = req.headers['transfer-encoding']
+  return (contentLength && parseInt(contentLength, 10) > 0) || transferEncoding
+}
+
+export function processBodyAsBytes(req: express.Request): Promise<Uint8Array> {
+  return new Promise((resolve) => {
+    const chunks: Buffer[] = []
+    req.on('data', (chunk) => chunks.push(chunk))
+    req.on('end', () => resolve(new Uint8Array(Buffer.concat(chunks))))
+  })
+}
+
+function decodeBodyStream(
+  req: express.Request,
+  maxSize: number | undefined,
+): Readable {
+  let stream: Readable = req
+  const contentEncoding = req.headers['content-encoding']
+  const contentLength = req.headers['content-length']
+
+  if (
+    maxSize !== undefined &&
+    contentLength &&
+    parseInt(contentLength, 10) > maxSize
+  ) {
+    throw new XRPCError(413, 'request entity too large')
+  }
+
+  let decoder: Transform | undefined
+  if (contentEncoding === 'gzip') {
+    decoder = createGunzip()
+  } else if (contentEncoding === 'deflate') {
+    decoder = createDeflate()
+  }
+
+  if (decoder) {
+    forwardStreamErrors(stream, decoder)
+    stream = stream.pipe(decoder)
+  }
+
+  if (maxSize !== undefined) {
+    const maxSizeChecker = new MaxSizeChecker(
+      maxSize,
+      () => new XRPCError(413, 'request entity too large'),
+    )
+    forwardStreamErrors(stream, maxSizeChecker)
+    stream = stream.pipe(maxSizeChecker)
+  }
+
+  return stream
+}

package/tests/_util.ts

@@ -0,0 +1,20 @@
+import * as http from 'http'
+import express from 'express'
+import * as xrpc from '../src/index'
+
+export async function createServer(
+  port: number,
+  server: xrpc.Server,
+): Promise<http.Server> {
+  const app = express()
+  app.use(server.router)
+  const httpServer = app.listen(port)
+  await new Promise((r) => httpServer.on('listening', r))
+  return httpServer
+}
+
+export async function closeServer(httpServer: http.Server) {
+  await new Promise((r) => {
+    httpServer.close(() => r(undefined))
+  })
+}

package/tests/auth.test.ts

@@ -0,0 +1,155 @@
+import * as http from 'http'
+import express from 'express'
+import xrpc, { XRPCError } from '@atproto/xrpc'
+import { createServer, closeServer } from './_util'
+import * as xrpcServer from '../src'
+import { AuthRequiredError } from '../src'
+
+const LEXICONS = [
+  {
+    lexicon: 1,
+    id: 'io.example.authTest',
+    defs: {
+      main: {
+        type: 'procedure',
+        input: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            properties: {
+              present: { type: 'boolean', const: true },
+            },
+          },
+        },
+        output: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            properties: {
+              username: { type: 'string' },
+              original: { type: 'string' },
+            },
+          },
+        },
+      },
+    },
+  },
+]
+
+describe('Auth', () => {
+  let s: http.Server
+  const server = xrpcServer.createServer(LEXICONS)
+  server.method('io.example.authTest', {
+    auth: createBasicAuth({ username: 'admin', password: 'password' }),
+    handler: ({ auth }) => {
+      return {
+        encoding: 'application/json',
+        body: {
+          username: auth?.credentials?.username,
+          original: auth?.artifacts?.original,
+        },
+      }
+    },
+  })
+  const client = xrpc.service(`http://localhost:8894`)
+  xrpc.addLexicons(LEXICONS)
+  beforeAll(async () => {
+    s = await createServer(8894, server)
+  })
+  afterAll(async () => {
+    await closeServer(s)
+  })
+
+  it('fails on bad auth before invalid request payload.', async () => {
+    try {
+      await client.call(
+        'io.example.authTest',
+        {},
+        { present: false },
+        {
+          headers: basicAuthHeaders({
+            username: 'admin',
+            password: 'wrong',
+          }),
+        },
+      )
+      throw new Error('Didnt throw')
+    } catch (e: any) {
+      expect(e instanceof XRPCError).toBeTruthy()
+      expect(e.success).toBeFalsy()
+      expect(e.error).toBe('AuthenticationRequired')
+      expect(e.message).toBe('Authentication Required')
+      expect(e.status).toBe(401)
+    }
+  })
+
+  it('fails on invalid request payload after good auth.', async () => {
+    try {
+      await client.call(
+        'io.example.authTest',
+        {},
+        { present: false },
+        {
+          headers: basicAuthHeaders({
+            username: 'admin',
+            password: 'password',
+          }),
+        },
+      )
+      throw new Error('Didnt throw')
+    } catch (e: any) {
+      expect(e instanceof XRPCError).toBeTruthy()
+      expect(e.success).toBeFalsy()
+      expect(e.error).toBe('InvalidRequest')
+      expect(e.message).toBe('Input/present must be true')
+      expect(e.status).toBe(400)
+    }
+  })
+
+  it('succeeds on good auth and payload.', async () => {
+    const res = await client.call(
+      'io.example.authTest',
+      {},
+      { present: true },
+      {
+        headers: basicAuthHeaders({
+          username: 'admin',
+          password: 'password',
+        }),
+      },
+    )
+    expect(res.success).toBe(true)
+    expect(res.data).toEqual({
+      username: 'admin',
+      original: 'YWRtaW46cGFzc3dvcmQ=',
+    })
+  })
+})
+
+function createBasicAuth(allowed: { username: string; password: string }) {
+  return function (ctx: { req: express.Request }) {
+    const header = ctx.req.headers.authorization ?? ''
+    if (!header.startsWith('Basic ')) {
+      throw new AuthRequiredError()
+    }
+    const original = header.replace('Basic ', '')
+    const [username, password] = Buffer.from(original, 'base64')
+      .toString()
+      .split(':')
+    if (username !== allowed.username || password !== allowed.password) {
+      throw new AuthRequiredError()
+    }
+    return {
+      credentials: { username },
+      artifacts: { original },
+    }
+  }
+}
+
+function basicAuthHeaders(creds: { username: string; password: string }) {
+  return {
+    authorization:
+      'Basic ' +
+      Buffer.from(`${creds.username}:${creds.password}`).toString('base64'),
+  }
+}

package/tests/bodies.test.ts

@@ -0,0 +1,240 @@
+import * as http from 'http'
+import { Readable } from 'stream'
+import { gzipSync } from 'zlib'
+import xrpc from '@atproto/xrpc'
+import { bytesToStream, cidForData } from '@atproto/common'
+import { randomBytes } from '@atproto/crypto'
+import { createServer, closeServer } from './_util'
+import * as xrpcServer from '../src'
+import logger from '../src/logger'
+
+const LEXICONS = [
+  {
+    lexicon: 1,
+    id: 'io.example.validationTest',
+    defs: {
+      main: {
+        type: 'procedure',
+        input: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['foo'],
+            properties: {
+              foo: { type: 'string' },
+              bar: { type: 'number' },
+            },
+          },
+        },
+        output: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['foo'],
+            properties: {
+              foo: { type: 'string' },
+              bar: { type: 'number' },
+            },
+          },
+        },
+      },
+    },
+  },
+  {
+    lexicon: 1,
+    id: 'io.example.validationTest2',
+    defs: {
+      main: {
+        type: 'query',
+        output: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['foo'],
+            properties: {
+              foo: { type: 'string' },
+              bar: { type: 'number' },
+            },
+          },
+        },
+      },
+    },
+  },
+  {
+    lexicon: 1,
+    id: 'io.example.blobTest',
+    defs: {
+      main: {
+        type: 'procedure',
+        input: {
+          encoding: '*/*',
+        },
+        output: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['cid'],
+            properties: {
+              cid: { type: 'string' },
+            },
+          },
+        },
+      },
+    },
+  },
+]
+
+const BLOB_LIMIT = 5000
+
+describe('Bodies', () => {
+  let s: http.Server
+  const server = xrpcServer.createServer(LEXICONS, {
+    payload: {
+      blobLimit: BLOB_LIMIT,
+    },
+  })
+  server.method(
+    'io.example.validationTest',
+    (ctx: { params: xrpcServer.Params; input?: xrpcServer.HandlerInput }) => ({
+      encoding: 'json',
+      body: ctx.input?.body,
+    }),
+  )
+  server.method('io.example.validationTest2', () => ({
+    encoding: 'json',
+    body: { wrong: 'data' },
+  }))
+  server.method(
+    'io.example.blobTest',
+    async (ctx: { input?: xrpcServer.HandlerInput }) => {
+      if (!(ctx.input?.body instanceof Readable))
+        throw new Error('Input not readable')
+      const buffers: Buffer[] = []
+      for await (const data of ctx.input.body) {
+        buffers.push(data)
+      }
+      const cid = await cidForData(Buffer.concat(buffers))
+      return {
+        encoding: 'json',
+        body: { cid: cid.toString() },
+      }
+    },
+  )
+  const client = xrpc.service(`http://localhost:8892`)
+  xrpc.addLexicons(LEXICONS)
+  beforeAll(async () => {
+    s = await createServer(8892, server)
+  })
+  afterAll(async () => {
+    await closeServer(s)
+  })
+
+  it('validates input and output bodies', async () => {
+    const res1 = await client.call(
+      'io.example.validationTest',
+      {},
+      {
+        foo: 'hello',
+        bar: 123,
+      },
+    )
+    expect(res1.success).toBeTruthy()
+    expect(res1.data.foo).toBe('hello')
+    expect(res1.data.bar).toBe(123)
+
+    await expect(client.call('io.example.validationTest', {})).rejects.toThrow(
+      `A request body is expected but none was provided`,
+    )
+    await expect(
+      client.call('io.example.validationTest', {}, {}),
+    ).rejects.toThrow(`Input must have the property "foo"`)
+    await expect(
+      client.call('io.example.validationTest', {}, { foo: 123 }),
+    ).rejects.toThrow(`Input/foo must be a string`)
+
+    // 500 responses don't include details, so we nab details from the logger.
+    let error: string | undefined
+    const origError = logger.error
+    logger.error = (obj, ...args) => {
+      error = obj.message
+      logger.error = origError
+      return logger.error(obj, ...args)
+    }
+
+    await expect(client.call('io.example.validationTest2')).rejects.toThrow(
+      'Internal Server Error',
+    )
+    expect(error).toEqual(`Output must have the property "foo"`)
+  })
+
+  it('supports blobs and compression', async () => {
+    const bytes = randomBytes(1024)
+    const expectedCid = await cidForData(bytes)
+
+    const { data: uncompressed } = await client.call(
+      'io.example.blobTest',
+      {},
+      bytes,
+      {
+        encoding: 'application/octet-stream',
+      },
+    )
+    expect(uncompressed.cid).toEqual(expectedCid.toString())
+
+    const { data: compressed } = await client.call(
+      'io.example.blobTest',
+      {},
+      gzipSync(bytes),
+      {
+        encoding: 'application/octet-stream',
+        headers: {
+          'content-encoding': 'gzip',
+        },
+      },
+    )
+    expect(compressed.cid).toEqual(expectedCid.toString())
+  })
+
+  it('supports max blob size (based on content-length)', async () => {
+    const bytes = randomBytes(BLOB_LIMIT + 1)
+
+    // Exactly the number of allowed bytes
+    await client.call('io.example.blobTest', {}, bytes.slice(0, BLOB_LIMIT), {
+      encoding: 'application/octet-stream',
+    })
+
+    // Over the number of allowed bytes
+    const promise = client.call('io.example.blobTest', {}, bytes, {
+      encoding: 'application/octet-stream',
+    })
+
+    await expect(promise).rejects.toThrow('request entity too large')
+  })
+
+  it('supports max blob size (missing content-length)', async () => {
+    // We stream bytes in these tests so that content-length isn't included.
+    const bytes = randomBytes(BLOB_LIMIT + 1)
+
+    // Exactly the number of allowed bytes
+    await client.call(
+      'io.example.blobTest',
+      {},
+      bytesToStream(bytes.slice(0, BLOB_LIMIT)),
+      {
+        encoding: 'application/octet-stream',
+      },
+    )
+
+    // Over the number of allowed bytes.
+    const promise = client.call(
+      'io.example.blobTest',
+      {},
+      bytesToStream(bytes),
+      {
+        encoding: 'application/octet-stream',
+      },
+    )
+
+    await expect(promise).rejects.toThrow('request entity too large')
+  })
+})