@atproto/xrpc-server 0.0.1 → 0.2.0

package/src/stream/subscription.ts

@@ -0,0 +1,175 @@
+import { wait } from '@atproto/common'
+import { WebSocket, ClientOptions } from 'ws'
+import { byMessage } from './stream'
+import { CloseCode, DisconnectError } from './types'
+
+export class Subscription<T = unknown> {
+  constructor(
+    public opts: ClientOptions & {
+      service: string
+      method: string
+      maxReconnectSeconds?: number
+      signal?: AbortSignal
+      validate: (obj: unknown) => T | undefined
+      onReconnectError?: (
+        error: unknown,
+        n: number,
+        initialSetup: boolean,
+      ) => void
+      getParams?: () =>
+        | Record<string, unknown>
+        | Promise<Record<string, unknown> | undefined>
+        | undefined
+    },
+  ) {}
+
+  async *[Symbol.asyncIterator](): AsyncGenerator<T> {
+    let initialSetup = true
+    let reconnects: number | null = null
+    const maxReconnectMs = 1000 * (this.opts.maxReconnectSeconds ?? 64)
+    while (true) {
+      if (reconnects !== null) {
+        const duration = initialSetup
+          ? Math.min(1000, maxReconnectMs)
+          : backoffMs(reconnects++, maxReconnectMs)
+        await wait(duration)
+      }
+      const ws = await this.getSocket()
+      const ac = new AbortController()
+      if (this.opts.signal) {
+        forwardSignal(this.opts.signal, ac)
+      }
+      ws.once('open', () => {
+        initialSetup = false
+        reconnects = 0
+      })
+      ws.once('close', (code, reason) => {
+        if (code === CloseCode.Abnormal) {
+          // Forward into an error to distinguish from a clean close
+          ac.abort(
+            new AbnormalCloseError(`Abnormal ws close: ${reason.toString()}`),
+          )
+        }
+      })
+      try {
+        const cancelable = { signal: ac.signal }
+        for await (const message of byMessage(ws, cancelable)) {
+          const t = message.header.t
+          const clone =
+            message.body !== undefined ? { ...message.body } : undefined
+          if (clone !== undefined && t !== undefined) {
+            clone['$type'] = t.startsWith('#') ? this.opts.method + t : t
+          }
+          const result = this.opts.validate(clone)
+          if (result !== undefined) {
+            yield result
+          }
+        }
+      } catch (_err) {
+        const err = _err?.['code'] === 'ABORT_ERR' ? _err['cause'] : _err
+        if (err instanceof DisconnectError) {
+          // We cleanly end the connection
+          ws.close(err.wsCode)
+          break
+        }
+        ws.close() // No-ops if already closed or closing
+        if (isReconnectable(err)) {
+          reconnects ??= 0 // Never reconnect with a null
+          this.opts.onReconnectError?.(err, reconnects, initialSetup)
+          continue
+        } else {
+          throw err
+        }
+      }
+      break // Other side cleanly ended stream and disconnected
+    }
+  }
+
+  private async getSocket() {
+    const params = (await this.opts.getParams?.()) ?? {}
+    const query = encodeQueryParams(params)
+    const url = `${this.opts.service}/xrpc/${this.opts.method}?${query}`
+    return new WebSocket(url, this.opts)
+  }
+}
+
+export default Subscription
+
+class AbnormalCloseError extends Error {
+  code = 'EWSABNORMALCLOSE'
+}
+
+function isReconnectable(err: unknown): boolean {
+  // Network errors are reconnectable.
+  // AuthenticationRequired and InvalidRequest XRPCErrors are not reconnectable.
+  // @TODO method-specific XRPCErrors may be reconnectable, need to consider. Receiving
+  // an invalid message is not current reconnectable, but the user can decide to skip them.
+  if (!err || typeof err['code'] !== 'string') return false
+  return networkErrorCodes.includes(err['code'])
+}
+
+const networkErrorCodes = [
+  'EWSABNORMALCLOSE',
+  'ECONNRESET',
+  'ECONNREFUSED',
+  'ECONNABORTED',
+  'EPIPE',
+  'ETIMEDOUT',
+  'ECANCELED',
+]
+
+function backoffMs(n: number, maxMs: number) {
+  const baseSec = Math.pow(2, n) // 1, 2, 4, ...
+  const randSec = Math.random() - 0.5 // Random jitter between -.5 and .5 seconds
+  const ms = 1000 * (baseSec + randSec)
+  return Math.min(ms, maxMs)
+}
+
+function encodeQueryParams(obj: Record<string, unknown>): string {
+  const params = new URLSearchParams()
+  Object.entries(obj).forEach(([key, value]) => {
+    const encoded = encodeQueryParam(value)
+    if (Array.isArray(encoded)) {
+      encoded.forEach((enc) => params.append(key, enc))
+    } else {
+      params.set(key, encoded)
+    }
+  })
+  return params.toString()
+}
+
+// Adapted from xrpc, but without any lex-specific knowledge
+function encodeQueryParam(value: unknown): string | string[] {
+  if (typeof value === 'string') {
+    return value
+  }
+  if (typeof value === 'number') {
+    return value.toString()
+  }
+  if (typeof value === 'boolean') {
+    return value ? 'true' : 'false'
+  }
+  if (typeof value === 'undefined') {
+    return ''
+  }
+  if (typeof value === 'object') {
+    if (value instanceof Date) {
+      return value.toISOString()
+    } else if (Array.isArray(value)) {
+      return value.flatMap(encodeQueryParam)
+    } else if (!value) {
+      return ''
+    }
+  }
+  throw new Error(`Cannot encode ${typeof value}s into query params`)
+}
+
+function forwardSignal(signal: AbortSignal, ac: AbortController) {
+  if (signal.aborted) {
+    return ac.abort(signal.reason)
+  } else {
+    signal.addEventListener('abort', () => ac.abort(signal.reason), {
+      signal: ac.signal,
+    })
+  }
+}

package/src/stream/types.ts

@@ -0,0 +1,43 @@
+import { z } from 'zod'
+
+export enum FrameType {
+  Message = 1,
+  Error = -1,
+}
+
+export const messageFrameHeader = z.object({
+  op: z.literal(FrameType.Message), // Frame op
+  t: z.string().optional(), // Message body type discriminator
+})
+export type MessageFrameHeader = z.infer<typeof messageFrameHeader>
+
+export const errorFrameHeader = z.object({
+  op: z.literal(FrameType.Error),
+})
+export const errorFrameBody = z.object({
+  error: z.string(), // Error code
+  message: z.string().optional(), // Error message
+})
+export type ErrorFrameHeader = z.infer<typeof errorFrameHeader>
+export type ErrorFrameBody<T extends string = string> = { error: T } & z.infer<
+  typeof errorFrameBody
+>
+
+export const frameHeader = z.union([messageFrameHeader, errorFrameHeader])
+export type FrameHeader = z.infer<typeof frameHeader>
+
+export class DisconnectError extends Error {
+  constructor(
+    public wsCode: CloseCode = CloseCode.Policy,
+    public xrpcCode?: string,
+  ) {
+    super()
+  }
+}
+
+// https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1
+export enum CloseCode {
+  Normal = 1000,
+  Abnormal = 1006,
+  Policy = 1008,
+}

package/src/types.ts

@@ -1,7 +1,12 @@
+import { IncomingMessage } from 'http'
 import express from 'express'
 import { isHttpError } from 'http-errors'
 import zod from 'zod'
-import { ResponseType, ResponseTypeStrings } from '@atproto/xrpc'
+import {
+  ResponseType,
+  ResponseTypeStrings,
+  ResponseTypeNames,
+} from '@atproto/xrpc'
 
 export type Options = {
   validateResponse?: boolean
@@ -52,6 +57,13 @@ export type XRPCHandler = (ctx: {
   res: express.Response
 }) => Promise<HandlerOutput> | HandlerOutput | undefined
 
+export type XRPCStreamHandler = (ctx: {
+  auth: HandlerAuth | undefined
+  params: Params
+  req: IncomingMessage
+  signal: AbortSignal
+}) => AsyncIterable<unknown>
+
 export type AuthOutput = HandlerAuth | HandlerError
 
 export type AuthVerifier = (ctx: {
@@ -59,11 +71,20 @@ export type AuthVerifier = (ctx: {
   res: express.Response
 }) => Promise<AuthOutput> | AuthOutput
 
+export type StreamAuthVerifier = (ctx: {
+  req: IncomingMessage
+}) => Promise<AuthOutput> | AuthOutput
+
 export type XRPCHandlerConfig = {
   auth?: AuthVerifier
   handler: XRPCHandler
 }
 
+export type XRPCStreamHandlerConfig = {
+  auth?: StreamAuthVerifier
+  handler: XRPCStreamHandler
+}
+
 export class XRPCError extends Error {
   constructor(
     public type: ResponseType,
@@ -75,7 +96,7 @@ export class XRPCError extends Error {
 
   get payload() {
     return {
-      error: this.customErrorName,
+      error: this.customErrorName ?? this.typeName,
       message:
         this.type === ResponseType.InternalServerError
           ? this.typeStr // Do not respond with error details for 500s
@@ -83,6 +104,10 @@ export class XRPCError extends Error {
     }
   }
 
+  get typeName(): string | undefined {
+    return ResponseTypeNames[this.type]
+  }
+
   get typeStr(): string | undefined {
     return ResponseTypeStrings[this.type]
   }

package/src/util.ts

@@ -2,7 +2,13 @@ import { Readable, Transform } from 'stream'
 import { createDeflate, createGunzip } from 'zlib'
 import express from 'express'
 import mime from 'mime-types'
-import { Lexicons, LexXrpcProcedure, LexXrpcQuery } from '@atproto/lexicon'
+import {
+  jsonToLex,
+  Lexicons,
+  LexXrpcProcedure,
+  LexXrpcQuery,
+  LexXrpcSubscription,
+} from '@atproto/lexicon'
 import { forwardStreamErrors, MaxSizeChecker } from '@atproto/common'
 import {
   UndecodedParams,
@@ -17,7 +23,7 @@ import {
 } from './types'
 
 export function decodeQueryParams(
-  def: LexXrpcProcedure | LexXrpcQuery,
+  def: LexXrpcProcedure | LexXrpcQuery | LexXrpcSubscription,
   params: UndecodedParams,
 ): Params {
   const decoded: Params = {}
@@ -50,7 +56,7 @@ export function decodeQueryParam(
   if (type === 'string' || type === 'datetime') {
     return String(value)
   }
-  if (type === 'number') {
+  if (type === 'float') {
     return Number(String(value))
   } else if (type === 'integer') {
     return Number(String(value)) | 0
@@ -59,6 +65,18 @@ export function decodeQueryParam(
   }
 }
 
+export function getQueryParams(url = ''): Record<string, string | string[]> {
+  const { searchParams } = new URL(url ?? '', 'http://x')
+  const result: Record<string, string | string[]> = {}
+  for (const key of searchParams.keys()) {
+    result[key] = searchParams.getAll(key)
+    if (result[key].length === 1) {
+      result[key] = result[key][0]
+    }
+  }
+  return result
+}
+
 export function validateInput(
   nsid: string,
   def: LexXrpcProcedure | LexXrpcQuery,
@@ -88,7 +106,15 @@ export function validateInput(
     def.input?.encoding &&
     (!inputEncoding || !isValidEncoding(def.input?.encoding, inputEncoding))
   ) {
-    throw new InvalidRequestError(`Invalid request encoding: ${inputEncoding}`)
+    if (!inputEncoding) {
+      throw new InvalidRequestError(
+        `Request encoding (Content-Type) required but not provided`,
+      )
+    } else {
+      throw new InvalidRequestError(
+        `Wrong request encoding (Content-Type): ${inputEncoding}`,
+      )
+    }
   }
 
   if (!inputEncoding) {
@@ -99,7 +125,8 @@ export function validateInput(
   // if input schema, validate
   if (def.input?.schema) {
     try {
-      lexicons.assertValidXrpcInput(nsid, req.body)
+      const lexBody = req.body ? jsonToLex(req.body) : req.body
+      req.body = lexicons.assertValidXrpcInput(nsid, lexBody)
     } catch (e) {
       throw new InvalidRequestError(e instanceof Error ? e.message : String(e))
     }
@@ -157,7 +184,10 @@ export function validateOutput(
   // output schema
   if (def.output?.schema) {
     try {
-      lexicons.assertValidXrpcOutput(nsid, output?.body)
+      const result = lexicons.assertValidXrpcOutput(nsid, output?.body)
+      if (output) {
+        output.body = result
+      }
     } catch (e) {
       throw new InternalServerError(e instanceof Error ? e.message : String(e))
     }
@@ -167,8 +197,9 @@ export function validateOutput(
 }
 
 export function normalizeMime(v: string) {
-  const fullType = mime.contentType(v)
   if (!v) return false
+  const fullType = mime.contentType(v)
+  if (!fullType) return false
   const shortType = fullType.split(';')[0]
   if (!shortType) return false
   return shortType

package/tests/_util.ts

@@ -1,6 +1,7 @@
 import * as http from 'http'
 import express from 'express'
-import * as xrpc from '../src/index'
+import * as xrpc from '../src'
+import { AuthRequiredError } from '../src'
 
 export async function createServer(
   port: number,
@@ -18,3 +19,37 @@ export async function closeServer(httpServer: http.Server) {
     httpServer.close(() => r(undefined))
   })
 }
+
+export function createBasicAuth(allowed: {
+  username: string
+  password: string
+}) {
+  return function (ctx: { req: http.IncomingMessage }) {
+    const header = ctx.req.headers.authorization ?? ''
+    if (!header.startsWith('Basic ')) {
+      throw new AuthRequiredError()
+    }
+    const original = header.replace('Basic ', '')
+    const [username, password] = Buffer.from(original, 'base64')
+      .toString()
+      .split(':')
+    if (username !== allowed.username || password !== allowed.password) {
+      throw new AuthRequiredError()
+    }
+    return {
+      credentials: { username },
+      artifacts: { original },
+    }
+  }
+}
+
+export function basicAuthHeaders(creds: {
+  username: string
+  password: string
+}) {
+  return {
+    authorization:
+      'Basic ' +
+      Buffer.from(`${creds.username}:${creds.password}`).toString('base64'),
+  }
+}

package/tests/auth.test.ts

@@ -1,9 +1,13 @@
 import * as http from 'http'
-import express from 'express'
-import xrpc, { XRPCError } from '@atproto/xrpc'
-import { createServer, closeServer } from './_util'
+import getPort from 'get-port'
+import xrpc, { ServiceClient, XRPCError } from '@atproto/xrpc'
 import * as xrpcServer from '../src'
-import { AuthRequiredError } from '../src'
+import {
+  createServer,
+  closeServer,
+  createBasicAuth,
+  basicAuthHeaders,
+} from './_util'
 
 const LEXICONS = [
   {
@@ -51,10 +55,13 @@ describe('Auth', () => {
       }
     },
   })
-  const client = xrpc.service(`http://localhost:8894`)
   xrpc.addLexicons(LEXICONS)
+
+  let client: ServiceClient
   beforeAll(async () => {
-    s = await createServer(8894, server)
+    const port = await getPort()
+    s = await createServer(port, server)
+    client = xrpc.service(`http://localhost:${port}`)
   })
   afterAll(async () => {
     await closeServer(s)
@@ -75,7 +82,7 @@ describe('Auth', () => {
       )
       throw new Error('Didnt throw')
     } catch (e: any) {
-      expect(e instanceof XRPCError).toBeTruthy()
+      expect(e).toBeInstanceOf(XRPCError)
       expect(e.success).toBeFalsy()
       expect(e.error).toBe('AuthenticationRequired')
       expect(e.message).toBe('Authentication Required')
@@ -98,7 +105,7 @@ describe('Auth', () => {
       )
       throw new Error('Didnt throw')
     } catch (e: any) {
-      expect(e instanceof XRPCError).toBeTruthy()
+      expect(e).toBeInstanceOf(XRPCError)
       expect(e.success).toBeFalsy()
       expect(e.error).toBe('InvalidRequest')
       expect(e.message).toBe('Input/present must be true')
@@ -125,31 +132,3 @@ describe('Auth', () => {
     })
   })
 })
-
-function createBasicAuth(allowed: { username: string; password: string }) {
-  return function (ctx: { req: express.Request }) {
-    const header = ctx.req.headers.authorization ?? ''
-    if (!header.startsWith('Basic ')) {
-      throw new AuthRequiredError()
-    }
-    const original = header.replace('Basic ', '')
-    const [username, password] = Buffer.from(original, 'base64')
-      .toString()
-      .split(':')
-    if (username !== allowed.username || password !== allowed.password) {
-      throw new AuthRequiredError()
-    }
-    return {
-      credentials: { username },
-      artifacts: { original },
-    }
-  }
-}
-
-function basicAuthHeaders(creds: { username: string; password: string }) {
-  return {
-    authorization:
-      'Basic ' +
-      Buffer.from(`${creds.username}:${creds.password}`).toString('base64'),
-  }
-}

package/tests/bodies.test.ts

@@ -1,8 +1,9 @@
 import * as http from 'http'
 import { Readable } from 'stream'
 import { gzipSync } from 'zlib'
-import xrpc from '@atproto/xrpc'
-import { bytesToStream, cidForData } from '@atproto/common'
+import getPort from 'get-port'
+import xrpc, { ServiceClient } from '@atproto/xrpc'
+import { bytesToStream, cidForCbor } from '@atproto/common'
 import { randomBytes } from '@atproto/crypto'
 import { createServer, closeServer } from './_util'
 import * as xrpcServer from '../src'
@@ -22,7 +23,7 @@ const LEXICONS = [
             required: ['foo'],
             properties: {
               foo: { type: 'string' },
-              bar: { type: 'number' },
+              bar: { type: 'integer' },
             },
           },
         },
@@ -33,7 +34,7 @@ const LEXICONS = [
             required: ['foo'],
             properties: {
               foo: { type: 'string' },
-              bar: { type: 'number' },
+              bar: { type: 'integer' },
             },
           },
         },
@@ -53,7 +54,7 @@ const LEXICONS = [
             required: ['foo'],
             properties: {
               foo: { type: 'string' },
-              bar: { type: 'number' },
+              bar: { type: 'integer' },
             },
           },
         },
@@ -113,17 +114,22 @@ describe('Bodies', () => {
       for await (const data of ctx.input.body) {
         buffers.push(data)
       }
-      const cid = await cidForData(Buffer.concat(buffers))
+      const cid = await cidForCbor(Buffer.concat(buffers))
       return {
         encoding: 'json',
         body: { cid: cid.toString() },
       }
     },
   )
-  const client = xrpc.service(`http://localhost:8892`)
   xrpc.addLexicons(LEXICONS)
+
+  let client: ServiceClient
+  let url: string
   beforeAll(async () => {
-    s = await createServer(8892, server)
+    const port = await getPort()
+    s = await createServer(port, server)
+    url = `http://localhost:${port}`
+    client = xrpc.service(url)
   })
   afterAll(async () => {
     await closeServer(s)
@@ -151,6 +157,14 @@ describe('Bodies', () => {
     await expect(
       client.call('io.example.validationTest', {}, { foo: 123 }),
     ).rejects.toThrow(`Input/foo must be a string`)
+    await expect(
+      client.call(
+        'io.example.validationTest',
+        {},
+        { foo: 'hello', bar: 123 },
+        { encoding: 'image/jpeg' },
+      ),
+    ).rejects.toThrow(`Wrong request encoding (Content-Type): image/jpeg`)
 
     // 500 responses don't include details, so we nab details from the logger.
     let error: string | undefined
@@ -169,7 +183,7 @@ describe('Bodies', () => {
 
   it('supports blobs and compression', async () => {
     const bytes = randomBytes(1024)
-    const expectedCid = await cidForData(bytes)
+    const expectedCid = await cidForCbor(bytes)
 
     const { data: uncompressed } = await client.call(
       'io.example.blobTest',
@@ -237,4 +251,31 @@ describe('Bodies', () => {
 
     await expect(promise).rejects.toThrow('request entity too large')
   })
+
+  it('requires any parsable Content-Type for blob uploads', async () => {
+    // not a real mimetype, but correct syntax
+    await client.call('io.example.blobTest', {}, randomBytes(BLOB_LIMIT), {
+      encoding: 'some/thing',
+    })
+  })
+
+  // @TODO: figure out why this is failing dependent on the prev test being run
+  // https://github.com/bluesky-social/atproto/pull/550/files#r1106400413
+  it.skip('errors on an empty Content-type on blob upload', async () => {
+    // empty mimetype, but correct syntax
+    const res = await fetch(`${url}/xrpc/io.example.blobTest`, {
+      method: 'post',
+      headers: { 'Content-Type': '' },
+      body: randomBytes(BLOB_LIMIT),
+      // @ts-ignore see note in @atproto/xrpc/client.ts
+      duplex: 'half',
+    })
+    const resBody = await res.json()
+    const status = res.status
+    expect(status).toBe(400)
+    expect(resBody.error).toBe('InvalidRequest')
+    expect(resBody.message).toBe(
+      'Request encoding (Content-Type) required but not provided',
+    )
+  })
 })