@toa.io/extensions.exposition 1.0.0-alpha.28 → 1.0.0-alpha.283

package/source/HTTP/formats/index.ts

@@ -1,22 +1,36 @@
-import { type Buffer } from 'node:buffer'
-import * as json from './json'
-import * as yaml from './yaml'
-import * as msgpack from './msgpack'
-import * as text from './text'
+import * as json from './json.js'
+import * as yaml from './yaml.js'
+import * as msgpack from './msgpack.js'
+import * as text from './text.js'
+import * as form from './form.js'
 
+/** What a reply may be encoded as, and so what `accept` is negotiated against. */
 export const formats: Record<string, Format> = {
-  [msgpack.type]: msgpack,
-  [yaml.type]: yaml,
   [json.type]: json,
-  [text.type]: text
+  [yaml.type]: yaml,
+  [text.type]: text,
+  [msgpack.type]: msgpack
 }
 
 export const types = Object.keys(formats)
 
-export interface Format {
+/**
+ * What a request body may arrive as. Everything a reply may be, and a form besides — which
+ * is read and never written, so it is not something a reply can be asked for.
+ */
+export const decoders: Record<string, Decoder> = {
+  ...formats,
+  [form.type]: form
+}
+
+export interface Decoder {
   readonly type: string
+
+  decode: (buffer: Buffer, charset?: string) => any
+}
+
+export interface Format extends Decoder {
   readonly multipart: string
 
   encode: (value: any) => Buffer
-  decode: (buffer: Buffer) => any
 }

package/source/HTTP/formats/json.ts

@@ -1,10 +1,8 @@
-import { Buffer } from 'node:buffer'
-
 export const type = 'application/json'
 export const multipart = 'multipart/json'
 
-export function decode (buffer: Buffer): any {
-  const text = buffer.toString()
+export function decode (buffer: Buffer, charset = 'utf-8'): any {
+  const text = buffer.toString(charset as BufferEncoding)
 
   return JSON.parse(text)
 }

package/source/HTTP/formats/msgpack.ts

@@ -1,4 +1,3 @@
-import { type Buffer } from 'node:buffer'
 import { pack, unpack } from 'msgpackr'
 
 export function decode (buffer: Buffer): any {

package/source/HTTP/formats/text.ts

@@ -1,10 +1,8 @@
-import { Buffer } from 'node:buffer'
-
 export const type = 'text/plain'
 export const multipart = 'multipart/text'
 
-export function decode (buffer: Buffer): any {
-  return buffer.toString()
+export function decode (buffer: Buffer, charset = 'utf-8'): any {
+  return buffer.toString(charset as BufferEncoding)
 }
 
 export function encode (value: { toString: () => string }): Buffer {

package/source/HTTP/formats/yaml.test.ts

@@ -0,0 +1,19 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { encode } from './yaml.js'
+
+it('encodes error values as their public properties', () => {
+  const value = new (class InvalidUsernameError extends Error {
+    public readonly code = 'INVALID_USERNAME'
+    public override readonly message = 'Username is not meeting the requirements'
+  })()
+
+  assert.strictEqual(encode(value).toString(), 'code: INVALID_USERNAME\nmessage: Username is not meeting the requirements\n')
+})
+
+it('writes a value the way its toJSON says', () => {
+  const secret = { unwrap: () => 's3cret', toJSON: () => '<REDACTED>' }
+
+  assert.strictEqual(encode({ a: 1, b: secret }).toString(), 'a: 1\nb: <REDACTED>\n')
+})

package/source/HTTP/formats/yaml.ts

@@ -1,17 +1,37 @@
-import { Buffer } from 'node:buffer'
-import * as yaml from 'js-yaml'
+import { yaml } from '@toa.io/generic'
 
 export const type = 'application/yaml'
 export const multipart = 'multipart/yaml'
 
-export function decode (buffer: Buffer): any {
-  const text = buffer.toString()
+export function decode (buffer: Buffer, charset = 'utf-8'): any {
+  const text = buffer.toString(charset as BufferEncoding)
 
   return yaml.load(text)
 }
 
 export function encode (value: any): Buffer {
-  const text = yaml.dump(value, { lineWidth: -1, noRefs: true })
+  const serializable = value instanceof Error ? Object.assign({}, value) : represent(value)
+  const text = yaml.dump(serializable, { lineWidth: -1, noRefs: true })
 
   return Buffer.from(text)
 }
+
+/** What says how it is to be written — a redacted secret — is written that way, as in JSON. */
+function represent (value: unknown): unknown {
+  if (typeof value !== 'object' || value === null || value instanceof Date || Buffer.isBuffer(value))
+    return value
+
+  if ('toJSON' in value && typeof value.toJSON === 'function')
+    return value.toJSON()
+
+  if (Array.isArray(value))
+    return value.map(represent)
+
+  const object = value as Record<string, unknown>
+  const represented: Record<string, unknown> = {}
+
+  for (const key of Object.keys(object))
+    represented[key] = represent(object[key])
+
+  return represented
+}

package/source/HTTP/index.ts

@@ -1,4 +1,6 @@
-export * from './Server'
-export * from './messages'
-export * from './exceptions'
-export * from './Context'
+export * from './Server.js'
+export * from './messages.js'
+export * from './exceptions.js'
+export * from './Context.js'
+export * from './types.js'
+export * from './Probe.js'

package/source/HTTP/messages.test.ts

@@ -1,13 +1,19 @@
-import { Readable } from 'node:stream'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { PassThrough, Readable } from 'node:stream'
+import * as streamConsumers from 'node:stream/consumers'
 import { generate } from 'randomstring'
 import * as msgpack from 'msgpackr'
-import { read } from './messages'
-import { BadRequest, UnsupportedMediaType } from './exceptions'
-import { Timing } from './Timing'
-import type { Context } from './Context'
+import { multipart, read, type OutgoingMessage } from './messages.js'
+import { BadRequest, UnsupportedMediaType } from './exceptions.js'
+import { formats, types } from './formats/index.js'
+import { Timing } from './Timing.js'
+import type * as http from 'node:http'
+import type { Context } from './Context.js'
 
 beforeEach(() => {
-  jest.clearAllMocks()
+  resetCalls()
 })
 
 describe('read', () => {
@@ -19,7 +25,7 @@ describe('read', () => {
     const context = createContext(path, headers, json)
     const output = await read(context)
 
-    expect(output).toStrictEqual(input)
+    assert.deepStrictEqual(output, input)
   })
 
   it('should parse application/yaml', async () => {
@@ -29,7 +35,7 @@ describe('read', () => {
     const request = createContext(path, headers, yaml)
     const value = await read(request)
 
-    expect(value).toStrictEqual({ foo: 1 })
+    assert.deepStrictEqual(value, { foo: 1 })
   })
 
   it('should parse application/mskpack', async () => {
@@ -40,7 +46,7 @@ describe('read', () => {
     const request = createContext(path, headers, msg)
     const output = await read(request)
 
-    expect(output).toStrictEqual(input)
+    assert.deepStrictEqual(output, input)
   })
 
   it('should parse text/plain', async () => {
@@ -50,7 +56,18 @@ describe('read', () => {
     const request = createContext(path, headers, input)
     const output = await read(request)
 
-    expect(output).toStrictEqual(input)
+    assert.deepStrictEqual(output, input)
+  })
+
+  it('should parse application/json with charset parameter', async () => {
+    const path = generate()
+    const headers = { 'content-type': 'application/json; charset=utf-8' }
+    const input = { [generate()]: generate() }
+    const json = JSON.stringify(input)
+    const context = createContext(path, headers, json)
+    const output = await read(context)
+
+    assert.deepStrictEqual(output, input)
   })
 
   it('should throw on unsupported request media type', async () => {
@@ -58,7 +75,7 @@ describe('read', () => {
     const headers = { 'content-type': 'wtf/' + generate() }
     const request = createContext(path, headers)
 
-    await expect(read(request)).rejects.toThrow(UnsupportedMediaType)
+    await assert.rejects(read(request), UnsupportedMediaType)
   })
 
   it('should throw on malformed content', async () => {
@@ -67,23 +84,103 @@ describe('read', () => {
     const headers = { 'content-type': 'application/json' }
     const request = createContext(path, headers, text)
 
-    await expect(read(request)).rejects.toThrow(BadRequest)
+    await assert.rejects(read(request), BadRequest)
+  })
+
+  it('should parse application/x-www-form-urlencoded', async () => {
+    const headers = { 'content-type': 'application/x-www-form-urlencoded' }
+    const form = 'grant_type=authorization_code&code=SplxlO&scope=a+b'
+    const request = createContext(generate(), headers, form)
+
+    assert.deepStrictEqual(await read(request), {
+      grant_type: 'authorization_code',
+      code: 'SplxlO',
+      scope: 'a b'
+    })
+  })
+
+  it('should read a repeated form name as the list it is', async () => {
+    const headers = { 'content-type': 'application/x-www-form-urlencoded' }
+    const request = createContext(generate(), headers, 'resource=one&resource=two')
+
+    assert.deepStrictEqual(await read(request), { resource: ['one', 'two'] })
+  })
+
+  it('should not offer a form as a reply encoding', () => {
+    assert.ok(!types.includes('application/x-www-form-urlencoded'),
+      'a form is read and never written')
+  })
+
+  it('should output correct mulitpart format', async () => {
+    const response = new class extends PassThrough {
+      public readonly headers = new Headers()
+
+      public setHeader (key: string, value: string): this {
+        this.headers.set(key, value)
+
+        return this
+      }
+    }()
+
+    const context = { encoder: formats['text/plain'] } as unknown as Context
+    const message = { body: Readable.from(['Hello', 'New', 'World']) } as unknown as OutgoingMessage
+
+    const framed = multipart(message, context, response as unknown as http.ServerResponse)
+
+    const result = await streamConsumers.text(framed)
+
+    assert.strictEqual(result, [
+      '--cut',
+      '',
+      'ACK',
+      '--cut',
+      '',
+      'Hello',
+      '--cut',
+      '',
+      'New',
+      '--cut',
+      '',
+      'World',
+      '--cut',
+      '',
+      'FIN',
+      '--cut--'].join('\r\n'))
   })
 })
 
-export function createContext
-(url: string, headers: Record<string, string> = {}, content: string | Buffer = ''):
-jest.MockedObject<Context> {
-  const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content)
-  const stream = Readable.from(buffer)
+export function createContext (url: string, headers: Record<string, string> = {}, content: string | Buffer = ''):
+Context {
+  const data = Buffer.isBuffer(content) ? content : Buffer.from(content)
+  const stream = Readable.from(data)
+  let consumed = false
 
   const mock: Partial<Context> = {
     request: Object.assign(stream, {
       url,
       headers
     }) as unknown as Context['request'],
-    timing: new Timing(false)
+    url: new URL(url, 'https://host.local'),
+    timing: new Timing(),
+    buffer: async () => {
+      if (consumed)
+        throw new Error('Request body already consumed')
+
+      consumed = true
+
+      return data
+    }
   }
 
-  return mock as unknown as jest.MockedObject<Context>
+  return mock as unknown as Context
+}
+
+function resetCalls (target = [], seen = new Set()) {
+  if (target === null || typeof target !== 'object' || seen.has(target)) return
+
+  seen.add(target)
+
+  for (const value of Object.values(target))
+    if (typeof value === 'function' && value.mock !== undefined) value.mock.resetCalls()
+    else resetCalls(value, seen)
 }

package/source/HTTP/messages.ts

@@ -1,18 +1,40 @@
+import { readFileSync } from 'node:fs'
 import { Readable } from 'node:stream'
-import { buffer } from 'node:stream/consumers'
-import { formats } from './formats'
-import { BadRequest, NotAcceptable, UnsupportedMediaType } from './exceptions'
-import type { Context } from './Context'
-import type * as http from 'node:http'
-
-export function write
-(context: Context, response: http.ServerResponse, message: OutgoingMessage): void {
+import { pipeline } from 'node:stream/promises'
+import { createHash } from 'node:crypto'
+import * as contentType from 'content-type'
+import { console } from 'openspan'
+import { type Format, decoders } from './formats/index.js'
+import { BadRequest, NotAcceptable, UnsupportedMediaType } from './exceptions.js'
+import type { Context } from './Context.js'
+import type { ServerResponse } from './types.js'
+
+const server = `Exposition/${JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')).version}` +
+  ((process.env.TOA_CONTEXT === undefined ? '' : ` ${process.env.TOA_CONTEXT}`) +
+    (process.env.TOA_ENV === undefined ? '' : `/${process.env.TOA_ENV}`))
+
+const pending = new Map<string, PendingStream>()
+
+export async function write (context: Context, response: ServerResponse, message: OutgoingMessage): Promise<void> {
   for (const transform of context.pipelines.response)
-    transform(message)
+    await transform(message)
 
+  if (message?.status !== undefined)
+    response.statusCode = message.status
+
+  response.setHeader('server', server)
   message.headers?.forEach((value, key) => response.setHeader(key, value))
   context.timing.append(response)
 
+  if (response.destroyed) {
+    console.warn('Request destroyed prematurely', { path: context.url.pathname })
+
+    return
+  }
+
+  response.on('error', (exception: Error) =>
+    console.warn('HTTP response error', { path: context.url.pathname, exception }))
+
   if (message.body instanceof Readable)
     stream(message, context, response)
   else
@@ -20,27 +42,38 @@ export function write
 }
 
 export async function read (context: Context): Promise<any> {
-  const type = context.request.headers['content-type']
+  const header = context.request.headers['content-type']
 
-  if (type === undefined)
+  if (header === undefined)
     return undefined
 
-  if (!(type in formats))
+  const { type, parameters } = contentType.parse(header)
+
+  if (!(type in decoders))
     throw new UnsupportedMediaType()
 
-  const format = formats[type]
-  const buf = await context.timing.capture('buffer', buffer(context.request))
+  const format = decoders[type]
+  const buf = await context.buffer()
 
   try {
-    return format.decode(buf)
-  } catch {
+    return format.decode(buf, parameters.charset)
+  } catch (error: unknown) {
+    console.debug('Failed to decode message', {
+      path: context.url.pathname,
+      error: error?.toString?.()
+    })
+
     throw new BadRequest()
   }
 }
 
-function send
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+function send (message: OutgoingMessage, context: Context, response: ServerResponse): void {
   if (message.body === undefined || message.body === null) {
+    // a HEAD reply carries no body but must still report the length a GET would
+    // have returned, so a length already set by a directive is left alone
+    if (!response.hasHeader('content-length'))
+      response.setHeader('content-length', '0')
+
     response.end()
 
     return
@@ -51,29 +84,60 @@ function send
 
   const buf = context.encoder.encode(message.body)
 
-  response
-    .setHeader('content-type', context.encoder.type)
-    .appendHeader('vary', 'accept')
-    .end(buf)
+  if (message.etag === true && conditional(context, response, buf))
+    return
+
+  response.setHeader('content-type', context.encoder.type)
+  response.setHeader('content-length', buf.length.toString())
+  response.appendHeader('vary', 'accept')
+  response.end(buf)
 }
 
-function stream
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
-  const encoded = message.headers !== undefined && message.headers.has('content-type')
+/**
+ * Tags a reply that carries no version with a hash of the body being sent, and answers
+ * `304` when the client already has it. The tag is taken from the encoded body rather
+ * than from a serialization of its own, so it identifies the representation — which is
+ * what `vary` says.
+ */
+function conditional (context: Context, response: ServerResponse, buf: Buffer): boolean {
+  const etag = `"${createHash('sha256').update(buf).digest('hex')}"`
 
-  if (encoded)
-    message.body.pipe(response)
-  else
-    multipart(message, context, response)
+  response.setHeader('etag', etag)
 
-  message.body.on('error', (e: Error) => {
-    console.error(e)
-    response.end()
-  })
+  if (context.request.headers['if-none-match'] !== etag)
+    return false
+
+  response.setHeader('content-length', '0')
+  response.appendHeader('vary', 'accept')
+
+  response.statusCode = 304
+  response.end()
+
+  return true
 }
 
-function multipart
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+function stream (message: OutgoingMessage, context: Context, response: ServerResponse): void {
+  const encoded = message.headers !== undefined && message.headers.has('content-type')
+  const source: Readable = encoded ? message.body : multipart(message, context, response)
+
+  // not awaited: a reply that streams is written long after the request is answered, and a
+  // realtime subscription outlives the span the reply was produced in
+  //
+  // `pipeline` carries an error to every stage and destroys them. `pipe` leaves the stages it
+  // built behind, and an `error` on a stream nobody listens to is an uncaught exception.
+  pipeline(source, response)
+    .catch((exception: Error) =>
+      console.warn('Message stream error', { path: context.url.pathname, exception }))
+
+  if (context.debug)
+    debugStream(context, response)
+}
+
+/**
+ * Frames an object stream as `multipart/*`: an `ACK` part, the parts themselves, then `FIN`.
+ * The body is a `Readable`; `write` reached here by testing it.
+ */
+export function multipart (message: OutgoingMessage, context: Context, response: ServerResponse): Readable {
   if (context.encoder === null)
     throw new NotAcceptable()
 
@@ -81,10 +145,20 @@ function multipart
 
   response.setHeader('content-type', `${encoder.multipart}; boundary=${BOUNDARY}`)
 
-  message.body
-    .map((part: unknown) => Buffer.concat([CUT, encoder.encode(part), CRLF]))
-    .on('end', () => response.end(FINALCUT))
-    .pipe(response)
+  return Readable.from(frames(message.body as Readable, encoder))
+}
+
+async function * frames (body: Readable, encoder: Format): AsyncGenerator<Buffer> {
+  yield Buffer.concat([CUT, CRLF, encoder.encode('ACK'), CRLF, CUT])
+
+  for await (const part of body)
+    yield Buffer.concat([
+      CRLF /* indicates no boundary headers */,
+      encoder.encode(part),
+      CRLF,
+      CUT])
+
+  yield Buffer.concat([CRLF, encoder.encode('FIN'), CRLF, FINALCUT])
 }
 
 const BOUNDARY = 'cut'
@@ -92,10 +166,48 @@ const CUT = Buffer.from(`--${BOUNDARY}\r\n`)
 const CRLF = Buffer.from('\r\n')
 const FINALCUT = Buffer.from(`--${BOUNDARY}--`)
 
+const PENDING_DEBUG_INTERVAL = 30000
+
+let pendingInterval: NodeJS.Timeout | null = null
+
+function debugStream (context: Context, response: ServerResponse): void {
+  const ctx = { method: context.request.method, path: context.url.pathname }
+
+  console.debug('Stream opened', ctx)
+  pending.set(context.id, ctx)
+
+  response.on('close', () => {
+    console.debug('Stream closed', ctx)
+    pending.delete(context.id)
+
+    if (pending.size === 0) {
+      if (pendingInterval !== null)
+        clearInterval(pendingInterval)
+
+      pendingInterval = null
+    }
+  })
+
+  if (pendingInterval === null)
+    pendingInterval = setInterval(() =>
+      console.debug('Pending streams', { size: pending.size }),
+    PENDING_DEBUG_INTERVAL)
+}
+
 export interface OutgoingMessage {
   status?: number
   headers?: Headers
   body?: any
+
+  /** tag the response with a hash of the encoded body, see `conditional` */
+  etag?: boolean
+
+  /**
+   * Built by the gateway rather than returned by an operation, as a request is `authentic`
+   * when it was made by one: what is in it is the gateway's own, and the checks that answer
+   * for what an operation returns have nothing to say about it.
+   */
+  authentic?: boolean
 }
 
 export interface Query {
@@ -103,8 +215,15 @@ export interface Query {
 
   id?: string
   criteria?: string
+  search?: string
+  sample?: number
   sort?: string
   omit?: string
   limit?: string
   version?: number
 }
+
+interface PendingStream {
+  method: string
+  path: string
+}

package/source/HTTP/types.ts

@@ -0,0 +1,39 @@
+import type { Readable, Writable } from 'node:stream'
+import type { OutgoingHttpHeaders } from 'node:http'
+import type { IncomingHttpHeaders } from 'node:http2'
+
+/**
+ * What Exposition requires of a request. Both `http.IncomingMessage` and
+ * `http2.Http2ServerRequest` satisfy it, so the pipeline is written once.
+ */
+export interface IncomingMessage extends Readable {
+  url: string
+  method: string
+  complete: boolean
+  headers: IncomingHeaders
+  socket: { remoteAddress?: string | undefined }
+}
+
+/**
+ * What Exposition requires of a response.
+ *
+ * `setHeader` and `appendHeader` return `void` because `Http2ServerResponse` returns
+ * nothing from them, so neither can be chained.
+ */
+export interface ServerResponse extends Writable {
+  statusCode: number
+  writableEnded: boolean
+  setHeader: (name: string, value: number | string | string[]) => void
+  appendHeader: (name: string, value: string | string[]) => void
+  hasHeader: (name: string) => boolean
+  writeHead: (status: number, headers?: OutgoingHttpHeaders) => ServerResponse
+}
+
+/**
+ * HTTP/2's header map: HTTP/1.1's, plus the pseudo-headers. `:authority` is typed here
+ * so the authority can be read without narrowing at every call site.
+ */
+export type IncomingHeaders = IncomingHttpHeaders
+
+/** `h2c` is cleartext HTTP/2: there is no in-process TLS to negotiate ALPN with. */
+export type Protocol = 'h1' | 'h2c'