@toa.io/extensions.exposition 1.0.0-alpha.28 → 1.0.0-alpha.282

package/source/HTTP/Server.ts

@@ -1,102 +1,208 @@
+import assert from 'node:assert'
 import fs from 'node:fs'
 import os from 'node:os'
 import * as http from 'node:http'
+import * as http2 from 'node:http2'
 import { once } from 'node:events'
 import { setTimeout } from 'node:timers/promises'
+import { console, current, decide, decode, run, type SpanContext } from 'openspan'
 import { Connector } from '@toa.io/core'
-import { type OutgoingMessage, write } from './messages'
-import { ClientError, Exception } from './exceptions'
-import { Context } from './Context'
-import type { IncomingMessage } from './Context'
+import { type OutgoingMessage, write } from './messages.js'
+import { ClientError, Exception } from './exceptions.js'
+import { Context } from './Context.js'
+import { PROBE, Probe } from './Probe.js'
+import type { IncomingMessage, Protocol, ServerResponse } from './types.js'
+import type { Bouncer } from '../Annotation.js'
 
 export class Server extends Connector {
-  private readonly server: http.Server = http.createServer()
+  private readonly server: http.Server | http2.Http2Server
   private readonly properties: Properties
   private readonly authorities: Record<string, string>
-  private process?: Processing
-  private ready: boolean = false
-  private startedAt: number = 0
+
+  /** Tracked for the drain: `Http2Server` has no `closeIdleConnections`. */
+  private readonly sessions = new Set<http2.ServerHttp2Session>()
+
+  private readonly probe: Probe
+
+  private process?: Processor
 
   private constructor (properties: Properties) {
     super()
 
     this.properties = properties
     this.authorities = Object.fromEntries(Object.entries(properties.authorities).map(([key, value]) => [value, key]))
+    this.server = instantiate(properties.protocol)
+    this.probe = new Probe(properties.probe)
+
+    this.server.on('request', (req, res) =>
+      this.listener(req as unknown as IncomingMessage, res as unknown as ServerResponse))
 
-    this.server.on('request', (req, res) => this.listener(req, res))
+    if (properties.protocol === 'h1') this.h1(this.server as http.Server)
+    else this.h2(this.server as http2.Http2Server)
   }
 
   public static create (options: Options): Server {
-    const properties: Properties = Object.assign({}, DEFAULTS, options)
+    const properties: Properties = { ...DEFAULTS, ...options }
 
     return new Server(properties)
   }
 
-  public attach (process: Processing): void {
+  public attach (process: Processor): void {
     this.process = process
   }
 
   protected override async open (): Promise<void> {
-    this.startedAt = Date.now()
+    // answers 503 from here; the gateway's dependencies have settled by the time `open` runs
+    await this.probe.listen()
+
     this.server.listen(this.properties.port)
 
     await once(this.server, 'listening')
 
     console.info('HTTP Server is listening')
 
-    await setTimeout(this.properties.delay)
-
-    this.ready = true
+    this.probe.complete()
 
     console.info('Ready')
   }
 
   protected override async close (): Promise<void> {
+    await this.probe.close()
+
     this.server.close()
-    this.ready = false
 
-    console.info('HTTP Server stopped accepting new connections')
+    // GOAWAY lets in-flight streams finish; `Http2Server` has no `closeIdleConnections`
+    if (this.properties.protocol === 'h1')
+      (this.server as http.Server).closeIdleConnections()
+    else
+      for (const session of this.sessions) session.close()
+
+    console.info('Stopped accepting new connections')
 
-    await once(this.server, 'close')
+    // keep-alive clients hold connections open indefinitely, so the drain is bounded
+    await Promise.race([
+      once(this.server, 'close'),
+      setTimeout(this.properties.drain, undefined, { ref: false })
+    ])
 
-    console.info('HTTP Server has been stopped')
+    if (this.properties.protocol === 'h1')
+      (this.server as http.Server).closeAllConnections()
+    else
+      for (const session of this.sessions) session.destroy()
+
+    console.info('Stopped')
   }
 
-  private listener (request: http.IncomingMessage, response: http.ServerResponse): void {
-    if (request.method === undefined || !this.properties.methods.has(request.method)) {
-      response.writeHead(501).end()
+  /** A malformed HTTP/1.1 request has no framing to answer in, so the status line is written by hand. */
+  private h1 (server: http.Server): void {
+    server.on('clientError', (error, socket) => {
+      console.warn('Client connection error', error)
+
+      if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\n\r\n')
+      else socket.destroy()
+    })
+  }
+
+  /** HTTP/2 has no `clientError`: failures surface per session, per stream, or per frame. */
+  private h2 (server: http2.Http2Server): void {
+    server.on('session', (session) => {
+      this.sessions.add(session)
+      session.on('close', () => this.sessions.delete(session))
+    })
+
+    server.on('sessionError', (error) => console.warn('Session error', error))
+    server.on('streamError', (error) => console.warn('Stream error', error))
+
+    server.on('unknownProtocol', (socket) => {
+      console.warn('Unknown protocol')
+      socket.destroy()
+    })
+  }
+
+  private listener (request: IncomingMessage, response: ServerResponse): void {
+    request.once('error', (error) => {
+      console.warn('Request error', errorAttributes(request, error))
+
+      if (!response.writableEnded)
+        response.destroy()
+    })
+
+    // no listener on `request.socket`: under HTTP/2 it is the session's socket, shared by
+    // every concurrent stream, and removing listeners on it would strip the siblings'
+
+    const host = authorityOf(request)?.toLowerCase()
+
+    if (host === undefined || !HOST.test(host)) {
+      console.warn('Request without a valid authority', errorAttributes(request, new Error('Invalid authority')))
+
+      response.writeHead(400).end()
 
       return
     }
 
-    if (request.url === '/.ready') {
-      if (this.ready)
-        response.writeHead(200, { 'cache-control': 'no-store' }).end()
-      else {
-        const remaining = (Math.ceil((Date.now() - this.startedAt) / 1000)).toString()
+    // directives and components read `host`; HTTP/2 sends `:authority` instead, and what a
+    // component sees must not depend on the protocol the request arrived over
+    request.headers.host ??= host
 
-        response.writeHead(503, { 'retry-after': remaining }).end()
-      }
+    const url = parse(request, host)
+
+    if (url instanceof Error) {
+      console.warn('Invalid request', errorAttributes(request, url))
+
+      response.writeHead(400).end()
 
       return
     }
 
-    if (request.headers.host === undefined || !(request.headers.host in this.authorities)) {
-      response.writeHead(404).end('Unknown authority')
+    if (request.method === undefined || !this.properties.methods.has(request.method)) {
+      response.writeHead(501).end()
 
       return
     }
 
-    const authority = this.authorities[request.headers.host]
-    const context = new Context(authority, request as IncomingMessage, this.properties)
+    assert(this.process !== undefined, 'Request processor is not attached')
+
+    const authority = this.authorities[host] ?? host
+
+    // if the request carries no trace context, the trace starts here
+    const remote = trace(request.headers)
+
+    const processing = remote === null
+      ? this.serve(request, response, authority, url)
+      : run(remote, async () => await this.serve(request, response, authority, url))
 
-    this.process!(context)
-      .then(this.success(context, response))
-      .catch(this.fail(context, response))
+    processing.catch((error) => {
+      console.error('Request processing failed', error)
+
+      if (!response.writableEnded)
+        response.writeHead(500).end()
+    })
+  }
+
+  // eslint-disable-next-line max-params
+  private async serve (request: IncomingMessage,
+    response: ServerResponse,
+    authority: string,
+    url: URL): Promise<void> {
+    await console.span({
+      name: `${request.method} ${request.url}`,
+      kind: 'server',
+      service: 'exposition',
+      attributes: { method: request.method, url: request.url, authority }
+    }, async () => {
+      response.setHeader('ray', current()!.traceId)
+
+      const context = new Context(authority, request, this.properties, url)
+
+      await this.process!(context)
+        .then(this.success(context, response))
+        .catch(this.fail(context, response))
+        .finally(() => request.removeAllListeners('error'))
+    })
   }
 
-  private success (context: Context, response: http.ServerResponse) {
-    return (message: OutgoingMessage) => {
+  private success (context: Context, response: ServerResponse) {
+    return async (message: OutgoingMessage) => {
       let status = message.status
 
       if (status === undefined)
@@ -104,68 +210,193 @@ export class Server extends Connector {
           status = 404
         else if (context.request.method === 'POST')
           status = 201
-        else if (message.body === undefined)
+        else if (message.body === undefined && context.request.method !== 'HEAD')
           status = 204
         else
           status = 200
 
-      response.statusCode = message.status = status
-      write(context, response, message)
+      message.status = status
+
+      await write(context, response, message)
     }
   }
 
-  private fail (context: Context, response: http.ServerResponse) {
+  private fail (context: Context, response: ServerResponse) {
     return async (exception: Error) => {
-      if (!context.request.complete)
-        await adam(context.request)
+      try {
+        // Over HTTP/2 the reply is followed by RST_STREAM(NO_ERROR), which tells the client
+        // to stop sending without discarding the response — so the body is never read.
+        if (!context.request.complete && this.properties.protocol === 'h1')
+          await adam(context.request)
+
+        const status = exception instanceof Exception ? exception.status : 500
+        const span = current()
+
+        // https://opentelemetry.io/docs/specs/semconv/http/http-spans/#status
+        if (status >= 500 && span !== undefined)
+          span.status = 'error'
+
+        if (!response.writableEnded) {
+          response.statusCode = status
+
+          const message: OutgoingMessage = { status: response.statusCode }
+
+          // eslint-disable-next-line max-depth
+          if (exception instanceof Exception && exception.headers !== undefined)
+            message.headers = exception.headers
+
+          // eslint-disable-next-line max-depth
+          if (context.encoder === null)
+            message.body = undefined
+          else if (exception instanceof ClientError || this.properties.debug)
+            message.body =
+              exception instanceof Exception
+                ? exception.body
+                : (this.properties.debug && exception.stack) ?? exception.message
+
+          await write(context, response, message)
+        }
+      } catch (final) {
+        console.error('Error in error handler', final)
+
+        if (!response.writableEnded)
+          try {
+            response.writeHead(500).end()
+          } catch {
+            // Nothing more we can do
+          }
+      }
+    }
+  }
+}
 
-      response.statusCode = exception instanceof Exception ? exception.status : 500
+function instantiate (protocol: Protocol): http.Server | http2.Http2Server {
+  if (protocol === 'h1')
+    return http.createServer()
 
-      const message: OutgoingMessage = { status: response.statusCode }
-      const verbose = exception instanceof ClientError || this.properties.debug
+  return http2.createServer({
+    // realtime pins one stream per subscription, and they all share a session
+    maxSessionMemory: SESSION_MEMORY,
+    settings: { initialWindowSize: WINDOW }
+  })
+}
 
-      if (verbose)
-        message.body =
-          exception instanceof Exception
-            ? exception.body
-            : exception.stack ?? exception.message
+/**
+ * The authority the request is addressed to. HTTP/2 carries it in `:authority` and omits
+ * `host` entirely, so reading `host` alone would leave every HTTP/2 request unattributed.
+ */
+function authorityOf (request: IncomingMessage): string | undefined {
+  return request.headers[':authority'] ?? request.headers.host
+}
 
-      write(context, response, message)
-    }
+/** Parsing the URL is how a request is validated, so the `Context` is handed the result. */
+function parse (request: IncomingMessage, authority: string): URL | Error {
+  try {
+    return new URL(request.url, `https://${authority}`)
+  } catch (error) {
+    return error as Error
   }
 }
 
 // https://github.com/whatwg/fetch/issues/1254
-async function adam (request: http.IncomingMessage): Promise<any> {
+async function adam (request: IncomingMessage): Promise<void> {
   const devnull = fs.createWriteStream(os.devNull)
 
-  request.pipe(devnull)
+  devnull.on('error', () => undefined)
+  request.pipe(devnull).on('error', () => undefined)
 
-  return once(request, 'end')
+  await once(request, 'end')
+}
+
+function errorAttributes (request: IncomingMessage, error: Error & any): RequestErrorAttributes {
+  const attributes: RequestErrorAttributes = {
+    path: request.url,
+    method: request.method,
+    name: error.name
+  }
+
+  if (typeof error.code === 'string')
+    attributes.code = error.code
+
+  if (typeof error.stack === 'string')
+    attributes.stack = error.stack
+
+  return attributes
 }
 
 export const PORT = 8000
-export const DELAY = 3 // seconds
+
+export const DRAIN = 10 // seconds
+
+/** Megabytes a single HTTP/2 session may hold, over Node's default of 10. */
+const SESSION_MEMORY = 128
+
+/** Per-stream flow control window. The default 64 KiB throttles in proportion to RTT. */
+const WINDOW = 1024 * 1024
+
+/**
+ * Extracts the remote trace context from the request headers.
+ *
+ * The `ray` header adopts the trace by ID only and does not bypass sampling:
+ * the sampling decision is made by the server.
+ */
+function trace (headers: IncomingMessage['headers']): SpanContext | null {
+  if (typeof headers.traceparent === 'string')
+    return decode(headers.traceparent)
+
+  // adopting a trace by ID does not bypass sampling
+  if (typeof headers.ray === 'string' && RAY.test(headers.ray) && headers.ray !== ZERO_RAY)
+    return { traceId: headers.ray.toLowerCase(), sampled: decide() }
+
+  return null
+}
+
+/**
+ * A host name with an optional port, or a bracketed IPv6 literal. The URL parser admits
+ * `,` `;` `=` `(` `)` and quotes in a host, and the authority is written into criteria.
+ */
+const HOST = /^(?:[a-z0-9.-]{1,253}|\[[0-9a-f:.]{2,45}\])(?::\d{1,5})?$/
+
+const RAY = /^[\da-f]{32}$/i
+const ZERO_RAY = '0'.repeat(32)
 
 const DEFAULTS: Omit<Properties, 'authorities'> = {
-  methods: new Set<string>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']),
+  methods: new Set<string>(['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'LOCK', 'UNLOCK']),
   debug: false,
-  trace: false,
   port: PORT,
-  delay: DELAY * 1000
+  drain: DRAIN * 1000,
+  protocol: 'h1',
+  probe: PROBE
 }
 
 interface Properties {
   authorities: Record<string, string>
   methods: Set<string>
   debug: boolean
-  trace: boolean
   port: number
-  delay: number
+  drain: number
+  protocol: Protocol
+
+  /** Port of the readiness probe, which is HTTP/1.1 whatever the gateway serves. */
+  probe: number
+
+  /** The header the client address is read from; the connection's address without one. */
+  ip?: string
+
+  /** Failed authentications an address may make; none are metered unless set. */
+  bouncer?: Bouncer
 }
 
 export type Options = { authorities: Properties['authorities'] } & {
   [K in Exclude<keyof Properties, 'authorities'>]?: Properties[K]
 }
 
-export type Processing = (input: Context) => Promise<OutgoingMessage>
+export type Processor = (input: Context) => Promise<OutgoingMessage>
+
+interface RequestErrorAttributes {
+  path: string
+  method: string
+  name: string
+  code?: string
+  stack?: string
+}

package/source/HTTP/Timing.ts

@@ -1,19 +1,11 @@
 import { performance } from 'node:perf_hooks'
-import type { ServerResponse } from 'node:http'
+import type { ServerResponse } from './types.js'
 
 export class Timing {
-  private readonly skip: boolean
   private readonly start = performance.now()
   private readonly breakpoints: Breakpoint[] = []
 
-  public constructor (enabled: boolean) {
-    this.skip = !enabled
-  }
-
-  public async capture<T> (id: string, promise: Promise<T>): Promise<T> {
-    if (this.skip)
-      return promise
-
+  public async capture<T>(id: string, promise: Promise<T>): Promise<T> {
     const start = performance.now()
     const result = promise instanceof Promise ? await promise : promise
 
@@ -23,9 +15,6 @@ export class Timing {
   }
 
   public append (response: ServerResponse): void {
-    if (this.skip)
-      return
-
     this.breakpoints.push({ id: 'total', duration: performance.now() - this.start })
 
     for (const breakpoint of this.breakpoints)

package/source/HTTP/address.test.ts

@@ -0,0 +1,22 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { address } from './address.js'
+import type { IncomingMessage } from './types.js'
+
+const request = (headers: Record<string, string>): IncomingMessage =>
+  ({ headers, socket: { remoteAddress: '9.9.9.9' } }) as unknown as IncomingMessage
+
+it('should be nothing without a header named', () => {
+  assert.equal(address(request({ 'x-forwarded-for': '1.1.1.1' })), undefined)
+})
+
+it('should be the last value of the named header', () => {
+  assert.equal(address(request({ 'x-forwarded-for': '1.1.1.1, 2.2.2.2' }), 'x-forwarded-for'), '2.2.2.2')
+  assert.equal(address(request({ 'cf-connecting-ip': '3.3.3.3' }), 'cf-connecting-ip'), '3.3.3.3')
+})
+
+it('should be nothing when the named header is absent or empty', () => {
+  assert.equal(address(request({}), 'x-real-ip'), undefined)
+  assert.equal(address(request({ 'x-real-ip': ' ' }), 'x-real-ip'), undefined)
+})

package/source/HTTP/address.ts

@@ -0,0 +1,17 @@
+import type { IncomingMessage } from './types.js'
+
+/**
+ * The client address: the value of the header the deployment names, and nothing otherwise.
+ * Of a header holding a list, the last value is the one the edge in front appended. The
+ * connection's address is never it: behind an edge it is the edge's.
+ */
+export function address (request: IncomingMessage, header?: string): string | undefined {
+  if (header === undefined)
+    return
+
+  const value = request.headers[header]
+  const raw = Array.isArray(value) ? value[value.length - 1] : value
+  const last = raw?.slice(raw.lastIndexOf(',') + 1).trim()
+
+  return last === '' ? undefined : last
+}

package/source/HTTP/exceptions.ts

@@ -2,10 +2,14 @@ export class Exception extends Error {
   public readonly status: number
   public readonly body?: any
 
-  protected constructor (status: number, body?: any) {
+  /** Headers the status is not complete without, like `Retry-After` on a 429. */
+  public readonly headers?: Headers
+
+  protected constructor (status: number, body?: any, headers?: Headers) {
     super()
     this.status = status
     this.body = body
+    this.headers = headers
   }
 }
 
@@ -36,27 +40,33 @@ export class NotFound extends ClientError {
   }
 }
 
-export class Conflict extends ClientError {
-  public constructor (body?: any) {
-    super(409, body)
+export class MethodNotAllowed extends ClientError {
+  public constructor () {
+    super(405)
   }
 }
 
-export class UnprocessableEntity extends ClientError {
+export class NotAcceptable extends ClientError {
+  public constructor () {
+    super(406)
+  }
+}
+
+export class Conflict extends ClientError {
   public constructor (body?: any) {
-    super(422, body)
+    super(409, body)
   }
 }
 
-export class MethodNotAllowed extends ClientError {
+export class PreconditionFailed extends ClientError {
   public constructor () {
-    super(405)
+    super(412)
   }
 }
 
-export class NotAcceptable extends ClientError {
-  public constructor () {
-    super(406)
+export class RequestEntityTooLarge extends ClientError {
+  public constructor (body?: any) {
+    super(413, body)
   }
 }
 
@@ -66,8 +76,22 @@ export class UnsupportedMediaType extends ClientError {
   }
 }
 
-export class PreconditionFailed extends ClientError {
-  public constructor () {
-    super(412)
+export class UnprocessableEntity extends ClientError {
+  public constructor (body?: any) {
+    super(422, body)
+  }
+}
+
+export class TooManyRequests extends ClientError {
+  /** @param retry Seconds until the request would be admitted. */
+  public constructor (retry?: number) {
+    super(429, undefined,
+      retry === undefined ? undefined : new Headers({ 'retry-after': String(retry) }))
+  }
+}
+
+export class ServiceUnavailable extends Exception {
+  public constructor (body?: any) {
+    super(503, body)
   }
 }

package/source/HTTP/formats/index.ts

@@ -1,14 +1,13 @@
-import { type Buffer } from 'node:buffer'
-import * as json from './json'
-import * as yaml from './yaml'
-import * as msgpack from './msgpack'
-import * as text from './text'
+import * as json from './json.js'
+import * as yaml from './yaml.js'
+import * as msgpack from './msgpack.js'
+import * as text from './text.js'
 
 export const formats: Record<string, Format> = {
-  [msgpack.type]: msgpack,
-  [yaml.type]: yaml,
   [json.type]: json,
-  [text.type]: text
+  [yaml.type]: yaml,
+  [text.type]: text,
+  [msgpack.type]: msgpack
 }
 
 export const types = Object.keys(formats)
@@ -18,5 +17,5 @@ export interface Format {
   readonly multipart: string
 
   encode: (value: any) => Buffer
-  decode: (buffer: Buffer) => any
+  decode: (buffer: Buffer, charset?: string) => any
 }

package/source/HTTP/formats/json.ts

@@ -1,10 +1,8 @@
-import { Buffer } from 'node:buffer'
-
 export const type = 'application/json'
 export const multipart = 'multipart/json'
 
-export function decode (buffer: Buffer): any {
-  const text = buffer.toString()
+export function decode (buffer: Buffer, charset = 'utf-8'): any {
+  const text = buffer.toString(charset as BufferEncoding)
 
   return JSON.parse(text)
 }

package/source/HTTP/formats/msgpack.ts

@@ -1,4 +1,3 @@
-import { type Buffer } from 'node:buffer'
 import { pack, unpack } from 'msgpackr'
 
 export function decode (buffer: Buffer): any {