@toa.io/extensions.exposition 1.0.0-alpha.29 → 1.0.0-alpha.291

package/source/HTTP/Probe.ts

@@ -0,0 +1,99 @@
+import * as http from 'node:http'
+import { console } from 'openspan'
+
+/**
+ * The readiness probe, on a port of its own.
+ *
+ * It cannot share the port the gateway serves traffic on: kubelet speaks HTTP/1.1 only, and a
+ * cleartext HTTP/2 server answers nothing else on its port — there is no ALPN to negotiate with.
+ *
+ * It is the gateway's own readiness, not the process's. The identity composition nested in this
+ * process connects before route discovery has settled, so a probe tied to that composition would
+ * report ready while the gateway still had no routes and no open port.
+ */
+export class Probe {
+  private readonly server = http.createServer()
+  private readonly port: number
+  private readonly path: string
+
+  private ready = false
+  private startedAt = 0
+  private listening = false
+  private skipped = false
+
+  public constructor(port: number, path: string = PATH) {
+    this.port = port
+    this.path = path
+
+    this.server.on('request', (request, response) => this.listener(request, response))
+  }
+
+  public async listen(): Promise<void> {
+    this.startedAt = Date.now()
+
+    try {
+      await new Promise<void>((resolve, reject) => {
+        this.server.once('error', reject)
+        this.server.listen(this.port, () => {
+          this.server.removeListener('error', reject)
+          resolve()
+        })
+      })
+    } catch (error: any) {
+      // processes sharing a host share the port; in a pod nothing else claims it
+      if (error?.code === 'EADDRINUSE') {
+        this.skipped = true
+
+        console.warn('Ready probe port already in use, skipping', { port: this.port })
+
+        return
+      }
+
+      throw error
+    }
+
+    this.listening = true
+
+    // a probe answers while the process runs; it must never be the reason it keeps running
+    this.server.unref()
+  }
+
+  /** The gateway is listening and has routes: answer `200`. */
+  public complete(): void {
+    this.ready = true
+
+    // the IPC signal is not tied to the probe: a process that gave up the shared port is
+    // still ready, and pm2 `wait_ready` would otherwise block until `listen_timeout`
+    process.send?.('ready')
+  }
+
+  public async close(): Promise<void> {
+    this.ready = false
+
+    if (!this.listening || this.skipped) return
+
+    this.listening = false
+
+    // keep-alive connections would otherwise hold the server handle, delaying the exit
+    this.server.closeAllConnections()
+
+    await new Promise<void>((resolve) => this.server.close(() => resolve()))
+  }
+
+  private listener(request: http.IncomingMessage, response: http.ServerResponse): void {
+    if (request.url !== this.path) {
+      response.writeHead(404).end()
+
+      return
+    }
+
+    if (this.ready) response.writeHead(200, { 'cache-control': 'no-store' }).end()
+    else {
+      const remaining = Math.ceil((Date.now() - this.startedAt) / 1000).toString()
+
+      response.writeHead(503, { 'retry-after': remaining }).end()
+    }
+  }
+}
+
+export const PATH = '/.ready'

package/source/HTTP/Server.ts

@@ -1,171 +1,430 @@
+import assert from 'node:assert'
 import fs from 'node:fs'
 import os from 'node:os'
 import * as http from 'node:http'
+import * as http2 from 'node:http2'
 import { once } from 'node:events'
 import { setTimeout } from 'node:timers/promises'
+import { console, current, decide, decode, run, type SpanContext } from 'openspan'
 import { Connector } from '@toa.io/core'
-import { type OutgoingMessage, write } from './messages'
-import { ClientError, Exception } from './exceptions'
-import { Context } from './Context'
-import type { IncomingMessage } from './Context'
+import { type OutgoingMessage, write } from './messages.js'
+import { ClientError, Exception } from './exceptions.js'
+import { Context } from './Context.js'
+import { Probe } from './Probe.js'
+import { PORT, PROBE } from '../const.js'
+import type { IncomingMessage, Protocol, ServerResponse } from './types.js'
+import type { Bouncer, MCP, OAuth, RPC } from '../Annotation.js'
 
 export class Server extends Connector {
-  private readonly server: http.Server = http.createServer()
+  private readonly server: http.Server | http2.Http2Server
   private readonly properties: Properties
-  private readonly authorities: Record<string, string>
-  private process?: Processing
-  private ready: boolean = false
-  private startedAt: number = 0
+  /** Host to authority. A map: a `Host` of `constructor` names what a plain object answers. */
+  private readonly authorities: Map<string, string>
 
-  private constructor (properties: Properties) {
+  /** Tracked for the drain: `Http2Server` has no `closeIdleConnections`. */
+  private readonly sessions = new Set<http2.ServerHttp2Session>()
+
+  private readonly probe: Probe
+
+  private process?: Processor
+
+  private constructor(properties: Properties) {
     super()
 
     this.properties = properties
-    this.authorities = Object.fromEntries(Object.entries(properties.authorities).map(([key, value]) => [value, key]))
-
-    this.server.on('request', (req, res) => this.listener(req, res))
+    this.authorities = new Map(
+      Object.entries(properties.authorities).map(([key, value]) => [value, key])
+    )
+    this.server = instantiate(properties.protocol)
+    this.probe = new Probe(properties.probe)
+
+    this.server.on('request', (req, res) =>
+      this.listener(req as unknown as IncomingMessage, res as unknown as ServerResponse)
+    )
+
+    if (properties.protocol === 'h1') this.h1(this.server as http.Server)
+    else this.h2(this.server as http2.Http2Server)
   }
 
-  public static create (options: Options): Server {
-    const properties: Properties = Object.assign({}, DEFAULTS, options)
+  public static create(options: Options): Server {
+    const properties: Properties = { ...DEFAULTS, ...options }
 
     return new Server(properties)
   }
 
-  public attach (process: Processing): void {
+  public attach(process: Processor): void {
     this.process = process
   }
 
-  protected override async open (): Promise<void> {
-    this.startedAt = Date.now()
+  protected override async open(): Promise<void> {
+    // answers 503 from here; the gateway's dependencies have settled by the time `open` runs
+    await this.probe.listen()
+
     this.server.listen(this.properties.port)
 
     await once(this.server, 'listening')
 
     console.info('HTTP Server is listening')
 
-    await setTimeout(this.properties.delay)
-
-    this.ready = true
+    this.probe.complete()
 
     console.info('Ready')
   }
 
-  protected override async close (): Promise<void> {
+  protected override async close(): Promise<void> {
+    await this.probe.close()
+
     this.server.close()
-    this.ready = false
 
-    console.info('HTTP Server stopped accepting new connections')
+    // GOAWAY lets in-flight streams finish; `Http2Server` has no `closeIdleConnections`
+    if (this.properties.protocol === 'h1')
+      (this.server as http.Server).closeIdleConnections()
+    else for (const session of this.sessions) session.close()
+
+    console.info('Stopped accepting new connections')
+
+    // keep-alive clients hold connections open indefinitely, so the drain is bounded
+    await Promise.race([
+      once(this.server, 'close'),
+      setTimeout(this.properties.drain, undefined, { ref: false })
+    ])
 
-    await once(this.server, 'close')
+    if (this.properties.protocol === 'h1')
+      (this.server as http.Server).closeAllConnections()
+    else for (const session of this.sessions) session.destroy()
 
-    console.info('HTTP Server has been stopped')
+    console.info('Stopped')
   }
 
-  private listener (request: http.IncomingMessage, response: http.ServerResponse): void {
-    if (request.method === undefined || !this.properties.methods.has(request.method)) {
-      response.writeHead(501).end()
+  /** A malformed HTTP/1.1 request has no framing to answer in, so the status line is written by hand. */
+  private h1(server: http.Server): void {
+    server.on('clientError', (error, socket) => {
+      console.warn('Client connection error', error)
+
+      if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\n\r\n')
+      else socket.destroy()
+    })
+  }
+
+  /** HTTP/2 has no `clientError`: failures surface per session, per stream, or per frame. */
+  private h2(server: http2.Http2Server): void {
+    server.on('session', (session) => {
+      this.sessions.add(session)
+      session.on('close', () => this.sessions.delete(session))
+    })
+
+    server.on('sessionError', (error) => console.warn('Session error', error))
+    server.on('streamError', (error) => console.warn('Stream error', error))
+
+    server.on('unknownProtocol', (socket) => {
+      console.warn('Unknown protocol')
+      socket.destroy()
+    })
+  }
+
+  private listener(request: IncomingMessage, response: ServerResponse): void {
+    request.once('error', (error) => {
+      console.warn('Request error', errorAttributes(request, error))
+
+      if (!response.writableEnded) response.destroy()
+    })
+
+    // no listener on `request.socket`: under HTTP/2 it is the session's socket, shared by
+    // every concurrent stream, and removing listeners on it would strip the siblings'
+
+    const host = authorityOf(request)?.toLowerCase()
+
+    if (host === undefined || !HOST.test(host)) {
+      console.warn(
+        'Request without a valid authority',
+        errorAttributes(request, new Error('Invalid authority'))
+      )
+
+      response.writeHead(400).end()
 
       return
     }
 
-    if (request.url === '/.ready') {
-      if (this.ready)
-        response.writeHead(200, { 'cache-control': 'no-store' }).end()
-      else {
-        const remaining = (Math.ceil((Date.now() - this.startedAt) / 1000)).toString()
+    // directives and components read `host`; HTTP/2 sends `:authority` instead, and what a
+    // component sees must not depend on the protocol the request arrived over
+    request.headers.host ??= host
 
-        response.writeHead(503, { 'retry-after': remaining }).end()
-      }
+    const url = parse(request, host)
+
+    if (url instanceof Error) {
+      console.warn('Invalid request', errorAttributes(request, url))
+
+      response.writeHead(400).end()
 
       return
     }
 
-    if (request.headers.host === undefined || !(request.headers.host in this.authorities)) {
-      response.writeHead(404).end('Unknown authority')
+    if (request.method === undefined || !this.properties.methods.has(request.method)) {
+      response.writeHead(501).end()
 
       return
     }
 
-    const authority = this.authorities[request.headers.host]
-    const context = new Context(authority, request as IncomingMessage, this.properties)
+    assert(this.process !== undefined, 'Request processor is not attached')
+
+    const authority = this.authorities.get(host) ?? host
 
-    this.process!(context)
-      .then(this.success(context, response))
-      .catch(this.fail(context, response))
+    // if the request carries no trace context, the trace starts here
+    const remote = trace(request.headers)
+
+    const processing =
+      remote === null
+        ? this.serve(request, response, authority, url)
+        : run(remote, async () => await this.serve(request, response, authority, url))
+
+    processing.catch((error) => {
+      console.error('Request processing failed', error)
+
+      if (!response.writableEnded) response.writeHead(500).end()
+    })
   }
 
-  private success (context: Context, response: http.ServerResponse) {
-    return (message: OutgoingMessage) => {
+  // eslint-disable-next-line max-params
+  private async serve(
+    request: IncomingMessage,
+    response: ServerResponse,
+    authority: string,
+    url: URL
+  ): Promise<void> {
+    await console.span(
+      {
+        name: `${request.method} ${request.url}`,
+        kind: 'server',
+        service: 'exposition',
+        attributes: { method: request.method, url: request.url, authority }
+      },
+      async () => {
+        response.setHeader('ray', current()!.traceId)
+
+        const context = new Context(authority, request, this.properties, url)
+
+        await this.process!(context)
+          .then(this.success(context, response))
+          .catch(this.fail(context, response))
+          .finally(() => request.removeAllListeners('error'))
+      }
+    )
+  }
+
+  private success(context: Context, response: ServerResponse) {
+    return async (message: OutgoingMessage) => {
       let status = message.status
 
       if (status === undefined)
-        if (message.body === null)
-          status = 404
-        else if (context.request.method === 'POST')
-          status = 201
-        else if (message.body === undefined)
+        if (message.body === null) status = 404
+        else if (context.request.method === 'POST') status = 201
+        else if (message.body === undefined && context.request.method !== 'HEAD')
           status = 204
-        else
-          status = 200
+        else status = 200
 
-      response.statusCode = message.status = status
-      write(context, response, message)
+      message.status = status
+
+      await write(context, response, message)
     }
   }
 
-  private fail (context: Context, response: http.ServerResponse) {
+  private fail(context: Context, response: ServerResponse) {
     return async (exception: Error) => {
-      if (!context.request.complete)
-        await adam(context.request)
+      try {
+        // Over HTTP/2 the reply is followed by RST_STREAM(NO_ERROR), which tells the client
+        // to stop sending without discarding the response — so the body is never read.
+        if (!context.request.complete && this.properties.protocol === 'h1')
+          await adam(context.request)
+
+        const status = exception instanceof Exception ? exception.status : 500
+        const span = current()
+
+        // https://opentelemetry.io/docs/specs/semconv/http/http-spans/#status
+        if (status >= 500 && span !== undefined) span.status = 'error'
+
+        if (!response.writableEnded) {
+          response.statusCode = status
+
+          const message: OutgoingMessage = {
+            status: response.statusCode,
+            authentic: true
+          }
+
+          // eslint-disable-next-line max-depth
+          if (exception instanceof Exception && exception.headers !== undefined)
+            message.headers = exception.headers
+
+          // eslint-disable-next-line max-depth
+          if (context.encoder === null) message.body = undefined
+          else if (exception instanceof ClientError || this.properties.debug)
+            message.body =
+              exception instanceof Exception
+                ? exception.body
+                : ((this.properties.debug && exception.stack) ?? exception.message)
+
+          await write(context, response, message)
+        }
+      } catch (final) {
+        console.error('Error in error handler', final)
+
+        if (!response.writableEnded)
+          try {
+            response.writeHead(500).end()
+          } catch {
+            // Nothing more we can do
+          }
+      }
+    }
+  }
+}
 
-      response.statusCode = exception instanceof Exception ? exception.status : 500
+function instantiate(protocol: Protocol): http.Server | http2.Http2Server {
+  if (protocol === 'h1') return http.createServer()
 
-      const message: OutgoingMessage = { status: response.statusCode }
-      const verbose = exception instanceof ClientError || this.properties.debug
+  return http2.createServer({
+    // realtime pins one stream per subscription, and they all share a session
+    maxSessionMemory: SESSION_MEMORY,
+    settings: { initialWindowSize: WINDOW }
+  })
+}
 
-      if (verbose)
-        message.body =
-          exception instanceof Exception
-            ? exception.body
-            : exception.stack ?? exception.message
+/**
+ * The authority the request is addressed to. HTTP/2 carries it in `:authority` and omits
+ * `host` entirely, so reading `host` alone would leave every HTTP/2 request unattributed.
+ */
+function authorityOf(request: IncomingMessage): string | undefined {
+  return request.headers[':authority'] ?? request.headers.host
+}
 
-      write(context, response, message)
-    }
+/** Parsing the URL is how a request is validated, so the `Context` is handed the result. */
+function parse(request: IncomingMessage, authority: string): URL | Error {
+  try {
+    return new URL(request.url, `https://${authority}`)
+  } catch (error) {
+    return error as Error
   }
 }
 
 // https://github.com/whatwg/fetch/issues/1254
-async function adam (request: http.IncomingMessage): Promise<any> {
+async function adam(request: IncomingMessage): Promise<void> {
   const devnull = fs.createWriteStream(os.devNull)
 
-  request.pipe(devnull)
+  devnull.on('error', () => undefined)
+  request.pipe(devnull).on('error', () => undefined)
+
+  await once(request, 'end')
+}
+
+function errorAttributes(
+  request: IncomingMessage,
+  error: Error & any
+): RequestErrorAttributes {
+  const attributes: RequestErrorAttributes = {
+    path: request.url,
+    method: request.method,
+    name: error.name
+  }
+
+  if (typeof error.code === 'string') attributes.code = error.code
+
+  if (typeof error.stack === 'string') attributes.stack = error.stack
 
-  return once(request, 'end')
+  return attributes
 }
 
-export const PORT = 8000
-export const DELAY = 3 // seconds
+export const DRAIN = 10 // seconds
+
+/** Megabytes a single HTTP/2 session may hold, over Node's default of 10. */
+const SESSION_MEMORY = 128
+
+/** Per-stream flow control window. The default 64 KiB throttles in proportion to RTT. */
+const WINDOW = 1024 * 1024
+
+/**
+ * Extracts the remote trace context from the request headers.
+ *
+ * The `ray` header adopts the trace by ID only and does not bypass sampling:
+ * the sampling decision is made by the server.
+ */
+function trace(headers: IncomingMessage['headers']): SpanContext | null {
+  if (typeof headers.traceparent === 'string') return decode(headers.traceparent)
+
+  // adopting a trace by ID does not bypass sampling
+  if (
+    typeof headers.ray === 'string' &&
+    RAY.test(headers.ray) &&
+    headers.ray !== ZERO_RAY
+  )
+    return { traceId: headers.ray.toLowerCase(), sampled: decide() }
+
+  return null
+}
+
+/**
+ * A host name with an optional port, or a bracketed IPv6 literal. The URL parser admits
+ * `,` `;` `=` `(` `)` and quotes in a host, and the authority is written into criteria.
+ */
+const HOST = /^(?:[a-z0-9.-]{1,253}|\[[0-9a-f:.]{2,45}\])(?::\d{1,5})?$/
+
+const RAY = /^[\da-f]{32}$/i
+const ZERO_RAY = '0'.repeat(32)
 
 const DEFAULTS: Omit<Properties, 'authorities'> = {
-  methods: new Set<string>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']),
+  methods: new Set<string>([
+    'OPTIONS',
+    'GET',
+    'HEAD',
+    'POST',
+    'PUT',
+    'PATCH',
+    'DELETE',
+    'LOCK',
+    'UNLOCK'
+  ]),
   debug: false,
-  trace: false,
   port: PORT,
-  delay: DELAY * 1000
+  drain: DRAIN * 1000,
+  protocol: 'h1',
+  probe: PROBE
 }
 
 interface Properties {
   authorities: Record<string, string>
   methods: Set<string>
   debug: boolean
-  trace: boolean
   port: number
-  delay: number
+  drain: number
+  protocol: Protocol
+
+  /** Port of the readiness probe, which is HTTP/1.1 whatever the gateway serves. */
+  probe: number
+
+  /** The header the client address is read from; the connection's address without one. */
+  ip?: string
+
+  /** Failed authentications an address may make; none are metered unless set. */
+  bouncer?: Bouncer
+
+  /** The authorization server this context exposes; none is exposed without it. */
+  oauth?: OAuth
+
+  /** JSON-RPC at `/.rpc`; none is served without it. */
+  rpc?: RPC
+
+  /** MCP at `/.mcp`; none is served without it. */
+  mcp?: MCP
 }
 
 export type Options = { authorities: Properties['authorities'] } & {
   [K in Exclude<keyof Properties, 'authorities'>]?: Properties[K]
 }
 
-export type Processing = (input: Context) => Promise<OutgoingMessage>
+export type Processor = (input: Context) => Promise<OutgoingMessage>
+
+interface RequestErrorAttributes {
+  path: string
+  method: string
+  name: string
+  code?: string
+  stack?: string
+}

package/source/HTTP/Timing.ts

@@ -1,40 +1,14 @@
 import { performance } from 'node:perf_hooks'
-import type { ServerResponse } from 'node:http'
+import type { ServerResponse } from './types.js'
 
+/** How long the request took, from received to written: one number, the header's. */
 export class Timing {
-  private readonly skip: boolean
   private readonly start = performance.now()
-  private readonly breakpoints: Breakpoint[] = []
 
-  public constructor (enabled: boolean) {
-    this.skip = !enabled
+  public append(response: ServerResponse): void {
+    response.setHeader(
+      'server-timing',
+      `total;dur=${(performance.now() - this.start).toFixed(3)}`
+    )
   }
-
-  public async capture<T> (id: string, promise: Promise<T>): Promise<T> {
-    if (this.skip)
-      return promise
-
-    const start = performance.now()
-    const result = promise instanceof Promise ? await promise : promise
-
-    this.breakpoints.push({ id, duration: performance.now() - start })
-
-    return result
-  }
-
-  public append (response: ServerResponse): void {
-    if (this.skip)
-      return
-
-    this.breakpoints.push({ id: 'total', duration: performance.now() - this.start })
-
-    for (const breakpoint of this.breakpoints)
-      response.appendHeader('server-timing',
-        `${breakpoint.id};dur=${breakpoint.duration.toFixed(3)}`)
-  }
-}
-
-interface Breakpoint {
-  id: string
-  duration: number
 }

package/source/HTTP/address.test.ts

@@ -0,0 +1,28 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { address } from './address.js'
+import type { IncomingMessage } from './types.js'
+
+const request = (headers: Record<string, string>): IncomingMessage =>
+  ({ headers, socket: { remoteAddress: '9.9.9.9' } }) as unknown as IncomingMessage
+
+it('should be nothing without a header named', () => {
+  assert.equal(address(request({ 'x-forwarded-for': '1.1.1.1' })), undefined)
+})
+
+it('should be the last value of the named header', () => {
+  assert.equal(
+    address(request({ 'x-forwarded-for': '1.1.1.1, 2.2.2.2' }), 'x-forwarded-for'),
+    '2.2.2.2'
+  )
+  assert.equal(
+    address(request({ 'cf-connecting-ip': '3.3.3.3' }), 'cf-connecting-ip'),
+    '3.3.3.3'
+  )
+})
+
+it('should be nothing when the named header is absent or empty', () => {
+  assert.equal(address(request({}), 'x-real-ip'), undefined)
+  assert.equal(address(request({ 'x-real-ip': ' ' }), 'x-real-ip'), undefined)
+})