@toa.io/extensions.exposition 1.0.0-alpha.27 → 1.0.0-alpha.270

package/source/HTTP/messages.ts

@@ -1,18 +1,38 @@
 import { Readable } from 'node:stream'
-import { buffer } from 'node:stream/consumers'
+import { createHash } from 'node:crypto'
+import * as contentType from 'content-type'
+import { console } from 'openspan'
 import { formats } from './formats'
 import { BadRequest, NotAcceptable, UnsupportedMediaType } from './exceptions'
 import type { Context } from './Context'
 import type * as http from 'node:http'
 
-export function write
-(context: Context, response: http.ServerResponse, message: OutgoingMessage): void {
+const server = `Exposition/${require('../../package.json').version}` +
+  ((process.env.TOA_CONTEXT === undefined ? '' : ` ${process.env.TOA_CONTEXT}`) +
+    (process.env.TOA_ENV === undefined ? '' : `/${process.env.TOA_ENV}`))
+
+const pending = new Map<string, PendingStream>()
+
+export async function write (context: Context, response: http.ServerResponse, message: OutgoingMessage): Promise<void> {
   for (const transform of context.pipelines.response)
-    transform(message)
+    await transform(message)
+
+  if (message?.status !== undefined)
+    response.statusCode = message.status
 
+  response.setHeader('server', server)
   message.headers?.forEach((value, key) => response.setHeader(key, value))
   context.timing.append(response)
 
+  if (response.destroyed) {
+    console.warn('Request destroyed prematurely', { path: context.url.pathname })
+
+    return
+  }
+
+  response.on('error', (exception: Error) =>
+    console.warn('HTTP response error', { path: context.url.pathname, exception }))
+
   if (message.body instanceof Readable)
     stream(message, context, response)
   else
@@ -20,27 +40,38 @@ export function write
 }
 
 export async function read (context: Context): Promise<any> {
-  const type = context.request.headers['content-type']
+  const header = context.request.headers['content-type']
 
-  if (type === undefined)
+  if (header === undefined)
     return undefined
 
+  const { type, parameters } = contentType.parse(header)
+
   if (!(type in formats))
     throw new UnsupportedMediaType()
 
   const format = formats[type]
-  const buf = await context.timing.capture('buffer', buffer(context.request))
+  const buf = await context.buffer()
 
   try {
-    return format.decode(buf)
-  } catch {
+    return format.decode(buf, parameters.charset)
+  } catch (error: unknown) {
+    console.debug('Failed to decode message', {
+      path: context.url.pathname,
+      error: error?.toString?.()
+    })
+
     throw new BadRequest()
   }
 }
 
-function send
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+function send (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   if (message.body === undefined || message.body === null) {
+    // a HEAD reply carries no body but must still report the length a GET would
+    // have returned, so a length already set by a directive is left alone
+    if (!response.hasHeader('content-length'))
+      response.setHeader('content-length', '0')
+
     response.end()
 
     return
@@ -51,14 +82,41 @@ function send
 
   const buf = context.encoder.encode(message.body)
 
+  if (message.etag === true && conditional(context, response, buf))
+    return
+
   response
     .setHeader('content-type', context.encoder.type)
+    .setHeader('content-length', buf.length.toString())
     .appendHeader('vary', 'accept')
     .end(buf)
 }
 
-function stream
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+/**
+ * Tags a reply that carries no version with a hash of the body being sent, and answers
+ * `304` when the client already has it. The tag is taken from the encoded body rather
+ * than from a serialization of its own, so it identifies the representation — which is
+ * what `vary` says.
+ */
+function conditional (context: Context, response: http.ServerResponse, buf: Buffer): boolean {
+  const etag = `"${createHash('sha256').update(buf).digest('hex')}"`
+
+  response.setHeader('etag', etag)
+
+  if (context.request.headers['if-none-match'] !== etag)
+    return false
+
+  response
+    .setHeader('content-length', '0')
+    .appendHeader('vary', 'accept')
+
+  response.statusCode = 304
+  response.end()
+
+  return true
+}
+
+function stream (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   const encoded = message.headers !== undefined && message.headers.has('content-type')
 
   if (encoded)
@@ -66,14 +124,16 @@ function stream
   else
     multipart(message, context, response)
 
-  message.body.on('error', (e: Error) => {
-    console.error(e)
+  message.body.on('error', (exception: Error) => {
+    console.warn('Message stream error', { path: context.url.pathname, exception })
     response.end()
   })
+
+  if (context.debug)
+    debugStream(context, response)
 }
 
-function multipart
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+export function multipart (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   if (context.encoder === null)
     throw new NotAcceptable()
 
@@ -81,9 +141,26 @@ function multipart
 
   response.setHeader('content-type', `${encoder.multipart}; boundary=${BOUNDARY}`)
 
+  response.write(Buffer.concat([
+    CUT,
+    CRLF,
+    encoder.encode('ACK'),
+    CRLF,
+    CUT
+  ]))
+
   message.body
-    .map((part: unknown) => Buffer.concat([CUT, encoder.encode(part), CRLF]))
-    .on('end', () => response.end(FINALCUT))
+    .map((part: unknown) => Buffer.concat([
+      CRLF /* indicates no boundary headers */,
+      encoder.encode(part),
+      CRLF,
+      CUT]))
+    .on('end', () => response.end(Buffer.concat([
+      CRLF,
+      encoder.encode('FIN'),
+      CRLF,
+      FINALCUT
+    ])))
     .pipe(response)
 }
 
@@ -92,10 +169,41 @@ const CUT = Buffer.from(`--${BOUNDARY}\r\n`)
 const CRLF = Buffer.from('\r\n')
 const FINALCUT = Buffer.from(`--${BOUNDARY}--`)
 
+const PENDING_DEBUG_INTERVAL = 30000
+
+let pendingInterval: NodeJS.Timeout | null = null
+
+function debugStream (context: Context, response: http.ServerResponse): void {
+  const ctx = { method: context.request.method, path: context.url.pathname }
+
+  console.debug('Stream opened', ctx)
+  pending.set(context.id, ctx)
+
+  response.on('close', () => {
+    console.debug('Stream closed', ctx)
+    pending.delete(context.id)
+
+    if (pending.size === 0) {
+      if (pendingInterval !== null)
+        clearInterval(pendingInterval)
+
+      pendingInterval = null
+    }
+  })
+
+  if (pendingInterval === null)
+    pendingInterval = setInterval(() =>
+      console.debug('Pending streams', { size: pending.size }),
+    PENDING_DEBUG_INTERVAL)
+}
+
 export interface OutgoingMessage {
   status?: number
   headers?: Headers
   body?: any
+
+  /** tag the response with a hash of the encoded body, see `conditional` */
+  etag?: boolean
 }
 
 export interface Query {
@@ -103,8 +211,15 @@ export interface Query {
 
   id?: string
   criteria?: string
+  search?: string
+  sample?: number
   sort?: string
   omit?: string
   limit?: string
   version?: number
 }
+
+interface PendingStream {
+  method: string
+  path: string
+}

package/source/Interception.ts

@@ -5,6 +5,11 @@ export class Interception implements Interceptor {
 
   public constructor (interceptors: Interceptor[]) {
     this.interceptors = interceptors
+
+    // interceptors are module singletons, so a second gateway in one process —
+    // which is how the features run — would otherwise inherit the first one's state
+    for (const interceptor of interceptors)
+      interceptor.reset?.()
   }
 
   public async intercept (input: Input): Promise<Output> {
@@ -21,4 +26,7 @@ export class Interception implements Interceptor {
 
 export interface Interceptor {
   intercept: (input: Input) => Output | Promise<Output>
+
+  /** Discards whatever the interceptor accumulated while serving. */
+  reset?: () => void
 }

package/source/Introspection.ts

@@ -0,0 +1,11 @@
+import type { Remote } from '@toa.io/core'
+
+export interface Introspection {
+  route?: Record<string, Schema>
+  query?: Record<string, Schema>
+  input?: Schema
+  output?: Schema
+  errors?: string[]
+}
+
+export type Schema = Awaited<ReturnType<Remote['explain']>>['input']

package/source/Mapping.ts

@@ -1,51 +1,85 @@
 import { type Parameter } from './RTD'
 import { Query } from './Query'
+import type { Introspection, Schema } from './Introspection'
+import type { QueryString } from './Query'
 import type * as http from './HTTP'
 import type * as syntax from './RTD/syntax'
 import type * as core from '@toa.io/core'
 
 export abstract class Mapping {
-  public static create (query?: syntax.Query): Mapping {
-    if (query === undefined || query === null)
-      return new InputMapping()
+  protected readonly query: Query
+  public abstract readonly queryable: boolean
 
-    const q = new Query(query)
+  public constructor (query: Query) {
+    this.query = query
+  }
+
+  public static create (query?: syntax.Query | null): Mapping {
+    const q = new Query(query!)
 
-    return new QueryableMapping(q)
+    return queryable(query)
+      ? new QueryableMapping(q)
+      : new InputMapping(q)
   }
 
-  public abstract fit (input: any, qs: http.Query, parameters: Parameter[]): core.Request
+  public explain (introspection: Introspection): Record<string, Schema> | null {
+    return this.query.explain(introspection)
+  }
+
+  protected assign (input: any, qs: QueryString): void {
+    if (qs.parameters !== null) {
+      if (typeof input !== 'object' || input === null)
+        throw new Error('Input must be an object to embed query parameters')
+
+      Object.assign(input, qs.parameters)
+    }
+  }
+
+  public abstract fit (input: any, query: http.Query, parameters: Parameter[]): core.Request
 }
 
 class QueryableMapping extends Mapping {
-  private readonly query: Query
+  public override readonly queryable = true
 
-  public constructor (query: Query) {
-    super()
+  public fit (input: any, query: http.Query, parameters: Parameter[]): core.Request {
+    const request: core.Request = {}
+    const qs = this.query.fit(query, parameters)
 
-    this.query = query
-  }
+    if (input === undefined && qs.parameters !== null)
+      input = {}
 
-  public fit (input: any, qs: http.Query, parameters: Parameter[]): core.Request {
-    const query = this.query.fit(qs, parameters)
+    this.assign(input, qs)
 
-    return {
-      input,
-      query
-    }
+    if (input !== undefined)
+      request.input = input
+
+    if (qs.query !== null)
+      request.query = qs.query
+
+    return request
   }
 }
 
 class InputMapping extends Mapping {
-  public fit (input: any, _: unknown, parameters: Parameter[]): core.Request {
+  public override readonly queryable = false
+
+  public fit (input: any, query: http.Query, parameters: Parameter[]): core.Request {
     const request: core.Request = {}
+    const qs = this.query.fit(query, parameters)
 
-    if (input === undefined && parameters.length > 0)
+    if (input === undefined &&
+      (this.query.parameterized || parameters.length > 0 || qs.parameters !== null))
       input = {}
 
-    if (typeof input === 'object' && input !== null)
+    if (parameters.length > 0) {
+      if (typeof input !== 'object' || input === null)
+        throw new Error('Input must be an object to embed parameters')
+
       for (const parameter of parameters)
         input[parameter.name] = parameter.value
+    }
+
+    this.assign(input, qs)
 
     if (input !== undefined)
       request.input = input
@@ -53,3 +87,12 @@ class InputMapping extends Mapping {
     return request
   }
 }
+
+export function queryable (query?: syntax.Query | null): boolean {
+  if (query === undefined || query === null)
+    return false
+
+  const keys = Object.keys(query)
+
+  return !(keys.length === 1 && keys[0] === 'parameters')
+}

package/source/Query.test.ts

@@ -16,7 +16,7 @@ it('should combine request criteria', async () => {
   const instance = new Query(query)
   const result = instance.fit({ criteria: 'qux==4' }, parameters)
 
-  expect(result.criteria).toStrictEqual('(bar==2;baz==3);(foo==1);(qux==4)')
+  expect(result.query!.criteria).toStrictEqual('(bar==2;baz==3);(foo==1);(qux==4)')
 })
 
 it('should set id parameter as query.id', async () => {
@@ -32,6 +32,6 @@ it('should set id parameter as query.id', async () => {
   const instance = new Query(query)
   const result = instance.fit({}, parameters)
 
-  expect(result.criteria).toBeUndefined()
-  expect(result.id).toStrictEqual(id)
+  expect(result.query!.criteria).toBeUndefined()
+  expect(result.query!.id).toStrictEqual(id)
 })

package/source/Query.ts

@@ -1,42 +1,124 @@
+import assert from 'node:assert'
 import * as http from './HTTP'
 import { type Parameter } from './RTD'
 import * as schemas from './schemas'
+import { queryable } from './Mapping'
+import type { Introspection, Schema } from './Introspection'
 import type * as syntax from './RTD/syntax'
 import type * as core from '@toa.io/core'
 
 export class Query {
+  public readonly parameterized: boolean
+
   private readonly query: syntax.Query
   private readonly closed: boolean = false
   private readonly prepend: ',' | ';' = ';'
+  private readonly queryable: boolean
+  private readonly searchable: boolean
 
   public constructor (query: syntax.Query) {
-    if (query.criteria !== undefined) {
-      if (query.criteria.endsWith(';'))
-        query.criteria = query.criteria.slice(0, -1)
-      else
-        this.closed = true
+    this.parameterized = query?.parameters !== undefined
+    this.queryable = queryable(query)
+    this.searchable = query?.search === true
+
+    if (this.queryable) {
+      query.omit ??= { value: 0, range: [0, 1000] }
+      query.limit ??= { value: 10, range: [1, 100] }
+
+      if (query.criteria !== undefined) {
+        // eslint-disable-next-line max-depth
+        if (query.criteria.endsWith(';'))
+          query.criteria = query.criteria.slice(0, -1)
+        else
+          this.closed = true
+
+        // eslint-disable-next-line max-depth
+        if (query.criteria.startsWith(',') || query.criteria.startsWith(';')) {
+          this.prepend = query.criteria[0] as ',' | ';'
+
+          query.criteria = query.criteria.slice(1)
+        }
+      }
+    }
+
+    this.query = query
+  }
+
+  public fit (query: http.Query, parameters: Parameter[]): QueryString {
+    const qs = this.split(query)
 
-      if (query.criteria.startsWith(',') || query.criteria.startsWith(';')) {
-        this.prepend = query.criteria[0] as ',' | ';'
+    if (qs.query !== null) {
+      const error = schemas.querystring.fit(qs.query)
+
+      if (error !== null)
+        throw new http.BadRequest('Query ' + error.message)
+
+      this.fitCriteria(qs.query, parameters)
+      this.fitRanges(qs.query)
+      this.fitSort(qs.query)
+
+      if (this.query.deleted !== undefined)
+        (qs.query as core.Query).deleted = this.query.deleted
+    }
+
+    return {
+      query: qs.query as core.Query,
+      parameters: qs.parameters
+    }
+  }
 
-        query.criteria = query.criteria.slice(1)
+  public explain (introspection: Introspection): Record<string, Schema> | null {
+    if (this.query?.parameters === undefined || introspection.input?.type !== 'object')
+      return null
+
+    let query: Record<string, Schema> | null = null
+
+    for (const parameter of this.query.parameters) {
+      const schema = introspection.input.properties[parameter]
+
+      if (schema !== undefined) {
+        query ??= {}
+        query[parameter] = schema
       }
+
+      delete introspection.input.properties[parameter]
     }
 
-    this.query = query
+    return query
   }
 
-  public fit (query: http.Query, parameters: Parameter[]): core.Query {
-    const error = schemas.querystring.fit(query)
+  private split (query: http.Query): {
+    query: http.Query | null
+    parameters: Record<string, string> | null
+  } {
+    let parameters: Record<string, string> | null = null
+
+    if (this.query?.parameters !== undefined)
+      for (const key in query)
+        // eslint-disable-next-line max-depth
+        if (this.query.parameters.includes(key)) {
+          parameters ??= {}
+          parameters[key] = query[key] as string
+
+          delete query[key]
+        }
 
-    if (error !== null)
-      throw new http.BadRequest('Query ' + error.message)
+    if (!this.queryable) {
+      const keys = Object.keys(query)
 
-    this.fitCriteria(query, parameters)
-    this.fitRanges(query)
-    this.fitSort(query)
+      if (keys.length > 0)
+        throw new http.BadRequest(`Query parameter '${keys[0]}' is not allowed`)
 
-    return query as core.Query
+      query = null!
+    }
+
+    if (query?.search !== undefined && !this.searchable)
+      throw new http.BadRequest('Query search is not allowed')
+
+    return {
+      query,
+      parameters
+    }
   }
 
   private fitCriteria (query: http.Query, parameters: Parameter[]): void {
@@ -77,6 +159,9 @@ export class Query {
   private fitRanges (qs: http.Query): void {
     const query = qs as core.Query
 
+    assert.ok(this.query.limit !== undefined, 'Query limit must be defined')
+    assert.ok(this.query.omit !== undefined, 'Query limit range must be defined')
+
     if (qs.limit !== undefined)
       query.limit = fit(qs.limit, this.query.limit.range, 'limit')
     else
@@ -114,3 +199,8 @@ interface CriteriaGroup {
   criteria: string
   operator: ',' | ';'
 }
+
+export interface QueryString {
+  query: core.Query | null
+  parameters: Record<string, string> | null
+}

package/source/RTD/Context.ts

@@ -9,5 +9,8 @@ export interface Context<TExtension = any> {
     readonly factory: DirectiveFactory
     stack: Directive[]
   }
-  readonly extension?: TExtension
+
+  /** The route being built, accumulated on the way down and restored on the way up. */
+  path: string
+  readonly extension: TExtension
 }

package/source/RTD/Directives.ts

@@ -6,15 +6,20 @@ import type { Output } from '../io'
 export interface Directives {
   preflight: (context: Context, parameters: Parameter[]) => Promise<Output>
   settle: (context: Context, response: OutgoingMessage) => Promise<void>
+  dispose: () => void
 }
 
 export interface DirectiveFactory {
-  create: (directives: syntax.Directive[]) => Directives
+  create: (directives: syntax.Directive[], route: string) => Directives
+  dispose: () => void
 }
 
 export interface DirectiveSet {
   family: DirectiveFamily
   directives: any[]
+
+  /** qualified directive names, e.g. `auth:id` */
+  names?: string[]
 }
 
 export interface DirectiveFamily<TDirective = any, TExtension = any> {
@@ -23,6 +28,12 @@ export interface DirectiveFamily<TDirective = any, TExtension = any> {
 
   create: (name: string, ...rest: any[]) => TDirective
 
+  /**
+   * Puts the directives of one route in the order they must run in. Called once, when
+   * the set is built — the order cannot depend on the request.
+   */
+  arrange?: (directives: TDirective[]) => void
+
   preflight?: (directives: TDirective[],
     request: Context & TExtension,
     parameters: Parameter[]) => Output | Promise<Output>
@@ -30,4 +41,6 @@ export interface DirectiveFamily<TDirective = any, TExtension = any> {
   settle?: (directives: TDirective[],
     request: Context & TExtension,
     response: OutgoingMessage) => void | Promise<void>
+
+  dispose?: (directives: TDirective[]) => void
 }

package/source/RTD/Endpoint.ts

@@ -5,6 +5,9 @@ import type * as RTD from './index'
 
 export interface Endpoint {
   call: (context: http.Context, parameters: RTD.Parameter[]) => Promise<http.OutgoingMessage>
+
+  explain: (parameters: RTD.Parameter[]) => Promise<unknown>
+
   close: () => Promise<void>
 }
 

package/source/RTD/Method.ts

@@ -1,15 +1,31 @@
+import type { Parameter } from './Match'
 import type { Endpoint } from './Endpoint'
 import type { Directives } from './Directives'
 
 export class Method {
   public readonly endpoint: Endpoint | null
   public readonly directives: Directives
+  private introspection: unknown | null = null
+  private introspecting: Promise<unknown> | null = null
 
   public constructor (endpoint: Endpoint | null, directives: Directives) {
     this.endpoint = endpoint
     this.directives = directives
   }
 
+  public async explain (parameters: Parameter[]): Promise<unknown> {
+    if (this.introspection !== null)
+      return this.introspection
+
+    if (this.introspecting === null)
+      // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
+      this.introspecting = this.endpoint?.explain(parameters)!
+
+    this.introspection = await this.introspecting
+
+    return this.introspection
+  }
+
   public async close (): Promise<void> {
     await this.endpoint?.close()
   }