@toa.io/extensions.exposition 1.0.0-alpha.21 → 1.0.0-alpha.210

package/source/HTTP/messages.ts

@@ -1,18 +1,37 @@
 import { Readable } from 'node:stream'
 import { buffer } from 'node:stream/consumers'
+import { console } from 'openspan'
 import { formats } from './formats'
 import { BadRequest, NotAcceptable, UnsupportedMediaType } from './exceptions'
 import type { Context } from './Context'
 import type * as http from 'node:http'
 
-export function write
-(context: Context, response: http.ServerResponse, message: OutgoingMessage): void {
+const server = `Exposition/${require('../../package.json').version}` +
+  ((process.env.TOA_CONTEXT === undefined ? '' : ` ${process.env.TOA_CONTEXT}`) +
+    (process.env.TOA_ENV === undefined ? '' : `/${process.env.TOA_ENV}`))
+
+const pending = new Map<string, PendingStream>()
+
+export async function write (context: Context, response: http.ServerResponse, message: OutgoingMessage): Promise<void> {
   for (const transform of context.pipelines.response)
-    transform(message)
+    await transform(message)
 
+  if (message?.status !== undefined)
+    response.statusCode = message.status
+
+  response.setHeader('server', server)
   message.headers?.forEach((value, key) => response.setHeader(key, value))
   context.timing.append(response)
 
+  if (response.destroyed) {
+    console.warn('Request destroyed prematurely', { path: context.url.pathname })
+
+    return
+  }
+
+  response.on('error', (exception: Error) =>
+    console.warn('HTTP response error', { path: context.url.pathname, exception }))
+
   if (message.body instanceof Readable)
     stream(message, context, response)
   else
@@ -33,14 +52,21 @@ export async function read (context: Context): Promise<any> {
 
   try {
     return format.decode(buf)
-  } catch {
+  } catch (error: unknown) {
+    const entry: Record<string, unknown> = { path: context.url.pathname, error: error?.toString?.() }
+
+    if (context.debug)
+      entry.input = buf.toString('utf-8')
+
+    console.debug('Failed to decode message', entry)
+
     throw new BadRequest()
   }
 }
 
-function send
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+function send (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   if (message.body === undefined || message.body === null) {
+    response.setHeader('content-length', '0')
     response.end()
 
     return
@@ -53,12 +79,12 @@ function send
 
   response
     .setHeader('content-type', context.encoder.type)
+    .setHeader('content-length', buf.length.toString())
     .appendHeader('vary', 'accept')
     .end(buf)
 }
 
-function stream
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+function stream (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   const encoded = message.headers !== undefined && message.headers.has('content-type')
 
   if (encoded)
@@ -66,14 +92,16 @@ function stream
   else
     multipart(message, context, response)
 
-  message.body.on('error', (e: Error) => {
-    console.error(e)
+  message.body.on('error', (exception: Error) => {
+    console.warn('Message stream error', { path: context.url.pathname, exception })
     response.end()
   })
+
+  if (context.debug)
+    debugStream(context, response)
 }
 
-function multipart
-(message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
+export function multipart (message: OutgoingMessage, context: Context, response: http.ServerResponse): void {
   if (context.encoder === null)
     throw new NotAcceptable()
 
@@ -81,9 +109,26 @@ function multipart
 
   response.setHeader('content-type', `${encoder.multipart}; boundary=${BOUNDARY}`)
 
+  response.write(Buffer.concat([
+    CUT,
+    CRLF,
+    encoder.encode('ACK'),
+    CRLF,
+    CUT
+  ]))
+
   message.body
-    .map((part: unknown) => Buffer.concat([CUT, encoder.encode(part), CRLF]))
-    .on('end', () => response.end(FINALCUT))
+    .map((part: unknown) => Buffer.concat([
+      CRLF /* indicates no boundary headers */,
+      encoder.encode(part),
+      CRLF,
+      CUT]))
+    .on('end', () => response.end(Buffer.concat([
+      CRLF,
+      encoder.encode('FIN'),
+      CRLF,
+      FINALCUT
+    ])))
     .pipe(response)
 }
 
@@ -92,6 +137,34 @@ const CUT = Buffer.from(`--${BOUNDARY}\r\n`)
 const CRLF = Buffer.from('\r\n')
 const FINALCUT = Buffer.from(`--${BOUNDARY}--`)
 
+const PENDING_DEBUG_INTERVAL = 30000
+
+let pendingInterval: NodeJS.Timeout | null = null
+
+function debugStream (context: Context, response: http.ServerResponse): void {
+  const ctx = { method: context.request.method, path: context.url.pathname }
+
+  console.debug('Stream opened', ctx)
+  pending.set(context.id, ctx)
+
+  response.on('close', () => {
+    console.debug('Stream closed', ctx)
+    pending.delete(context.id)
+
+    if (pending.size === 0) {
+      if (pendingInterval !== null)
+        clearInterval(pendingInterval)
+
+      pendingInterval = null
+    }
+  })
+
+  if (pendingInterval === null)
+    pendingInterval = setInterval(() =>
+      console.debug('Pending streams', { size: pending.size }),
+    PENDING_DEBUG_INTERVAL)
+}
+
 export interface OutgoingMessage {
   status?: number
   headers?: Headers
@@ -103,8 +176,15 @@ export interface Query {
 
   id?: string
   criteria?: string
+  search?: string
+  sample?: number
   sort?: string
   omit?: string
   limit?: string
   version?: number
 }
+
+interface PendingStream {
+  method: string
+  path: string
+}

package/source/Introspection.ts

@@ -0,0 +1,11 @@
+import type { Remote } from '@toa.io/core'
+
+export interface Introspection {
+  route?: Record<string, Schema>
+  query?: Record<string, Schema>
+  input?: Schema
+  output?: Schema
+  errors?: string[]
+}
+
+export type Schema = Awaited<ReturnType<Remote['explain']>>['input']

package/source/Mapping.ts

@@ -1,50 +1,98 @@
 import { type Parameter } from './RTD'
 import { Query } from './Query'
+import type { Introspection, Schema } from './Introspection'
+import type { QueryString } from './Query'
 import type * as http from './HTTP'
 import type * as syntax from './RTD/syntax'
 import type * as core from '@toa.io/core'
 
 export abstract class Mapping {
-  public static create (query?: syntax.Query): Mapping {
-    if (query === undefined || query === null)
-      return new InputMapping()
+  protected readonly query: Query
+  public abstract readonly queryable: boolean
 
-    const q = new Query(query)
+  public constructor (query: Query) {
+    this.query = query
+  }
+
+  public static create (query?: syntax.Query | null): Mapping {
+    const q = new Query(query!)
+
+    return queryable(query)
+      ? new QueryableMapping(q)
+      : new InputMapping(q)
+  }
+
+  public explain (introspection: Introspection): Record<string, Schema> | null {
+    return this.query.explain(introspection)
+  }
+
+  protected assign (input: any, qs: QueryString): void {
+    if (qs.parameters !== null) {
+      if (typeof input !== 'object' || input === null)
+        throw new Error('Input must be an object to embed query parameters')
 
-    return new QueryableMapping(q)
+      Object.assign(input, qs.parameters)
+    }
   }
 
-  public abstract fit (input: any, qs: http.Query, parameters: Parameter[]): core.Request
+  public abstract fit (input: any, query: http.Query, parameters: Parameter[]): core.Request
 }
 
 class QueryableMapping extends Mapping {
-  private readonly query: Query
+  public override readonly queryable = true
 
-  public constructor (query: Query) {
-    super()
+  public fit (input: any, query: http.Query, parameters: Parameter[]): core.Request {
+    const request: core.Request = {}
+    const qs = this.query.fit(query, parameters)
 
-    this.query = query
-  }
+    if (input === undefined && qs.parameters !== null)
+      input = {}
 
-  public fit (input: any, qs: http.Query, parameters: Parameter[]): core.Request {
-    const query = this.query.fit(qs, parameters)
+    this.assign(input, qs)
 
-    return {
-      input,
-      query
-    }
+    if (input !== undefined)
+      request.input = input
+
+    if (qs.query !== null)
+      request.query = qs.query
+
+    return request
   }
 }
 
 class InputMapping extends Mapping {
-  public fit (input: any, _: unknown, parameters: Parameter[]): core.Request {
-    if (input === undefined && parameters.length > 0)
+  public override readonly queryable = false
+
+  public fit (input: any, query: http.Query, parameters: Parameter[]): core.Request {
+    const request: core.Request = {}
+    const qs = this.query.fit(query, parameters)
+
+    if (input === undefined &&
+      (this.query.parameterized || parameters.length > 0 || qs.parameters !== null))
       input = {}
 
-    if (typeof input === 'object' && input !== null)
+    if (parameters.length > 0) {
+      if (typeof input !== 'object' || input === null)
+        throw new Error('Input must be an object to embed parameters')
+
       for (const parameter of parameters)
         input[parameter.name] = parameter.value
+    }
 
-    return { input }
+    this.assign(input, qs)
+
+    if (input !== undefined)
+      request.input = input
+
+    return request
   }
 }
+
+export function queryable (query?: syntax.Query | null): boolean {
+  if (query === undefined || query === null)
+    return false
+
+  const keys = Object.keys(query)
+
+  return !(keys.length === 1 && keys[0] === 'parameters')
+}

package/source/Query.test.ts

@@ -16,7 +16,7 @@ it('should combine request criteria', async () => {
   const instance = new Query(query)
   const result = instance.fit({ criteria: 'qux==4' }, parameters)
 
-  expect(result.criteria).toStrictEqual('(foo==1);(bar==2;baz==3);(qux==4)')
+  expect(result.query!.criteria).toStrictEqual('(bar==2;baz==3);(foo==1);(qux==4)')
 })
 
 it('should set id parameter as query.id', async () => {
@@ -32,6 +32,6 @@ it('should set id parameter as query.id', async () => {
   const instance = new Query(query)
   const result = instance.fit({}, parameters)
 
-  expect(result.criteria).toBeUndefined()
-  expect(result.id).toStrictEqual(id)
+  expect(result.query!.criteria).toBeUndefined()
+  expect(result.query!.id).toStrictEqual(id)
 })

package/source/Query.ts

@@ -1,43 +1,128 @@
+import assert from 'node:assert'
 import * as http from './HTTP'
 import { type Parameter } from './RTD'
 import * as schemas from './schemas'
+import { queryable } from './Mapping'
+import type { Introspection, Schema } from './Introspection'
 import type * as syntax from './RTD/syntax'
 import type * as core from '@toa.io/core'
 
 export class Query {
+  public readonly parameterized: boolean
+
   private readonly query: syntax.Query
   private readonly closed: boolean = false
+  private readonly prepend: ',' | ';' = ';'
+  private readonly queryable: boolean
+  private readonly searchable: boolean
 
   public constructor (query: syntax.Query) {
-    if (query.criteria !== undefined) {
-      const open = query.criteria[query.criteria.length - 1] === ';'
-
-      if (open) query.criteria = query.criteria.slice(0, -1)
-      else this.closed = true
+    this.parameterized = query?.parameters !== undefined
+    this.queryable = queryable(query)
+    this.searchable = query?.search === true
+
+    if (this.queryable) {
+      query.omit ??= { value: 0, range: [0, 1000] }
+      query.limit ??= { value: 10, range: [1, 100] }
+
+      if (query.criteria !== undefined) {
+        // eslint-disable-next-line max-depth
+        if (query.criteria.endsWith(';'))
+          query.criteria = query.criteria.slice(0, -1)
+        else
+          this.closed = true
+
+        // eslint-disable-next-line max-depth
+        if (query.criteria.startsWith(',') || query.criteria.startsWith(';')) {
+          this.prepend = query.criteria[0] as ',' | ';'
+
+          query.criteria = query.criteria.slice(1)
+        }
+      }
     }
 
     this.query = query
   }
 
-  public fit (query: http.Query, parameters: Parameter[]): core.Query {
-    const error = schemas.querystring.fit(query)
+  public fit (query: http.Query, parameters: Parameter[]): QueryString {
+    const qs = this.split(query)
+
+    if (qs.query !== null) {
+      const error = schemas.querystring.fit(qs.query)
 
-    if (error !== null)
-      throw new http.BadRequest('Query ' + error.message)
+      if (error !== null)
+        throw new http.BadRequest('Query ' + error.message)
 
-    this.fitCriteria(query, parameters)
-    this.fitRanges(query)
-    this.fitSort(query)
+      this.fitCriteria(qs.query, parameters)
+      this.fitRanges(qs.query)
+      this.fitSort(qs.query)
+
+      if (this.query.deleted !== undefined)
+        (qs.query as core.Query).deleted = this.query.deleted
+    }
 
-    return query as core.Query
+    return {
+      query: qs.query as core.Query,
+      parameters: qs.parameters
+    }
   }
 
-  private fitCriteria (query: http.Query, parameters: Parameter[]): void {
-    const criteria: string[] = []
+  public explain (introspection: Introspection): Record<string, Schema> | null {
+    if (this.query?.parameters === undefined || introspection.input?.type !== 'object')
+      return null
 
-    if (this.query.criteria !== undefined)
-      criteria.push(this.query.criteria)
+    let query: Record<string, Schema> | null = null
+
+    for (const parameter of this.query.parameters) {
+      const schema = introspection.input.properties[parameter]
+
+      if (schema !== undefined) {
+        query ??= {}
+        query[parameter] = schema
+      }
+
+      delete introspection.input.properties[parameter]
+    }
+
+    return query
+  }
+
+  private split (query: http.Query): {
+    query: http.Query | null
+    parameters: Record<string, string> | null
+  } {
+    let parameters: Record<string, string> | null = null
+
+    if (this.query?.parameters !== undefined)
+      for (const key in query)
+        // eslint-disable-next-line max-depth
+        if (this.query.parameters.includes(key)) {
+          parameters ??= {}
+          parameters[key] = query[key] as string
+
+          delete query[key]
+        }
+
+    if (!this.queryable) {
+      const keys = Object.keys(query)
 
+      if (keys.length > 0)
+        throw new http.BadRequest(`Query parameter '${keys[0]}' is not allowed`)
+
+      query = null!
+    }
+
+    if (query?.search !== undefined && !this.searchable)
+      throw new http.BadRequest('Query search is not allowed')
+
+    return {
+      query,
+      parameters
+    }
+  }
+
+  private fitCriteria (query: http.Query, parameters: Parameter[]): void {
+    const groups: CriteriaGroup[] = []
     const idx = parameters.findIndex((parameter) => parameter.name === 'id')
 
     if (idx !== -1) {
@@ -47,32 +132,36 @@ export class Query {
     }
 
     if (parameters.length > 0) {
-      const chunks = parameters
+      const criteria = parameters
         .map(({ name, value }) => `${name}==${value}`)
         .join(';')
 
-      criteria.push(chunks)
+      groups.push({ criteria, operator: this.prepend })
     }
 
+    if (this.query.criteria !== undefined)
+      groups.push({ criteria: this.query.criteria, operator: ';' })
+
     if (query.criteria !== undefined)
-      if (this.closed) throw new http.BadRequest('Query criteria is closed.')
-      else criteria.push(query.criteria)
-
-    switch (criteria.length) {
-      case 0:
-        break
-      case 1:
-        query.criteria = criteria[0]
-        break
-      default:
-        query.criteria = '(' + criteria.join(');(') + ')'
-        break
-    }
+      if (this.closed)
+        throw new http.BadRequest('Query criteria is closed')
+      else
+        groups.push({ criteria: query.criteria, operator: WHATEVER })
+
+    if (groups.length > 0)
+      query.criteria = groups.reduce((acc, { criteria, operator }, i) => {
+        return i === groups.length - 1
+          ? `${acc}(${criteria})`
+          : `${acc}(${criteria})${operator}`
+      }, '')
   }
 
   private fitRanges (qs: http.Query): void {
     const query = qs as core.Query
 
+    assert.ok(this.query.limit !== undefined, 'Query limit must be defined')
+    assert.ok(this.query.omit !== undefined, 'Query limit range must be defined')
+
     if (qs.limit !== undefined)
       query.limit = fit(qs.limit, this.query.limit.range, 'limit')
     else
@@ -99,7 +188,19 @@ function fit (string: string, range: [number, number], name: string): number {
 
   if (number < range[0] || number > range[1])
     throw new http.BadRequest(`Query ${name} must be between ` +
-      `${range[0]} and ${range[1]} inclusive.`)
+      `${range[0]} and ${range[1]} inclusive`)
 
   return number
 }
+
+const WHATEVER = ';'
+
+interface CriteriaGroup {
+  criteria: string
+  operator: ',' | ';'
+}
+
+export interface QueryString {
+  query: core.Query | null
+  parameters: Record<string, string> | null
+}

package/source/RTD/Context.ts

@@ -9,5 +9,5 @@ export interface Context<TExtension = any> {
     readonly factory: DirectiveFactory
     stack: Directive[]
   }
-  readonly extension?: TExtension
+  readonly extension: TExtension
 }

package/source/RTD/Directives.ts

@@ -6,10 +6,12 @@ import type { Output } from '../io'
 export interface Directives {
   preflight: (context: Context, parameters: Parameter[]) => Promise<Output>
   settle: (context: Context, response: OutgoingMessage) => Promise<void>
+  dispose: () => void
 }
 
 export interface DirectiveFactory {
   create: (directives: syntax.Directive[]) => Directives
+  dispose: () => void
 }
 
 export interface DirectiveSet {
@@ -30,4 +32,6 @@ export interface DirectiveFamily<TDirective = any, TExtension = any> {
   settle?: (directives: TDirective[],
     request: Context & TExtension,
     response: OutgoingMessage) => void | Promise<void>
+
+  dispose?: (directives: TDirective[]) => void
 }

package/source/RTD/Endpoint.ts

@@ -5,6 +5,9 @@ import type * as RTD from './index'
 
 export interface Endpoint {
   call: (context: http.Context, parameters: RTD.Parameter[]) => Promise<http.OutgoingMessage>
+
+  explain: (parameters: RTD.Parameter[]) => Promise<unknown>
+
   close: () => Promise<void>
 }
 

package/source/RTD/Method.ts

@@ -1,15 +1,31 @@
+import type { Parameter } from './Match'
 import type { Endpoint } from './Endpoint'
 import type { Directives } from './Directives'
 
 export class Method {
   public readonly endpoint: Endpoint | null
   public readonly directives: Directives
+  private introspection: unknown | null = null
+  private introspecting: Promise<unknown> | null = null
 
   public constructor (endpoint: Endpoint | null, directives: Directives) {
     this.endpoint = endpoint
     this.directives = directives
   }
 
+  public async explain (parameters: Parameter[]): Promise<unknown> {
+    if (this.introspection !== null)
+      return this.introspection
+
+    if (this.introspecting === null)
+      // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
+      this.introspecting = this.endpoint?.explain(parameters)!
+
+    this.introspection = await this.introspecting
+
+    return this.introspection
+  }
+
   public async close (): Promise<void> {
     await this.endpoint?.close()
   }