@toa.io/extensions.exposition 1.0.0-alpha.26 → 1.0.0-alpha.262

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.exposition",
-  "version": "1.0.0-alpha.26",
+  "version": "1.0.0-alpha.262",
   "description": "Toa Exposition",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -17,39 +17,50 @@
     "access": "public"
   },
   "dependencies": {
-    "@toa.io/core": "1.0.0-alpha.26",
-    "@toa.io/generic": "1.0.0-alpha.26",
-    "@toa.io/schemas": "1.0.0-alpha.26",
-    "bcryptjs": "2.4.3",
-    "error-value": "0.3.0",
-    "js-yaml": "4.1.0",
-    "matchacho": "0.3.5",
-    "msgpackr": "1.10.1",
-    "negotiator": "0.6.3",
+    "@simplewebauthn/server": "13.3.2",
+    "@toa.io/core": "1.0.0-alpha.262",
+    "@toa.io/generic": "1.0.0-alpha.254",
+    "@toa.io/schemas": "1.0.0-alpha.254",
+    "bcryptjs": "3.0.3",
+    "content-type": "2.0.0",
+    "error-value": "0.4.4",
+    "jose": "6.2.8",
+    "js-yaml": "5.2.3",
+    "lru-cache": "11.5.2",
+    "matchacho": "0.6.0",
+    "minimatch": "10.2.6",
+    "msgpackr": "2.0.5",
+    "negotiator": "1.0.0",
+    "openspan": "1.0.0-alpha.254",
     "paseto": "3.1.4"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node"
-  },
   "scripts": {
-    "test": "jest",
-    "transpile": "tsc && npm run transpile:bans && npm run transpile:basic && npm run transpile:tokens && npm run transpile:roles && npm run transpile:federation",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest",
+    "transpile": "tsc && npm run transpile:bans && npm run transpile:basic && npm run transpile:tokens && npm run transpile:roles && npm run transpile:federation && npm run transpile:credentials && npm run transpile:keys && npm run transpile:passkeys && npm run transpile:otp",
     "transpile:bans": "tsc -p ./components/identity.bans",
     "transpile:basic": "tsc -p ./components/identity.basic",
     "transpile:tokens": "tsc -p ./components/identity.tokens",
     "transpile:roles": "tsc -p ./components/identity.roles",
-    "pretranspile:federation": "js-yaml components/identity.federation/manifest.toa.yaml | jq -M '{ type: \"object\", properties: {configuration: .configuration.schema, entity: .entity.schema }, additionalProperties: false}' > schemas.json && json2ts -i schemas.json -o components/identity.federation/source/schemas.ts && rm schemas.json",
     "transpile:federation": "tsc -p ./components/identity.federation",
+    "transpile:credentials": "tsc -p ./components/identity.credentials",
+    "transpile:keys": "tsc -p ./components/identity.keys",
+    "transpile:passkeys": "tsc -p ./components/identity.passkeys",
+    "transpile:otp": "tsc -p ./components/identity.otp",
     "features": "cucumber-js",
-    "features:security": "cucumber-js --tags @security"
+    "features:security": "cucumber-js --tags @security",
+    "features:octets": "cucumber-js features/octets.*"
   },
   "devDependencies": {
-    "@toa.io/agent": "1.0.0-alpha.26",
-    "@toa.io/extensions.storages": "1.0.0-alpha.26",
-    "@types/bcryptjs": "2.4.3",
-    "@types/cors": "2.8.13",
-    "@types/negotiator": "0.6.1"
+    "@toa.io/extensions.storages": "*",
+    "@types/cors": "2.8.19",
+    "@types/negotiator": "0.6.5",
+    "jest-esbuild": "0.4.0"
+  },
+  "jest": {
+    "transform": {
+      "^.+\\.ts$": "jest-esbuild"
+    },
+    "testEnvironment": "node"
   },
-  "gitHead": "0ddd9260762d1fe2b8be88eb7b9725e3f7838299"
+  "gitHead": "e14059cf5cc364d933f2a4b3d7114f3b5ec0764f"
 }

package/readme.md

@@ -136,7 +136,13 @@ exposition:
 | `class`       | Ingress class.                                                                                                    |
 | `annotations` | Ingress annotations.                                                                                              |
 | `debug`       | Output server errors. Default `false`.                                                                            |
-| `trace`       | Output [server timing](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing). Default `false`. |
+
+### Observability
+
+Each response contains a `ray` header with the trace ID of the request, and
+[server timing](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing) headers.
+
+Incoming trace context is continued, see [Tracing](documentation/tracing.md).
 
 ### Context resources
 
@@ -180,6 +186,25 @@ exposition:
       GET: foo.bar.observe
 ```
 
+## Ready probe
+
+The gateway serves `GET /.ready` on port `8000` for Kubernetes startup/readiness probes.
+
+It returns `200` only after:
+
+1. The in-process identity composition has connected
+2. Initial route discovery has settled
+3. The HTTP server is listening
+4. The startup delay has elapsed (default `3s`)
+
+Before listen, probes get connection refused; after listen but before ready, `503`.
+When ready, the process also sends `process.send('ready')` (for PM2 `wait_ready`).
+
+The nested identity composition shares this process. Deployment disables telemetry's port-`8001`
+probe (`TOA_TELEMETRY_READY=false`) so only the gateway `/.ready` is used.
+
+See also [telemetry ready probe](../telemetry/readme.md#ready-probe).
+
 ## See Also
 
 - [Protocol support](documentation/protocol.md)

package/schemas/annotation.cos.yaml

@@ -1,6 +1,12 @@
 authorities*: <string>
 class: string
+resources:
+  cpu: &constraint
+    type: array
+    items: { type: string }
+    minItems: 2
+    maxItems: 2
+  memory: *constraint
 annotations: <string>
 debug: boolean
-trace: boolean
 /: ~

package/schemas/io/throttle.cos.yaml

@@ -0,0 +1,36 @@
+definitions:
+  component: &component
+    type: string
+    enum: [ip, path]
+  condition: &condition
+    type: object
+    properties:
+      status:
+        type: number
+type: object
+properties:
+  key:
+    anyOf:
+      - *component
+      - type: array
+        items: *component
+        minItems: 1
+  condition:
+    anyOf:
+      - *condition
+      - type: array
+        items: *condition
+        minItems: 1
+  requests:
+    type: integer
+    minimum: 1
+  interval: &interval
+    description: Seconds
+    type: integer
+    minimum: 1
+  cooldown: *interval
+required:
+  - key
+  - requests
+  - interval
+  - cooldown

package/schemas/method.cos.yaml

@@ -1,8 +1,9 @@
-verb: /^(GET|POST|PUT|PATCH|DELETE)$/
+verb: /^(GET|HEAD|POST|PUT|PATCH|DELETE|LOCK|UNLOCK)$/
 mapping:
   namespace: string
   component: string
   endpoint: string
   query:
     $ref: query
+  ...: false
 directives: [ref:directive]

package/schemas/node.cos.yaml

@@ -1,5 +1,7 @@
 protected: boolean
 isolated: boolean
+forward: string
 routes: [ref:route]
 methods: [ref:method]
 directives: [ref:directive]
+version: string

package/schemas/octets/put.cos.yaml

@@ -0,0 +1,28 @@
+type: object
+nullable: true
+properties:
+  location:
+    type: string
+  accept:
+    anyOf:
+      - type: string
+      - type: array
+        items:
+          type: string
+  limit:
+    type: string
+    pattern: ^(\d+(\.\d+)?)([kmgtKMBGT]i?)?[bB]?$
+  trust:
+    type: array
+    items:
+      type: string
+  workflow:
+    anyOf:
+      - &unit
+        type: object
+        patternProperties:
+          ^.+$:
+            type: string
+      - type: array
+        items: *unit
+additionalProperties: false

package/schemas/query.cos.yaml

@@ -1,17 +1,14 @@
-_: true
 id?: string
 criteria?: string
 sort?: string
-omit:
+search?: boolean
+sample?: integer
+omit?:
   $ref: range
-  default:
-    value: 0
-    range: [0, 1000]
-limit:
+limit?:
   $ref: range
-  default:
-    value: 10
-    range: [1, 1000]
-  required: [value]
 selectors?: [string]
 projection?: [string]
+parameters?: [string]
+deleted?: boolean
+_: true

package/schemas/querystring.cos.yaml

@@ -1,5 +1,6 @@
 id: string
 criteria: string
+search: string
 sort: string
 omit: number
 limit: number

package/source/Annotation.ts

@@ -1,8 +1,10 @@
+import type { Resources } from '@toa.io/operations'
+
 export interface Annotation {
   authorities: Record<string, string>
   class?: string
+  resources?: Resources
   annotations?: Record<string, string>
   debug?: boolean
-  trace?: boolean
   '/'?: object // parsed and validated by RTD.syntax.parse
 }

package/source/Branch.ts

@@ -5,4 +5,5 @@ export interface Branch {
   component: string
   isolated: boolean
   node: RTD.Node
+  version: string
 }

package/source/Composition.ts

@@ -18,12 +18,6 @@ export class Composition extends Connector {
     await composition.connect()
 
     this.depends(composition)
-
-    console.info('Composition complete.')
-  }
-
-  protected override dispose (): void {
-    console.info('Composition shutdown complete.')
   }
 }
 

package/source/Context.ts

@@ -1,8 +1,11 @@
 import type * as RTD from './RTD'
 
-export type Context = RTD.Context<Extension>
+// the trunk is built from the context annotation and belongs to no branch,
+// thus its endpoints must be fully qualified and are not versioned
+export type Context = RTD.Context<Extension | undefined>
 
 interface Extension {
   namespace: string
   component: string
+  version: string
 }

package/source/Directive.test.ts

@@ -11,14 +11,16 @@ const families: Array<jest.MockedObjectDeep<DirectiveFamily>> = [
     mandatory: true,
     create: jest.fn((_0: any, _1: any, _2: any) => generate() as any),
     preflight: jest.fn(),
-    settle: jest.fn()
+    settle: jest.fn(),
+    dispose: jest.fn()
   },
   {
     name: 'bar',
     mandatory: false,
     create: jest.fn((_0: string, _1: any, _2: any) => generate() as any),
     preflight: jest.fn(),
-    settle: jest.fn()
+    settle: jest.fn(),
+    dispose: jest.fn()
   }
 ]
 
@@ -65,7 +67,7 @@ it('should throw error if directive family is not found', async () => {
   }
 
   expect(() => factory.create([declaration]))
-    .toThrowError(`Directive family '${declaration.family}' is not found.`)
+    .toThrow(`Directive family '${declaration.family}' is not found`)
 })
 
 it('should apply directive', async () => {

package/source/Directive.ts

@@ -1,3 +1,4 @@
+import { console, type SpanOptions } from 'openspan'
 import type { Context, OutgoingMessage } from './HTTP'
 import type { Remotes } from './Remotes'
 import type { Output } from './io'
@@ -11,23 +12,37 @@ export class Directives implements RTD.Directives {
   }
 
   public async preflight (context: Context, parameters: RTD.Parameter[]): Promise<Output> {
+    let output = null
+
     for (const set of this.sets) {
       if (set.family.preflight === undefined)
         continue
 
-      const output = await set.family.preflight(set.directives, context, parameters)
+      const out = await console.span(options(set, 'preflight'),
+        async () => await set.family.preflight!(set.directives, context, parameters))
+
+      if (out === null)
+        continue
 
       if (output !== null)
-        return output
+        throw new Error('Multiple preflight directives responded')
+      else
+        output = out
     }
 
-    return null
+    return output
   }
 
   public async settle (context: Context, response: OutgoingMessage): Promise<void> {
     for (const set of this.sets)
       if (set.family.settle !== undefined)
-        await set.family.settle(set.directives, context, response)
+        await console.span(options(set, 'settle'),
+          async () => await set.family.settle!(set.directives, context, response))
+  }
+
+  public dispose (): void {
+    for (const set of this.sets)
+      set.family.dispose?.(set.directives)
   }
 }
 
@@ -35,6 +50,7 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
   private readonly remotes: Remotes
   private readonly families: Record<string, RTD.DirectiveFamily> = {}
   private readonly mandatory: string[] = []
+  private readonly instances: Directives[] = []
 
   public constructor (families: RTD.DirectiveFamily[], remotes: Remotes) {
     for (const family of families) {
@@ -54,16 +70,20 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
     declarations.sort((a, b) =>
       (mandatory.has(b.family) ? 1 : 0) - (mandatory.has(a.family) ? 1 : 0))
 
+    const names: Record<string, string[]> = {}
+
     for (const declaration of declarations) {
       const family = this.families[declaration.family]
 
       if (family === undefined)
-        throw new Error(`Directive family '${declaration.family}' is not found.`)
+        throw new Error(`Directive family '${declaration.family}' is not found`)
 
       const directive = family.create(declaration.name, declaration.value, this.remotes)
 
       groups[family.name] ??= []
       groups[family.name].push(directive)
+      names[family.name] ??= []
+      names[family.name].push(`${declaration.family}:${declaration.name}`)
       mandatory.delete(family.name)
     }
 
@@ -72,25 +92,47 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
     for (const family of mandatory)
       sets.push({
         family: this.families[family],
-        directives: []
+        directives: [],
+        names: []
       })
 
     for (const [family, directives] of Object.entries(groups))
       sets.push({
         family: this.families[family],
-        directives
+        directives,
+        names: names[family]
       })
 
-    return new Directives(sets)
+    const directives = new Directives(sets)
+
+    this.instances.push(directives)
+
+    return directives
+  }
+
+  public dispose (): void {
+    for (const directives of this.instances)
+      directives.dispose()
   }
 }
 
+function options (set: RTD.DirectiveSet, stage: 'preflight' | 'settle'): SpanOptions {
+  const options: SpanOptions = { name: `${set.family.name} ${stage}` }
+
+  if (set.names !== undefined && set.names.length > 0)
+    options.attributes = { directives: Array.from(new Set(set.names)).join(' ') }
+
+  return options
+}
+
 export const shortcuts: RTD.syntax.Shortcuts = new Map([
   ['anonymous', 'auth:anonymous'],
+  ['anyone', 'auth:anyone'],
   ['id', 'auth:id'],
   ['role', 'auth:role'],
   ['rule', 'auth:rule'],
   ['incept', 'auth:incept'],
   ['input', 'io:input'],
-  ['output', 'io:output']
+  ['output', 'io:output'],
+  ['languages', 'map:languages']
 ])

package/source/Endpoint.ts

@@ -1,17 +1,21 @@
-import { type Component } from '@toa.io/core'
-import { type Remotes } from './Remotes'
+import { Readable } from 'node:stream'
+import { createHash } from 'node:crypto'
+import { console } from 'openspan'
 import { Mapping } from './Mapping'
-import { type Context } from './Context'
 import * as http from './HTTP'
+import type { Introspection, Schema } from './Introspection'
+import type { Remote } from '@toa.io/core'
+import type { Remotes } from './Remotes'
+import type { Context } from './Context'
 import type * as RTD from './RTD'
 
 export class Endpoint implements RTD.Endpoint {
   private readonly endpoint: string
   private readonly mapping: Mapping
-  private readonly discovery: Promise<Component>
-  private remote: Component | null = null
+  private readonly discovery: Promise<Remote>
+  private remote: Remote | null = null
 
-  public constructor (endpoint: string, mapping: Mapping, discovery: Promise<Component>) {
+  public constructor (endpoint: string, mapping: Mapping, discovery: Promise<Remote>) {
     this.endpoint = endpoint
     this.mapping = mapping
     this.discovery = discovery
@@ -24,28 +28,34 @@ export class Endpoint implements RTD.Endpoint {
 
     this.remote ??= await this.discovery
 
+    const endpoint = this.remote.locator.id + '.' + this.endpoint
+
+    console.debug('Calling operation', { endpoint, request })
+
     const reply = await this.remote.invoke(this.endpoint, request)
 
-    if (context.debug)
-      console.debug('Call', `${this.remote.locator.id}.${this.endpoint}`, request)
+    console.debug('Received reply', { endpoint, reply: reply instanceof Readable ? '[Readable stream]' : reply })
 
     if (reply instanceof Error)
       throw new http.UnprocessableEntity(reply)
 
     const message: http.OutgoingMessage = {}
 
-    if (typeof reply === 'object' && reply !== null && '_version' in reply) {
+    // etag
+    if (reply !== null && reply !== undefined) {
       const etag = context.request.headers['if-none-match']
 
-      message.headers ??= new Headers()
+      if (this.conditionalGet(reply, etag, message))
+        return message
+    }
 
-      if (etag !== undefined && reply._version === this.version(etag)) {
-        message.status = 304
-        message.headers.set('etag', etag)
+    // last-modified
+    if (typeof reply === 'object' && reply !== null && ('_updated' in reply || '_created' in reply)) {
+      const timestamp: string = reply._updated ?? reply._created
+      const date = new Date(timestamp)
 
-        return message
-      } else
-        message.headers.set('etag', `"${reply._version.toString()}"`)
+      message.headers ??= new Headers()
+      message.headers.set('last-modified', date.toUTCString())
     }
 
     message.body = reply
@@ -53,30 +63,109 @@ export class Endpoint implements RTD.Endpoint {
     return message
   }
 
+  public async explain (parameters: RTD.Parameter[]): Promise<Introspection> {
+    this.remote ??= await this.discovery
+
+    const operation = await this.remote.explain(this.endpoint)
+
+    let route: Record<string, Schema> | null = null
+
+    if (operation.input?.type === 'object')
+      for (const parameter of parameters) {
+        const schema = operation.input.properties[parameter.name]
+
+        // eslint-disable-next-line max-depth
+        if (schema !== undefined) {
+          route ??= {}
+          route[parameter.name] = schema
+
+          delete operation.input.properties[parameter.name]
+        }
+      }
+
+    const query = this.mapping.explain(operation)
+    const introspection: Introspection = {}
+
+    if (route !== null)
+      introspection.route = route
+
+    if (query !== null)
+      introspection.query = query
+
+    Object.assign(introspection, operation)
+
+    return introspection
+  }
+
   public async close (): Promise<void> {
     this.remote ??= await this.discovery
 
     await this.remote.disconnect(INTERRUPT)
   }
 
+  private conditionalGet (reply: unknown, etag: string | undefined, message: http.OutgoingMessage): boolean {
+    message.headers ??= new Headers()
+
+    if (typeof reply === 'object' && reply !== null && '_version' in reply) {
+      const version = reply._version as number
+      const matched = etag === undefined ? null : this.matchVersion(etag)
+
+      if (etag !== undefined && matched !== null && version === matched) {
+        message.status = 304
+        message.headers.set('etag', etag)
+
+        return true
+      }
+
+      message.headers.set('etag', `"${version.toString()}"`)
+
+      return false
+    }
+
+    if (reply instanceof Readable)
+      return false
+
+    const hash = `"${createHash('sha256').update(JSON.stringify(reply)).digest('hex')}"`
+
+    if (etag === hash) {
+      message.status = 304
+      message.headers.set('etag', etag)
+
+      return true
+    }
+
+    message.headers.set('etag', hash)
+
+    return false
+  }
+
   private query (context: http.Context): http.Query {
     const query: http.Query = Object.fromEntries(context.url.searchParams)
     const etag = context.request.headers['if-match']
 
-    if (etag !== undefined)
+    if (etag !== undefined && this.mapping.queryable)
       query.version = this.version(etag)
 
     return query
   }
 
-  private version (etag: string): number {
+  private matchVersion (etag: string): number | null {
     const match = etag.match(ETAG)
 
     if (match === null)
-      throw new http.BadRequest('Invalid ETag.')
+      return null
 
     return Number.parseInt(match.groups!.version)
   }
+
+  private version (etag: string): number {
+    const version = this.matchVersion(etag)
+
+    if (version === null)
+      throw new http.BadRequest('Invalid ETag')
+
+    return version
+  }
 }
 
 export class EndpointsFactory implements RTD.EndpointsFactory {
@@ -88,21 +177,24 @@ export class EndpointsFactory implements RTD.EndpointsFactory {
 
   public create (method: RTD.syntax.Method, context: Context): Endpoint {
     if (method.mapping === undefined)
-      throw new Error('Cannot create Endpoint without mapping.')
+      throw new Error('Cannot create Endpoint without mapping')
 
     const mapping = Mapping.create(method.mapping.query)
-    const namespace = method.mapping.namespace ?? context.extension?.namespace
-    const component = method.mapping.component ?? context.extension?.component
+
+    const branch = context.extension
+
+    const namespace = method.mapping.namespace ?? branch?.namespace
+    const component = method.mapping.component ?? branch?.component
 
     if (namespace === undefined || component === undefined)
-      throw new Error('Annotation endpoints must be fully qualified.')
+      throw new Error('Annotation endpoints must be fully qualified')
 
-    const discovery = this.remotes.discover(namespace, component)
+    const discovery = this.remotes.discover(namespace, component, branch?.version)
 
     return new Endpoint(method.mapping.endpoint, mapping, discovery)
   }
 }
 
-const ETAG = /^"(?<version>\d{1,32})"$/
+const ETAG = /^(W\/)?"(?<version>\d{1,32})"$/
 
 const INTERRUPT = true