@toa.io/extensions.exposition 1.0.0-alpha.26 → 1.0.0-alpha.262

package/source/Factory.ts

@@ -1,9 +1,11 @@
 import assert from 'node:assert'
+import { createHash } from 'node:crypto'
+import { console, traces, type LevelName, type TracesOptions } from 'openspan'
 import { decode } from '@toa.io/generic'
 import { Tenant } from './Tenant'
 import { Gateway } from './Gateway'
 import { Remotes } from './Remotes'
-import { Tree, syntax } from './RTD'
+import { Tree } from './RTD'
 import { EndpointsFactory } from './Endpoint'
 import { families, interceptors } from './directives'
 import { DirectivesFactory } from './Directive'
@@ -11,6 +13,8 @@ import { Composition } from './Composition'
 import * as root from './root'
 import { Interception } from './Interception'
 import * as http from './HTTP'
+import type { Branch } from './Branch'
+import type { syntax } from './RTD'
 import type { Broadcast } from './Gateway'
 import type { Connector, Locator, extensions } from '@toa.io/core'
 
@@ -23,17 +27,28 @@ export class Factory implements extensions.Factory {
 
   public tenant (locator: Locator, node: syntax.Node): Connector {
     const broadcast: Broadcast = this.boot.bindings.broadcast(CHANNEL, locator.id)
+    const hash = createHash('sha256').update(JSON.stringify(node)).digest('hex')
 
-    return new Tenant(broadcast, locator, node)
+    const branch: Branch = {
+      namespace: locator.namespace,
+      component: locator.name,
+      isolated: locator.namespace === 'identity',
+      node,
+      version: hash
+    }
+
+    return new Tenant(broadcast, branch)
   }
 
   public service (): Connector | null {
     assert.ok(process.env.TOA_EXPOSITION_PROPERTIES,
       'TOA_EXPOSITION_PROPERTIES is undefined')
 
+    configureLogs()
+
     const options = decode<http.Options>(process.env.TOA_EXPOSITION_PROPERTIES)
     const broadcast: Broadcast = this.boot.bindings.broadcast(CHANNEL)
-    const server = http.Server.create({ ...options, methods: syntax.verbs })
+    const server = http.Server.create({ ...options })
     const remotes = new Remotes(this.boot)
     const node = root.resolve()
     const methods = new EndpointsFactory(remotes)
@@ -55,6 +70,20 @@ export class Factory implements extensions.Factory {
 }
 
 const CHANNEL = 'exposition'
+const LOGS_PREFIX = 'TOA_TELEMETRY_LOGS'
+const TRACES_ENV = 'TOA_TELEMETRY_TRACES'
+
+function configureLogs (): void {
+  const globEnv = process.env[LOGS_PREFIX]
+  const level: LevelName = process.env.TOA_DEV === '1' ? 'trace' : 'info'
+  const options = globEnv === undefined ? { level } : decode<{ level?: LevelName }>(globEnv)
+
+  console.configure({ level: options.level ?? level })
+
+  const tracesEnv = process.env[TRACES_ENV]
+
+  traces(tracesEnv === undefined ? {} : decode<TracesOptions>(tracesEnv))
+}
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-imports
 export type Bootloader = typeof import('@toa.io/boot')

package/source/Gateway.ts

@@ -1,8 +1,11 @@
+import assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
+import { console } from 'openspan'
 import { type bindings, Connector } from '@toa.io/core'
 import * as http from './HTTP'
 import { rethrow } from './exceptions'
 import type { Interception } from './Interception'
-import type { Method, Parameter, Tree } from './RTD'
+import type { Method, Node, Parameter, Tree, Match } from './RTD'
 import type { Label } from './discovery'
 import type { Branch } from './Branch'
 
@@ -10,6 +13,11 @@ export class Gateway extends Connector {
   private readonly broadcast: Broadcast
   private readonly tree: Tree
   private readonly interceptor: Interception
+  private readonly branches = new Map<string, Exposed>()
+  private lastMerge = 0
+  private lastPing = 0
+  private stopped = false
+  private resolveFirstMerge: (() => void) | null = null
 
   public constructor (broadcast: Broadcast, tree: Tree, interception: Interception) {
     super()
@@ -28,17 +36,20 @@ export class Gateway extends Connector {
     if (interception !== null)
       return interception
 
-    const match = this.tree.match(context.url.pathname)
+    const { node, parameters } = this.match(context)
 
-    if (match === null)
-      throw new http.NotFound('Route not found')
+    if (context.request.method === 'OPTIONS')
+      return await this.explain(node, parameters)
 
-    const { node, parameters } = match
+    let verb = context.request.method
 
-    if (!(context.request.method in node.methods))
+    if (!(verb in node.methods) && verb === 'HEAD' && 'GET' in node.methods)
+      verb = 'GET'
+
+    if (!(verb in node.methods))
       throw new http.MethodNotAllowed()
 
-    const method = node.methods[context.request.method]
+    const method = node.methods[verb]
 
     const interruption = await context.timing.capture('preflight',
       method.directives.preflight(context, parameters)).catch(rethrow)
@@ -55,13 +66,46 @@ export class Gateway extends Connector {
   protected override async open (): Promise<void> {
     await this.discover()
 
-    console.info('Gateway has started and is awaiting resource branches')
+    console.info('Gateway started')
   }
 
   protected override dispose (): void {
+    this.stopped = true
+
+    this.tree.dispose()
+
     console.info('Gateway is closed')
   }
 
+  private match (context: http.Context): Match {
+    const match = this.tree.match(context.url.pathname)
+
+    if (match === null) {
+      // the route may be missing because an expose has been lost
+      this.reping()
+
+      throw new http.NotFound('Route not found')
+    }
+
+    if (match.node.forward === null)
+      return match
+
+    const destination = match.node.forward.replace(/\/:([^/]+)/g,
+      (_, name) => {
+        const value = match.parameters.find((parameter) => parameter.name === name)?.value
+
+        assert.ok(value !== undefined, `Forwarded parameter '${name}' not found`)
+
+        return `/${value}`
+      })
+
+    const forward = this.tree.match(destination)
+
+    assert.ok(forward !== null, 'Forwarded route not found')
+
+    return forward
+  }
+
   private async call (method: Method, context: http.Context, parameters: Parameter[]): Promise<http.OutgoingMessage> {
     if (context.url.pathname[context.url.pathname.length - 1] !== '/')
       throw new http.NotFound('Trailing slash is required')
@@ -77,21 +121,128 @@ export class Gateway extends Connector {
       .catch(rethrow) as http.OutgoingMessage
   }
 
+  private async explain (node: Node, parameters: Parameter[]): Promise<http.OutgoingMessage> {
+    const body = await node.explain(parameters)
+    const allow = [...Object.keys(node.methods)].join(', ')
+    const headers = new Headers({ allow })
+
+    return { body, headers }
+  }
+
   private async discover (): Promise<void> {
+    const first = new Promise<void>((resolve) => {
+      this.resolveFirstMerge = resolve
+    })
+
     await this.broadcast.receive<Branch>('expose', this.merge.bind(this))
+
+    void this.knock()
+
+    await this.settled(first)
+  }
+
+  /**
+   * A single ping is enough only if every tenant is listening by then, which is
+   * not the case while the deployment is still rolling out.
+   */
+  private async knock (): Promise<void> {
+    for (const delay of KNOCK_DELAYS) {
+      if (this.stopped)
+        return
+
+      if (delay > 0)
+        await setTimeout(delay, undefined, { ref: false })
+
+      await this.ping()
+    }
+  }
+
+  private async ping (): Promise<void> {
+    if (this.stopped)
+      return
+
+    this.lastPing = Date.now()
+
     await this.broadcast.transmit<null>('ping', null)
+      .catch((exception: Error) => console.error('Discovery ping failed',
+        { message: exception.message }))
+  }
+
+  private reping (): void {
+    if (Date.now() - this.lastPing < PING_COOLDOWN)
+      return
+
+    void this.ping()
+  }
+
+  private async settled (first: Promise<void>): Promise<void> {
+    const deadline = Date.now() + SETTLE_TIMEOUT
+
+    await Promise.race([first, setTimeout(SETTLE_TIMEOUT)])
+
+    if (this.lastMerge === 0) {
+      console.warn('Discovery timed out waiting for the first expose')
+
+      return
+    }
+
+    while (Date.now() - this.lastMerge < SETTLE_QUIET) {
+      if (Date.now() >= deadline)
+        break
+
+      await setTimeout(SETTLE_POLL)
+    }
   }
 
   private merge (branch: Branch): void {
+    const id = branch.namespace + '.' + branch.component
+
+    const attributes = {
+      namespace: branch.namespace,
+      component: branch.component,
+      version: branch.version
+    }
+
+    const exposed = this.branches.get(id)
+
+    // rebuilding an identical branch would only tear down its live endpoints
+    if (exposed?.version === branch.version) {
+      this.tree.refresh(exposed.nodes)
+      console.debug('Branch refreshed', attributes)
+
+      return
+    }
+
+    let nodes: Node[]
+
     try {
-      this.tree.merge(branch.node, branch)
+      nodes = this.tree.merge(branch.node, branch)
+    } catch (exception: unknown) {
+      const message = exception instanceof Error ? exception.message : 'Unknown error'
+
+      console.error('Branch merge exception', { message, ...attributes })
 
-      console.info('Resource branch of ' +
-        `'${branch.namespace}.${branch.component}' has been merged`)
-    } catch (exception) {
-      console.error(exception)
+      return
     }
+
+    this.branches.set(id, { version: branch.version, nodes })
+    this.lastMerge = Date.now()
+    this.resolveFirstMerge?.()
+    this.resolveFirstMerge = null
+
+    console.info('Branch merged', attributes)
   }
 }
 
 export type Broadcast = bindings.Broadcast<Label>
+
+interface Exposed {
+  version: string
+  nodes: Node[]
+}
+
+const SETTLE_QUIET = 10_000
+const SETTLE_TIMEOUT = 30_000
+const SETTLE_POLL = 50
+const KNOCK_DELAYS = [0, 500, 1000, 1500]
+const PING_COOLDOWN = 5_000

package/source/HTTP/Context.ts

@@ -1,4 +1,7 @@
+import crypto from 'node:crypto'
+import { buffer } from 'node:stream/consumers'
 import Negotiator from 'negotiator'
+import { console } from 'openspan'
 import { Timing } from './Timing'
 import { type Format, formats, types } from './formats'
 import { read } from './messages'
@@ -6,6 +9,7 @@ import type { OutgoingMessage } from './messages'
 import type * as http from 'node:http'
 
 export class Context {
+  public readonly id: string
   public readonly authority: string
   public readonly request: IncomingMessage
   public readonly url: URL
@@ -19,13 +23,17 @@ export class Context {
     response: []
   }
 
+  private consumed = false
+
   public constructor (authority: string, request: IncomingMessage, properties: Properties) {
     this.authority = authority
     this.request = request
 
+    this.id = crypto.randomUUID()
     this.url = new URL(request.url, `https://${request.headers.host}`)
-    this.timing = new Timing(properties.trace)
+    this.timing = new Timing()
     this.debug = properties.debug
+    this.log(request)
 
     if (this.request.headers.accept !== undefined) {
       const match = SUBTYPE.exec(this.request.headers.accept)
@@ -49,12 +57,29 @@ export class Context {
       this.encoder = formats[mediaType]
   }
 
-  public async body<T> (): Promise<T> {
-    const value = await read(this)
+  public async buffer (): Promise<Buffer> {
+    this.consumed = true
+
+    return await buffer(this.request)
+  }
+
+  public async body<T>(): Promise<T> {
+    let value = this.consumed ? null : await read(this)
+
+    for (const transform of this.pipelines.body)
+      value = await transform(value)
+
+    return value
+  }
+
+  private log (request: IncomingMessage): void {
+    const headers = { ...request.headers }
+
+    if (headers.authorization !== undefined)
+      // only scheme
+      headers.authorization = headers.authorization.slice(0, headers.authorization.indexOf(' '))
 
-    return this.pipelines.body.length === 0
-      ? value
-      : this.pipelines.body.reduce((value, transform) => transform(value), value)
+    console.debug('Received request', { method: request.method, url: request.url, headers })
   }
 }
 
@@ -65,12 +90,11 @@ export interface IncomingMessage extends http.IncomingMessage {
 
 interface Pipelines {
   body: Array<(input: unknown) => unknown>
-  response: Array<(output: OutgoingMessage) => void>
+  response: Array<(output: OutgoingMessage) => void | Promise<void>>
 }
 
 interface Properties {
   debug: boolean
-  trace: boolean
 }
 
 const SUBTYPE = /^(?<type>\w{1,32})\/(vnd\.toa\.(?<subtype>\S{1,32})\+)(?<suffix>\S{1,32})$/