@toa.io/extensions.exposition 1.0.0-alpha.28 → 1.0.0-alpha.283

package/source/Endpoint.ts

@@ -1,17 +1,21 @@
-import { type Component } from '@toa.io/core'
-import { type Remotes } from './Remotes'
-import { Mapping } from './Mapping'
-import { type Context } from './Context'
-import * as http from './HTTP'
-import type * as RTD from './RTD'
+import { Readable } from 'node:stream'
+import { console } from 'openspan'
+import { Mapping } from './Mapping.js'
+import { redact } from './redact.js'
+import * as http from './HTTP/index.js'
+import type { Introspection, Schema } from './Introspection.js'
+import type { Remote } from '@toa.io/core'
+import type { Remotes } from './Remotes.js'
+import type { Context } from './Context.js'
+import type * as RTD from './RTD/index.js'
 
 export class Endpoint implements RTD.Endpoint {
   private readonly endpoint: string
   private readonly mapping: Mapping
-  private readonly discovery: Promise<Component>
-  private remote: Component | null = null
+  private readonly discovery: Promise<Remote>
+  private remote: Remote | null = null
 
-  public constructor (endpoint: string, mapping: Mapping, discovery: Promise<Component>) {
+  public constructor (endpoint: string, mapping: Mapping, discovery: Promise<Remote>) {
     this.endpoint = endpoint
     this.mapping = mapping
     this.discovery = discovery
@@ -24,28 +28,35 @@ export class Endpoint implements RTD.Endpoint {
 
     this.remote ??= await this.discovery
 
+    const endpoint = this.remote.locator.id + '.' + this.endpoint
+
+    console.debug('Calling operation', { endpoint, request: redact(request) })
+
     const reply = await this.remote.invoke(this.endpoint, request)
 
-    if (context.debug)
-      console.debug('Call', `${this.remote.locator.id}.${this.endpoint}`, request)
+    console.debug('Received reply',
+      { endpoint, reply: reply instanceof Readable ? '[Readable stream]' : redact(reply) })
 
     if (reply instanceof Error)
       throw new http.UnprocessableEntity(reply)
 
     const message: http.OutgoingMessage = {}
 
-    if (typeof reply === 'object' && reply !== null && '_version' in reply) {
+    // etag
+    if (reply !== null && reply !== undefined) {
       const etag = context.request.headers['if-none-match']
 
-      message.headers ??= new Headers()
+      if (this.conditionalGet(reply, etag, message))
+        return message
+    }
 
-      if (etag !== undefined && reply._version === this.version(etag)) {
-        message.status = 304
-        message.headers.set('etag', etag)
+    // last-modified
+    if (typeof reply === 'object' && reply !== null && ('_updated' in reply || '_created' in reply)) {
+      const timestamp: string = reply._updated ?? reply._created
+      const date = new Date(timestamp)
 
-        return message
-      } else
-        message.headers.set('etag', `"${reply._version.toString()}"`)
+      message.headers ??= new Headers()
+      message.headers.set('last-modified', date.toUTCString())
     }
 
     message.body = reply
@@ -53,30 +64,106 @@ export class Endpoint implements RTD.Endpoint {
     return message
   }
 
+  public async explain (parameters: RTD.Parameter[]): Promise<Introspection> {
+    this.remote ??= await this.discovery
+
+    const operation = await this.remote.explain(this.endpoint)
+
+    let route: Record<string, Schema> | null = null
+
+    if (operation.input?.type === 'object')
+      for (const parameter of parameters) {
+        const schema = operation.input.properties[parameter.name]
+
+        // eslint-disable-next-line max-depth
+        if (schema !== undefined) {
+          route ??= {}
+          route[parameter.name] = schema
+
+          delete operation.input.properties[parameter.name]
+        }
+      }
+
+    const query = this.mapping.explain(operation)
+    const introspection: Introspection = {}
+
+    if (route !== null)
+      introspection.route = route
+
+    if (query !== null)
+      introspection.query = query
+
+    Object.assign(introspection, operation)
+
+    return introspection
+  }
+
   public async close (): Promise<void> {
     this.remote ??= await this.discovery
 
     await this.remote.disconnect(INTERRUPT)
   }
 
+  private conditionalGet (reply: unknown, etag: string | undefined, message: http.OutgoingMessage): boolean {
+    message.headers ??= new Headers()
+
+    if (typeof reply === 'object' && reply !== null && '_version' in reply) {
+      const version = reply._version as number
+      const matched = etag === undefined ? null : this.matchVersion(etag)
+
+      if (etag !== undefined && matched !== null && version === matched) {
+        message.status = 304
+        message.headers.set('etag', etag)
+
+        return true
+      }
+
+      message.headers.set('etag', `"${version.toString()}"`)
+
+      return false
+    }
+
+    if (reply instanceof Readable)
+      return false
+
+    /*
+     * A reply that carries no version is tagged with a hash of its body. The body is
+     * serialized anyway when the response is written, so the tag is computed from what
+     * is actually sent rather than from a second serialization of the reply — which
+     * makes it specific to the negotiated representation, hence `vary: accept`.
+     */
+    message.etag = true
+
+    return false
+  }
+
   private query (context: http.Context): http.Query {
     const query: http.Query = Object.fromEntries(context.url.searchParams)
     const etag = context.request.headers['if-match']
 
-    if (etag !== undefined)
+    if (etag !== undefined && this.mapping.queryable)
       query.version = this.version(etag)
 
     return query
   }
 
-  private version (etag: string): number {
+  private matchVersion (etag: string): number | null {
     const match = etag.match(ETAG)
 
     if (match === null)
-      throw new http.BadRequest('Invalid ETag.')
+      return null
 
     return Number.parseInt(match.groups!.version)
   }
+
+  private version (etag: string): number {
+    const version = this.matchVersion(etag)
+
+    if (version === null)
+      throw new http.BadRequest('Invalid ETag')
+
+    return version
+  }
 }
 
 export class EndpointsFactory implements RTD.EndpointsFactory {
@@ -88,21 +175,24 @@ export class EndpointsFactory implements RTD.EndpointsFactory {
 
   public create (method: RTD.syntax.Method, context: Context): Endpoint {
     if (method.mapping === undefined)
-      throw new Error('Cannot create Endpoint without mapping.')
+      throw new Error('Cannot create Endpoint without mapping')
 
     const mapping = Mapping.create(method.mapping.query)
-    const namespace = method.mapping.namespace ?? context.extension?.namespace
-    const component = method.mapping.component ?? context.extension?.component
+
+    const branch = context.extension
+
+    const namespace = method.mapping.namespace ?? branch?.namespace
+    const component = method.mapping.component ?? branch?.component
 
     if (namespace === undefined || component === undefined)
-      throw new Error('Annotation endpoints must be fully qualified.')
+      throw new Error('Annotation endpoints must be fully qualified')
 
-    const discovery = this.remotes.discover(namespace, component)
+    const discovery = this.remotes.discover(namespace, component, branch?.version)
 
     return new Endpoint(method.mapping.endpoint, mapping, discovery)
   }
 }
 
-const ETAG = /^"(?<version>\d{1,32})"$/
+const ETAG = /^(W\/)?"(?<version>\d{1,32})"$/
 
 const INTERRUPT = true

package/source/Factory.ts

@@ -1,51 +1,69 @@
 import assert from 'node:assert'
-import { decode } from '@toa.io/generic'
-import { Tenant } from './Tenant'
-import { Gateway } from './Gateway'
-import { Remotes } from './Remotes'
-import { Tree, syntax } from './RTD'
-import { EndpointsFactory } from './Endpoint'
-import { families, interceptors } from './directives'
-import { DirectivesFactory } from './Directive'
-import { Composition } from './Composition'
-import * as root from './root'
-import { Interception } from './Interception'
-import * as http from './HTTP'
-import type { Broadcast } from './Gateway'
+import { createHash } from 'node:crypto'
+import { console, traces, type LevelName, type TracesOptions } from 'openspan'
+import { Tenant } from './Tenant.js'
+import { Gateway } from './Gateway.js'
+import { Remotes } from './Remotes.js'
+import { Tree } from './RTD/index.js'
+import { EndpointsFactory } from './Endpoint.js'
+import { families, interceptors } from './directives/index.js'
+import { DirectivesFactory } from './Directive.js'
+import { Composition } from './Composition.js'
+import * as root from './root.js'
+import { ATOM_GROUP } from './const.js'
+import { Interception } from './Interception.js'
+import * as http from './HTTP/index.js'
+import type { Branch } from './Branch.js'
+import type { syntax } from './RTD/index.js'
+import type { Broadcast } from './Gateway.js'
 import type { Connector, Locator, extensions } from '@toa.io/core'
 
 export class Factory implements extensions.Factory {
-  private readonly boot: Bootloader
+  private readonly host: Host
 
-  public constructor (boot: Bootloader) {
-    this.boot = boot
+  public constructor (host: Host) {
+    this.host = host
   }
 
-  public tenant (locator: Locator, node: syntax.Node): Connector {
-    const broadcast: Broadcast = this.boot.bindings.broadcast(CHANNEL, locator.id)
+  public async tenant (locator: Locator, node: syntax.Node): Promise<Connector> {
+    const broadcast: Broadcast = await this.host.broadcast(CHANNEL, locator.id)
+    const hash = createHash('sha256').update(JSON.stringify(node)).digest('hex')
 
-    return new Tenant(broadcast, locator, node)
+    // no timestamp: the tenant stamps each announcement with its own start time
+    const branch: Omit<Branch, 'timestamp'> = {
+      namespace: locator.namespace,
+      component: locator.name,
+      isolated: locator.namespace === 'identity',
+      node,
+      version: hash
+    }
+
+    return new Tenant(broadcast, branch)
   }
 
-  public service (): Connector | null {
+  public async service (): Promise<Connector | null> {
     assert.ok(process.env.TOA_EXPOSITION_PROPERTIES,
       'TOA_EXPOSITION_PROPERTIES is undefined')
 
-    const options = decode<http.Options>(process.env.TOA_EXPOSITION_PROPERTIES)
-    const broadcast: Broadcast = this.boot.bindings.broadcast(CHANNEL)
-    const server = http.Server.create({ ...options, methods: syntax.verbs })
-    const remotes = new Remotes(this.boot)
+    configureLogs()
+
+    const options = JSON.parse(process.env.TOA_EXPOSITION_PROPERTIES) as http.Options
+    const broadcast: Broadcast = await this.host.broadcast(CHANNEL)
+    const server = http.Server.create({ ...options })
+    const remotes = new Remotes(this.host)
     const node = root.resolve()
     const methods = new EndpointsFactory(remotes)
-    const directives = new DirectivesFactory(families, remotes)
-    const interception = new Interception(interceptors)
+    const directives = new DirectivesFactory(families, remotes, this.host, options)
+    const interception = new Interception(interceptors, options)
     const tree = new Tree(node, methods, directives)
 
-    const composition = new Composition(this.boot)
+    const composition = new Composition(this.host)
     const gateway = new Gateway(broadcast, tree, interception)
 
     gateway.depends(remotes)
     gateway.depends(composition)
+    // what the directives meter through; one atom per process, connected once
+    gateway.depends(this.host.atom(ATOM_GROUP))
 
     server.attach(gateway.process.bind(gateway))
     server.depends(gateway)
@@ -55,6 +73,35 @@ export class Factory implements extensions.Factory {
 }
 
 const CHANNEL = 'exposition'
+const LOGS_PREFIX = 'TOA_TELEMETRY_LOGS'
+const TRACES_ENV = 'TOA_TELEMETRY_TRACES'
+
+function configureLogs (): void {
+  const globEnv = process.env[LOGS_PREFIX]
+  const level: LevelName = process.env.TOA_DEV === '1' ? 'trace' : 'info'
+  const options = globEnv === undefined ? { level } : JSON.parse(globEnv) as { level?: LevelName }
+
+  console.configure({ level: options.level ?? level })
+
+  const tracesEnv = process.env[TRACES_ENV]
+
+  traces(tracesEnv === undefined ? development() : JSON.parse(tracesEnv) as TracesOptions)
+}
+
+/**
+ * Tracing is off unless it is configured. The console exporter is a local development
+ * mechanism, so it is turned on for `toa dev` and for a boot trace the CLI has already
+ * asked for (`runtime/boot/src/span.js`), and nowhere else — a deployment that wants
+ * traces annotates `telemetry.traces.exporters`.
+ *
+ * The gateway boots without the telemetry extension, hence the copy of
+ * `extensions/telemetry/source/extension.ts`.
+ */
+function development (): TracesOptions {
+  const local = process.env.TOA_DEV === '1' || process.env.TOA_BOOT_TRACE === '1'
+
+  return local ? { exporters: { console: {} } } : {}
+}
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-imports
-export type Bootloader = typeof import('@toa.io/boot')
+export type Host = extensions.Host

package/source/Gateway.ts

@@ -1,15 +1,25 @@
+import assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
+import { console } from 'openspan'
 import { type bindings, Connector } from '@toa.io/core'
-import * as http from './HTTP'
-import { rethrow } from './exceptions'
-import type { Interception } from './Interception'
-import type { Method, Parameter, Tree } from './RTD'
-import type { Label } from './discovery'
-import type { Branch } from './Branch'
+import * as http from './HTTP/index.js'
+import { rethrow } from './exceptions.js'
+import { decide } from './Branch.js'
+import type { Interception } from './Interception.js'
+import type { Method, Node, Parameter, Tree, Match } from './RTD/index.js'
+import type { Label } from './discovery.js'
+import type { Branch, Exposed } from './Branch.js'
 
 export class Gateway extends Connector {
   private readonly broadcast: Broadcast
   private readonly tree: Tree
   private readonly interceptor: Interception
+  private readonly branches = new Map<string, Exposed>()
+  private lastMerge = 0
+  private widestGap = 0
+  private lastPing = 0
+  private stopped = false
+  private resolveFirstMerge: (() => void) | null = null
 
   public constructor (broadcast: Broadcast, tree: Tree, interception: Interception) {
     super()
@@ -28,17 +38,20 @@ export class Gateway extends Connector {
     if (interception !== null)
       return interception
 
-    const match = this.tree.match(context.url.pathname)
+    const { node, parameters } = this.match(context)
 
-    if (match === null)
-      throw new http.NotFound('Route not found')
+    if (context.request.method === 'OPTIONS')
+      return await this.explain(node, parameters)
+
+    let verb = context.request.method
 
-    const { node, parameters } = match
+    if (!(verb in node.methods) && verb === 'HEAD' && 'GET' in node.methods)
+      verb = 'GET'
 
-    if (!(context.request.method in node.methods))
+    if (!(verb in node.methods))
       throw new http.MethodNotAllowed()
 
-    const method = node.methods[context.request.method]
+    const method = node.methods[verb]
 
     const interruption = await context.timing.capture('preflight',
       method.directives.preflight(context, parameters)).catch(rethrow)
@@ -55,13 +68,54 @@ export class Gateway extends Connector {
   protected override async open (): Promise<void> {
     await this.discover()
 
-    console.info('Gateway has started and is awaiting resource branches')
+    console.info('Gateway started')
+  }
+
+  /**
+   * Both of these reach into components, and a dependency is torn down only once this has
+   * returned — where `dispose` runs after every one of them already has. The throttling
+   * ticker firing in between calls an endpoint that has just been unbound, and reports the
+   * refusal as a failure to reconcile.
+   */
+  protected override async close (): Promise<void> {
+    this.stopped = true
+
+    this.tree.dispose()
   }
 
   protected override dispose (): void {
     console.info('Gateway is closed')
   }
 
+  private match (context: http.Context): Match {
+    const match = this.tree.match(context.url.pathname)
+
+    if (match === null) {
+      // the route may be missing because an expose has been lost
+      this.reping()
+
+      throw new http.NotFound('Route not found')
+    }
+
+    if (match.node.forward === null)
+      return match
+
+    const destination = match.node.forward.replace(/\/:([^/]+)/g,
+      (_, name) => {
+        const value = match.parameters.find((parameter) => parameter.name === name)?.value
+
+        assert.ok(value !== undefined, `Forwarded parameter '${name}' not found`)
+
+        return `/${value}`
+      })
+
+    const forward = this.tree.match(destination)
+
+    assert.ok(forward !== null, 'Forwarded route not found')
+
+    return forward
+  }
+
   private async call (method: Method, context: http.Context, parameters: Parameter[]): Promise<http.OutgoingMessage> {
     if (context.url.pathname[context.url.pathname.length - 1] !== '/')
       throw new http.NotFound('Trailing slash is required')
@@ -77,21 +131,157 @@ export class Gateway extends Connector {
       .catch(rethrow) as http.OutgoingMessage
   }
 
+  private async explain (node: Node, parameters: Parameter[]): Promise<http.OutgoingMessage> {
+    const body = await node.explain(parameters)
+    const allow = [...Object.keys(node.methods)].join(', ')
+    const headers = new Headers({ allow })
+
+    return { body, headers }
+  }
+
   private async discover (): Promise<void> {
+    const first = new Promise<void>((resolve) => {
+      this.resolveFirstMerge = resolve
+    })
+
     await this.broadcast.receive<Branch>('expose', this.merge.bind(this))
+
+    void this.knock()
+
+    await this.settled(first)
+  }
+
+  /**
+   * A single ping is enough only if every tenant is listening by then, which is
+   * not the case while the deployment is still rolling out.
+   */
+  private async knock (): Promise<void> {
+    for (const delay of KNOCK_DELAYS) {
+      if (this.stopped)
+        return
+
+      if (delay > 0)
+        await setTimeout(delay, undefined, { ref: false })
+
+      await this.ping()
+    }
+  }
+
+  private async ping (): Promise<void> {
+    if (this.stopped)
+      return
+
+    this.lastPing = Date.now()
+
     await this.broadcast.transmit<null>('ping', null)
+      .catch((exception: Error) => console.error('Discovery ping failed',
+        { message: exception.message }))
+  }
+
+  private reping (): void {
+    if (Date.now() - this.lastPing < PING_COOLDOWN)
+      return
+
+    void this.ping()
+  }
+
+  private async settled (first: Promise<void>): Promise<void> {
+    const deadline = Date.now() + SETTLE_TIMEOUT
+    const abort = new AbortController()
+
+    // an uncancelled timer keeps the process alive long after the race is won
+    await Promise.race([first, setTimeout(SETTLE_TIMEOUT, undefined, { signal: abort.signal })])
+      .finally(() => { abort.abort() })
+      .catch(() => {})
+
+    if (this.lastMerge === 0) {
+      console.warn('Discovery timed out waiting for the first expose')
+
+      return
+    }
+
+    while (Date.now() - this.lastMerge < this.quiet()) {
+      if (Date.now() >= deadline)
+        break
+
+      await setTimeout(SETTLE_POLL)
+    }
+  }
+
+  /**
+   * How long the branches must stay quiet before discovery counts as settled.
+   *
+   * Every tenant of a local composition answers the first ping at once, so a short window
+   * is enough; a rolling deployment brings them up seconds apart, and the window grows with
+   * the widest gap seen so far to keep waiting for the ones still starting.
+   */
+  private quiet (): number {
+    const adaptive = this.widestGap * SETTLE_QUIET_FACTOR
+
+    return Math.min(Math.max(adaptive, SETTLE_QUIET_MIN), SETTLE_QUIET_MAX)
   }
 
   private merge (branch: Branch): void {
+    const id = branch.namespace + '.' + branch.component
+
+    const attributes = {
+      namespace: branch.namespace,
+      component: branch.component,
+      version: branch.version
+    }
+
+    const exposed = this.branches.get(id)
+
+    if (exposed !== undefined)
+      switch (decide(exposed, branch)) {
+        // rebuilding an identical branch would only tear down its live endpoints
+        case 'refresh':
+          this.tree.refresh(exposed.nodes)
+          console.trace('Branch refreshed', attributes)
+
+          return
+
+        case 'superseded':
+          console.debug('Branch superseded', {
+            ...attributes,
+            timestamp: branch.timestamp,
+            exposed: exposed.timestamp
+          })
+
+          return
+      }
+
+    let nodes: Node[]
+
     try {
-      this.tree.merge(branch.node, branch)
+      nodes = this.tree.merge(branch.node, branch)
+    } catch (exception: unknown) {
+      const message = exception instanceof Error ? exception.message : 'Unknown error'
 
-      console.info('Resource branch of ' +
-        `'${branch.namespace}.${branch.component}' has been merged`)
-    } catch (exception) {
-      console.error(exception)
+      console.error('Branch merge exception', { message, ...attributes })
+
+      return
     }
+
+    this.branches.set(id, { version: branch.version, timestamp: branch.timestamp, nodes })
+
+    if (this.lastMerge !== 0)
+      this.widestGap = Math.max(this.widestGap, Date.now() - this.lastMerge)
+
+    this.lastMerge = Date.now()
+    this.resolveFirstMerge?.()
+    this.resolveFirstMerge = null
+
+    console.info('Branch merged', attributes)
   }
 }
 
 export type Broadcast = bindings.Broadcast<Label>
+
+const SETTLE_QUIET_MIN = 500
+const SETTLE_QUIET_MAX = 10_000
+const SETTLE_QUIET_FACTOR = 2
+const SETTLE_TIMEOUT = 30_000
+const SETTLE_POLL = 50
+const KNOCK_DELAYS = [0, 500, 1000, 1500]
+const PING_COOLDOWN = 5_000