@toa.io/extensions.exposition 1.0.0-alpha.273 → 1.0.0-alpha.276

package/features/steps/Gateway.ts

@@ -14,6 +14,7 @@ let instance: Connector | null = null
 @binding()
 export class Gateway {
   private default: boolean = true
+  private written: string[] = []
 
   @given('the annotation:')
   public async annotate (yaml: string): Promise<void> {
@@ -47,7 +48,15 @@ export class Gateway {
    */
   @given('the annotation of the introspection map')
   public async annotateMap (): Promise<void> {
-    const tree: syntax.Node = { routes: [], methods: [], directives: [] }
+    await this.annotateMapUnder()
+  }
+
+  /** The same routes under a root of the scenario's own: what an application declares around the map. */
+  @given('the annotation of the introspection map under:')
+  public async annotateMapUnder (yaml?: string): Promise<void> {
+    const tree: syntax.Node = yaml === undefined
+      ? { routes: [], methods: [], directives: [] }
+      : syntax.parse((parse(yaml) as { '/': object })['/'], shortcuts)
 
     for (const manifest of await manifests()) {
       const node = manifest.extensions?.[EXPOSITION] as syntax.Node | undefined
@@ -74,6 +83,9 @@ export class Gateway {
     const patch = parse(yaml) as object
     const configuration = Object.assign({}, def, patch)
 
+    // scenario-scoped, as the secrets it may refer to are: a configuration left behind
+    // outlives the secrets it points at, and the next scenario cannot resolve them
+    this.written.push(key)
     process.env[key] = JSON.stringify(configuration)
 
     await Gateway.stop()
@@ -81,6 +93,21 @@ export class Gateway {
     this.default = false
   }
 
+  /** The secrets a scenario's configuration refers to. */
+  @given('the configuration secrets:')
+  public async secrets (yaml: string): Promise<void> {
+    const secrets = parse(yaml) as Record<string, string>
+
+    for (const [name, value] of Object.entries(secrets)) {
+      this.written.push(SECRET + name)
+      process.env[SECRET + name] = value
+    }
+
+    await Gateway.stop()
+
+    this.default = false
+  }
+
   @given('the branch TTL is {float} second(s)')
   public async setBranchTTL (seconds: number): Promise<void> {
     process.env.__TESTING_EXPOSITION_BRANCH_TTL = String(seconds * 1000)
@@ -116,6 +143,11 @@ export class Gateway {
   public async cleanup (): Promise<void> {
     delete process.env.__TESTING_EXPOSITION_BRANCH_TTL
 
+    for (const key of this.written)
+      delete process.env[key]
+
+    this.written = []
+
     if (this.default)
       return
 
@@ -138,10 +170,14 @@ export class Gateway {
 
       process.env[key] ??= JSON.stringify(configuration)
     }
+
+    for (const [name, value] of Object.entries(DEFAULT_SECRETS))
+      process.env[SECRET + name] ??= value
   }
 }
 
 const EXPOSITION = '@toa.io/extensions.exposition'
+const SECRET = 'TOA_CONFIGURATION__'
 
 const DEFAULT_TREE = JSON.stringify({
   routes: [],
@@ -158,21 +194,31 @@ const DEFAULT_TREE = JSON.stringify({
 const DEFAULT_PROPERTIES: Partial<http.Options> = {
   authorities: {
     nex: 'nex.toa.io'
-  }
+  },
+  // `npm run features:h2c` runs the whole suite over cleartext HTTP/2
+  protocol: process.env.TOA_EXPOSITION_PROTOCOL === 'h2c' ? 'h2c' : 'h1'
 }
 
+// the identity components boot inside the gateway, and without a variable each would wait
+// for the values service, which these features do not run
 const DEFAULT_CONFIGURATION: Record<string, object> = {
+  // a component declaring configuration waits for `configuration.values`, which these
+  // features do not run; the variable is the local override that stands in for it
+  'realtime.streams': {},
+  'identity.basic': {},
+  'identity.federation': {},
+  'identity.otp': {},
+  'identity.passkeys': {},
   'identity.tokens': {
     keys: [
-      {
-        id: 'key0',
-        key: 'sTxL6qVOadKkUJwh3FveU53XgTEo3Sdfg7k2FfiIKfs'
-      },
-      {
-        id: 'legacy0',
-        key: 'k3.local.pIZT8-9Fa6U_QtfQHOSStfGtmyzPINyKQq2Xk-hd7vA',
-        format: 'paseto'
-      }
+      { id: 'key0', key: '$IDENTITY_TOKENS_KEY0' },
+      { id: 'legacy0', key: '$IDENTITY_TOKENS_LEGACY0', format: 'paseto' }
     ]
   }
 }
+
+// a secret is given as a reference, and comes to the component as an object
+const DEFAULT_SECRETS: Record<string, string> = {
+  IDENTITY_TOKENS_KEY0: 'sTxL6qVOadKkUJwh3FveU53XgTEo3Sdfg7k2FfiIKfs',
+  IDENTITY_TOKENS_LEGACY0: 'k3.local.pIZT8-9Fa6U_QtfQHOSStfGtmyzPINyKQq2Xk-hd7vA'
+}

package/features/steps/Parameters.ts

@@ -16,6 +16,10 @@ setDefaultTimeout(60 * 1000)
 
 process.env.TOA_DEV = '1'
 
+// the gateway answers for itself, as it does in a deployment: telemetry's probe tracks the
+// nested composition, which connects before route discovery settles
+process.env.TOA_TELEMETRY_READY ??= JSON.stringify(false)
+
 // export traces to the local Tempo (`docker compose up tempo grafana`),
 // unavailability of the endpoint is harmless
 process.env.TOA_TELEMETRY_TRACES ??= JSON.stringify({

package/features/steps/Probe.ts

@@ -0,0 +1,34 @@
+import assert from 'node:assert'
+import { Agent, request } from 'undici'
+import { after, binding, then, when } from 'cucumber-tsflow'
+import { PATH, PROBE } from '../../source/HTTP'
+
+@binding()
+export class Probe {
+  private readonly agent = new Agent()
+  private status: number | null = null
+  private headers: Record<string, string> = {}
+
+  @when('the ready probe is requested')
+  public async request (): Promise<void> {
+    const response = await request(`http://127.0.0.1:${PROBE}${PATH}`, { dispatcher: this.agent })
+
+    this.status = response.statusCode
+    this.headers = response.headers as Record<string, string>
+
+    await response.body.dump()
+  }
+
+  @then('the ready probe answers {int}')
+  public answers (status: number): void {
+    assert.equal(this.status, status)
+
+    if (status === 200)
+      assert.equal(this.headers['cache-control'], 'no-store')
+  }
+
+  @after()
+  public async close (): Promise<void> {
+    await this.agent.close()
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.exposition",
-  "version": "1.0.0-alpha.273",
+  "version": "1.0.0-alpha.276",
   "description": "Toa Exposition",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -20,7 +20,7 @@
     "@simplewebauthn/server": "13.3.3",
     "@toa.io/core": "1.0.0-alpha.273",
     "@toa.io/generic": "1.0.0-alpha.273",
-    "@toa.io/schemas": "1.0.0-alpha.273",
+    "@toa.io/schemas": "1.0.0-alpha.274",
     "bcryptjs": "3.0.3",
     "content-type": "2.1.0",
     "jose": "6.2.10",
@@ -46,6 +46,7 @@
     "transpile:passkeys": "tsc -p ./components/identity.passkeys",
     "transpile:otp": "tsc -p ./components/identity.otp",
     "features": "cucumber-js",
+    "features:h2c": "TOA_EXPOSITION_PROTOCOL=h2c TOA_AGENT_PROTOCOL=h2c cucumber-js",
     "features:security": "cucumber-js --tags @security",
     "features:octets": "cucumber-js features/octets.*"
   },
@@ -61,5 +62,5 @@
     },
     "testEnvironment": "node"
   },
-  "gitHead": "ad3284850ece95ffd5325fd0995707fc144c9c3d"
+  "gitHead": "6f81af829c7bf8a2db5a409f3d60ea40d15583ef"
 }

package/readme.md

@@ -188,21 +188,15 @@ exposition:
 
 ## Ready probe
 
-The gateway serves `GET /.ready` on port `8000` for Kubernetes startup/readiness probes.
+The gateway serves `GET /.ready` on port `8004`, not on the port it serves traffic on, and
+telemetry's probe is disabled in its process.
 
-It returns `200` only after:
+It answers `200` only after the in-process identity composition has connected, initial route
+discovery has settled, and the HTTP server is listening. Before that it answers `503` with a
+`retry-after`. When ready, the process also sends `process.send('ready')` for PM2 `wait_ready`.
 
-1. The in-process identity composition has connected
-2. Initial route discovery has settled
-3. The HTTP server is listening
-
-Before listen, probes get connection refused; after listen but before ready, `503`.
-When ready, the process also sends `process.send('ready')` (for PM2 `wait_ready`).
-
-The nested identity composition shares this process. Deployment disables telemetry's port-`8001`
-probe (`TOA_TELEMETRY_READY=false`) so only the gateway `/.ready` is used.
-
-See also [telemetry ready probe](../telemetry/readme.md#ready-probe).
+See also [telemetry ready probe](../telemetry/readme.md#ready-probe) and
+[reserved ports](../../documentation/ports.md).
 
 ## See Also
 

package/schemas/annotation.cos.yaml

@@ -24,8 +24,19 @@ properties:
     patternProperties:
       ^.+$:
         type: string
+  service:
+    type: object
+    properties:
+      annotations:
+        type: object
+        patternProperties:
+          ^.+$:
+            type: string
   debug:
     type: boolean
+  protocol:
+    type: string
+    enum: [h1, h2c]
   /: {}
   authorities:
     type: object

package/source/Annotation.ts

@@ -1,10 +1,21 @@
 import type { Resources } from '@toa.io/operations'
+import type { Protocol } from './HTTP'
 
 export interface Annotation {
   authorities: Record<string, string>
+
+  /**
+   * `h2c` requires an ingress controller that proxies cleartext HTTP/2 upstream.
+   * See `documentation/protocol.md`.
+   */
+  protocol?: Protocol
   class?: string
   resources?: Resources
   annotations?: Record<string, string>
+
+  /** The Service, as opposed to the Ingress that `annotations` above describes. */
+  service?: { annotations?: Record<string, string> }
+
   debug?: boolean
   '/'?: object // parsed and validated by RTD.syntax.parse
 }

package/source/HTTP/Context.ts

@@ -6,7 +6,7 @@ import { Timing } from './Timing'
 import { type Format, formats, types } from './formats'
 import { read } from './messages'
 import type { OutgoingMessage } from './messages'
-import type * as http from 'node:http'
+import type { IncomingMessage } from './types'
 
 export class Context {
   public readonly id: string
@@ -87,11 +87,6 @@ export class Context {
   }
 }
 
-export interface IncomingMessage extends http.IncomingMessage {
-  url: string
-  method: string
-}
-
 interface Pipelines {
   body: Array<(input: unknown) => unknown>
   response: Array<(output: OutgoingMessage) => void | Promise<void>>

package/source/HTTP/Probe.ts

@@ -0,0 +1,112 @@
+import * as http from 'node:http'
+import { console } from 'openspan'
+
+/**
+ * The readiness probe, on a port of its own.
+ *
+ * It cannot share the port the gateway serves traffic on: kubelet speaks HTTP/1.1 only, and a
+ * cleartext HTTP/2 server answers nothing else on its port — there is no ALPN to negotiate with.
+ *
+ * It is the gateway's own readiness, not the process's. The identity composition nested in this
+ * process connects before route discovery has settled, so a probe tied to that composition would
+ * report ready while the gateway still had no routes and no open port.
+ */
+export class Probe {
+  private readonly server = http.createServer()
+  private readonly port: number
+  private readonly path: string
+
+  private ready = false
+  private startedAt = 0
+  private listening = false
+  private skipped = false
+
+  public constructor (port: number, path: string = PATH) {
+    this.port = port
+    this.path = path
+
+    this.server.on('request', (request, response) => this.listener(request, response))
+  }
+
+  public async listen (): Promise<void> {
+    this.startedAt = Date.now()
+
+    try {
+      await new Promise<void>((resolve, reject) => {
+        this.server.once('error', reject)
+        this.server.listen(this.port, () => {
+          this.server.removeListener('error', reject)
+          resolve()
+        })
+      })
+    } catch (error: any) {
+      // processes sharing a host share the port; in a pod nothing else claims it
+      if (error?.code === 'EADDRINUSE') {
+        this.skipped = true
+
+        console.warn('Ready probe port already in use, skipping', { port: this.port })
+
+        return
+      }
+
+      throw error
+    }
+
+    this.listening = true
+
+    // a probe answers while the process runs; it must never be the reason it keeps running
+    this.server.unref()
+  }
+
+  /** The gateway is listening and has routes: answer `200`. */
+  public complete (): void {
+    this.ready = true
+
+    // the IPC signal is not tied to the probe: a process that gave up the shared port is
+    // still ready, and pm2 `wait_ready` would otherwise block until `listen_timeout`
+    process.send?.('ready')
+  }
+
+  public async close (): Promise<void> {
+    this.ready = false
+
+    if (!this.listening || this.skipped)
+      return
+
+    this.listening = false
+
+    // keep-alive connections would otherwise hold the server handle, delaying the exit
+    this.server.closeAllConnections()
+
+    await new Promise<void>((resolve) => this.server.close(() => resolve()))
+  }
+
+  private listener (request: http.IncomingMessage, response: http.ServerResponse): void {
+    if (request.url !== this.path) {
+      response.writeHead(404).end()
+
+      return
+    }
+
+    if (this.ready)
+      response.writeHead(200, { 'cache-control': 'no-store' }).end()
+    else {
+      const remaining = Math.ceil((Date.now() - this.startedAt) / 1000).toString()
+
+      response.writeHead(503, { 'retry-after': remaining }).end()
+    }
+  }
+}
+
+/**
+ * Reserved for this probe. `8001` is the Telemetry readiness probe's, and `toa export` refuses a
+ * port claimed twice — `toa mono` and a local run put every service in one process.
+ */
+export const PROBE = 8004
+export const PATH = '/.ready'
+
+/**
+ * The initial delay of the readiness probe. The server does not sleep for it: whoever
+ * probes is the one that waits, and doing it here as well only delayed the process twice.
+ */
+export const DELAY = 3 // seconds