@toa.io/extensions.exposition 1.0.0-alpha.274 → 1.0.0-alpha.276

package/documentation/protocol.md

@@ -69,6 +69,30 @@ See also:
 - [Content-Type: multipart](https://learn.microsoft.com/en-us/previous-versions/office/developer/exchange-server-2010/aa493937(v=exchg.140))
   at Microsoft
 
+## HTTP version
+
+The gateway serves HTTP/1.1 by default. `protocol: h2c` serves cleartext HTTP/2 instead.
+
+```yaml
+exposition:
+  protocol: h2c
+  service:
+    annotations:
+      projectcontour.io/upstream-protocol.h2c: "8000"
+```
+
+`h2c` requires an ingress controller that proxies cleartext HTTP/2 upstream, and the
+annotation that controller reads to do so. Contour, Traefik and Envoy-based controllers
+proxy it; ingress-nginx does not. A cleartext HTTP/2 server answers nothing else on the
+same port: there is no ALPN to negotiate with, so a controller that speaks HTTP/1.1 upstream
+reaches nothing.
+
+Under `h2c` a request refused before its body is read is answered and then cancelled with
+`RST_STREAM(NO_ERROR)`. The reply arrives; the upload stops where it is. A client streaming
+a request body sees that body cancelled — in Node, `ERR_STREAM_PREMATURE_CLOSE` on the stream
+it passed, which throws if nothing handles it. A real failure shows in the reply, or in the
+request rejecting because no reply came. A buffered body is unaffected.
+
 ## CORS
 
 [CORS](https://www.w3.org/TR/2020/SPSD-cors-20200602/) is supported,

package/features/octets.cloudinary.feature

@@ -144,7 +144,10 @@ Feature: Octets with Cloudinary storage
       """
       200 OK
       content-type: video/mp4
-      transfer-encoding: chunked
+      """
+    And the reply does not contain:
+      """
+      content-length
       """
 
     # after a while, Cloudinary returns a content-length response
@@ -169,7 +172,10 @@ Feature: Octets with Cloudinary storage
       """
       200 OK
       content-type: video/mp4
-      transfer-encoding: chunked
+      """
+    And the reply does not contain:
+      """
+      content-length
       """
 
     When the following request is received:

package/features/octets.download.feature

@@ -18,6 +18,8 @@ Feature: Download external resources
               meta: true
       """
 
+    Given the length of `https://avatars.githubusercontent.com/u/92763022?s=48&v=4` is captured as `length`
+
     When the following request is received:
       """
       POST / HTTP/1.1
@@ -43,7 +45,7 @@ Feature: Download external resources
     Then the following reply is sent:
       """
       200 OK
-      content-length: 1288
+      content-length: ${{ length }}
       content-type: image/png
       etag: "${{ checksum }}"
       """

package/features/probes.feature

@@ -1,14 +1,18 @@
 Feature: Probes
 
-  Scenario: Startup probe
+  # The probe is on a port of its own: kubelet speaks HTTP/1.1, and the gateway may serve h2c.
+  Scenario: The gateway answers its readiness probe
+    Given the Gateway is running
+    When the ready probe is requested
+    Then the ready probe answers 200
+
+  Scenario: The traffic port does not serve the probe
     Given the Gateway is running
-    And after 15 seconds
     When the following request is received:
       """
       GET /.ready HTTP/1.1
       """
     Then the following reply is sent:
       """
-      200 OK
-      cache-control: no-store
+      404 Not Found
       """

package/features/steps/Common.ts

@@ -1,8 +1,16 @@
+import * as assert from 'node:assert'
 import { binding, given } from 'cucumber-tsflow'
 import { timeout } from '@toa.io/generic'
+import { Captures } from './Captures'
 
-@binding()
+@binding([Captures])
 export class Common {
+  private readonly captures: Captures
+
+  public constructor (captures: Captures) {
+    this.captures = captures
+  }
+
   @given('after {float} second(s)')
   public async timeout (interval: number): Promise<void> {
     await timeout(interval * 1000)
@@ -11,4 +19,19 @@ export class Common {
   @given('the process is running')
   public async noop (): Promise<void> {
   }
+
+  /**
+   * A remote resource is whatever it is today. A scenario downloading one compares the reply
+   * against what the origin actually serves, not against a literal that goes stale when it changes.
+   */
+  @given('the length of `{}` is captured as `{word}`')
+  public async length (url: string, name: string): Promise<void> {
+    const response = await fetch(url)
+
+    assert.equal(response.status, 200, `${url} answered ${response.status}`)
+
+    const body = await response.arrayBuffer()
+
+    this.captures.set(name, String(body.byteLength))
+  }
 }

package/features/steps/Gateway.ts

@@ -83,6 +83,9 @@ export class Gateway {
     const patch = parse(yaml) as object
     const configuration = Object.assign({}, def, patch)
 
+    // scenario-scoped, as the secrets it may refer to are: a configuration left behind
+    // outlives the secrets it points at, and the next scenario cannot resolve them
+    this.written.push(key)
     process.env[key] = JSON.stringify(configuration)
 
     await Gateway.stop()
@@ -191,12 +194,17 @@ const DEFAULT_TREE = JSON.stringify({
 const DEFAULT_PROPERTIES: Partial<http.Options> = {
   authorities: {
     nex: 'nex.toa.io'
-  }
+  },
+  // `npm run features:h2c` runs the whole suite over cleartext HTTP/2
+  protocol: process.env.TOA_EXPOSITION_PROTOCOL === 'h2c' ? 'h2c' : 'h1'
 }
 
 // the identity components boot inside the gateway, and without a variable each would wait
 // for the values service, which these features do not run
 const DEFAULT_CONFIGURATION: Record<string, object> = {
+  // a component declaring configuration waits for `configuration.values`, which these
+  // features do not run; the variable is the local override that stands in for it
+  'realtime.streams': {},
   'identity.basic': {},
   'identity.federation': {},
   'identity.otp': {},

package/features/steps/Parameters.ts

@@ -16,6 +16,10 @@ setDefaultTimeout(60 * 1000)
 
 process.env.TOA_DEV = '1'
 
+// the gateway answers for itself, as it does in a deployment: telemetry's probe tracks the
+// nested composition, which connects before route discovery settles
+process.env.TOA_TELEMETRY_READY ??= JSON.stringify(false)
+
 // export traces to the local Tempo (`docker compose up tempo grafana`),
 // unavailability of the endpoint is harmless
 process.env.TOA_TELEMETRY_TRACES ??= JSON.stringify({

package/features/steps/Probe.ts

@@ -0,0 +1,34 @@
+import assert from 'node:assert'
+import { Agent, request } from 'undici'
+import { after, binding, then, when } from 'cucumber-tsflow'
+import { PATH, PROBE } from '../../source/HTTP'
+
+@binding()
+export class Probe {
+  private readonly agent = new Agent()
+  private status: number | null = null
+  private headers: Record<string, string> = {}
+
+  @when('the ready probe is requested')
+  public async request (): Promise<void> {
+    const response = await request(`http://127.0.0.1:${PROBE}${PATH}`, { dispatcher: this.agent })
+
+    this.status = response.statusCode
+    this.headers = response.headers as Record<string, string>
+
+    await response.body.dump()
+  }
+
+  @then('the ready probe answers {int}')
+  public answers (status: number): void {
+    assert.equal(this.status, status)
+
+    if (status === 200)
+      assert.equal(this.headers['cache-control'], 'no-store')
+  }
+
+  @after()
+  public async close (): Promise<void> {
+    await this.agent.close()
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.exposition",
-  "version": "1.0.0-alpha.274",
+  "version": "1.0.0-alpha.276",
   "description": "Toa Exposition",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -46,6 +46,7 @@
     "transpile:passkeys": "tsc -p ./components/identity.passkeys",
     "transpile:otp": "tsc -p ./components/identity.otp",
     "features": "cucumber-js",
+    "features:h2c": "TOA_EXPOSITION_PROTOCOL=h2c TOA_AGENT_PROTOCOL=h2c cucumber-js",
     "features:security": "cucumber-js --tags @security",
     "features:octets": "cucumber-js features/octets.*"
   },
@@ -61,5 +62,5 @@
     },
     "testEnvironment": "node"
   },
-  "gitHead": "7df32eb2872155c45681c299036b1efbeefd6518"
+  "gitHead": "6f81af829c7bf8a2db5a409f3d60ea40d15583ef"
 }

package/readme.md

@@ -188,21 +188,15 @@ exposition:
 
 ## Ready probe
 
-The gateway serves `GET /.ready` on port `8000` for Kubernetes startup/readiness probes.
+The gateway serves `GET /.ready` on port `8004`, not on the port it serves traffic on, and
+telemetry's probe is disabled in its process.
 
-It returns `200` only after:
+It answers `200` only after the in-process identity composition has connected, initial route
+discovery has settled, and the HTTP server is listening. Before that it answers `503` with a
+`retry-after`. When ready, the process also sends `process.send('ready')` for PM2 `wait_ready`.
 
-1. The in-process identity composition has connected
-2. Initial route discovery has settled
-3. The HTTP server is listening
-
-Before listen, probes get connection refused; after listen but before ready, `503`.
-When ready, the process also sends `process.send('ready')` (for PM2 `wait_ready`).
-
-The nested identity composition shares this process. Deployment disables telemetry's port-`8001`
-probe (`TOA_TELEMETRY_READY=false`) so only the gateway `/.ready` is used.
-
-See also [telemetry ready probe](../telemetry/readme.md#ready-probe).
+See also [telemetry ready probe](../telemetry/readme.md#ready-probe) and
+[reserved ports](../../documentation/ports.md).
 
 ## See Also
 

package/schemas/annotation.cos.yaml

@@ -24,8 +24,19 @@ properties:
     patternProperties:
       ^.+$:
         type: string
+  service:
+    type: object
+    properties:
+      annotations:
+        type: object
+        patternProperties:
+          ^.+$:
+            type: string
   debug:
     type: boolean
+  protocol:
+    type: string
+    enum: [h1, h2c]
   /: {}
   authorities:
     type: object

package/source/Annotation.ts

@@ -1,10 +1,21 @@
 import type { Resources } from '@toa.io/operations'
+import type { Protocol } from './HTTP'
 
 export interface Annotation {
   authorities: Record<string, string>
+
+  /**
+   * `h2c` requires an ingress controller that proxies cleartext HTTP/2 upstream.
+   * See `documentation/protocol.md`.
+   */
+  protocol?: Protocol
   class?: string
   resources?: Resources
   annotations?: Record<string, string>
+
+  /** The Service, as opposed to the Ingress that `annotations` above describes. */
+  service?: { annotations?: Record<string, string> }
+
   debug?: boolean
   '/'?: object // parsed and validated by RTD.syntax.parse
 }

package/source/HTTP/Context.ts

@@ -6,7 +6,7 @@ import { Timing } from './Timing'
 import { type Format, formats, types } from './formats'
 import { read } from './messages'
 import type { OutgoingMessage } from './messages'
-import type * as http from 'node:http'
+import type { IncomingMessage } from './types'
 
 export class Context {
   public readonly id: string
@@ -87,11 +87,6 @@ export class Context {
   }
 }
 
-export interface IncomingMessage extends http.IncomingMessage {
-  url: string
-  method: string
-}
-
 interface Pipelines {
   body: Array<(input: unknown) => unknown>
   response: Array<(output: OutgoingMessage) => void | Promise<void>>

package/source/HTTP/Probe.ts

@@ -0,0 +1,112 @@
+import * as http from 'node:http'
+import { console } from 'openspan'
+
+/**
+ * The readiness probe, on a port of its own.
+ *
+ * It cannot share the port the gateway serves traffic on: kubelet speaks HTTP/1.1 only, and a
+ * cleartext HTTP/2 server answers nothing else on its port — there is no ALPN to negotiate with.
+ *
+ * It is the gateway's own readiness, not the process's. The identity composition nested in this
+ * process connects before route discovery has settled, so a probe tied to that composition would
+ * report ready while the gateway still had no routes and no open port.
+ */
+export class Probe {
+  private readonly server = http.createServer()
+  private readonly port: number
+  private readonly path: string
+
+  private ready = false
+  private startedAt = 0
+  private listening = false
+  private skipped = false
+
+  public constructor (port: number, path: string = PATH) {
+    this.port = port
+    this.path = path
+
+    this.server.on('request', (request, response) => this.listener(request, response))
+  }
+
+  public async listen (): Promise<void> {
+    this.startedAt = Date.now()
+
+    try {
+      await new Promise<void>((resolve, reject) => {
+        this.server.once('error', reject)
+        this.server.listen(this.port, () => {
+          this.server.removeListener('error', reject)
+          resolve()
+        })
+      })
+    } catch (error: any) {
+      // processes sharing a host share the port; in a pod nothing else claims it
+      if (error?.code === 'EADDRINUSE') {
+        this.skipped = true
+
+        console.warn('Ready probe port already in use, skipping', { port: this.port })
+
+        return
+      }
+
+      throw error
+    }
+
+    this.listening = true
+
+    // a probe answers while the process runs; it must never be the reason it keeps running
+    this.server.unref()
+  }
+
+  /** The gateway is listening and has routes: answer `200`. */
+  public complete (): void {
+    this.ready = true
+
+    // the IPC signal is not tied to the probe: a process that gave up the shared port is
+    // still ready, and pm2 `wait_ready` would otherwise block until `listen_timeout`
+    process.send?.('ready')
+  }
+
+  public async close (): Promise<void> {
+    this.ready = false
+
+    if (!this.listening || this.skipped)
+      return
+
+    this.listening = false
+
+    // keep-alive connections would otherwise hold the server handle, delaying the exit
+    this.server.closeAllConnections()
+
+    await new Promise<void>((resolve) => this.server.close(() => resolve()))
+  }
+
+  private listener (request: http.IncomingMessage, response: http.ServerResponse): void {
+    if (request.url !== this.path) {
+      response.writeHead(404).end()
+
+      return
+    }
+
+    if (this.ready)
+      response.writeHead(200, { 'cache-control': 'no-store' }).end()
+    else {
+      const remaining = Math.ceil((Date.now() - this.startedAt) / 1000).toString()
+
+      response.writeHead(503, { 'retry-after': remaining }).end()
+    }
+  }
+}
+
+/**
+ * Reserved for this probe. `8001` is the Telemetry readiness probe's, and `toa export` refuses a
+ * port claimed twice — `toa mono` and a local run put every service in one process.
+ */
+export const PROBE = 8004
+export const PATH = '/.ready'
+
+/**
+ * The initial delay of the readiness probe. The server does not sleep for it: whoever
+ * probes is the one that waits, and doing it here as well only delayed the process twice.
+ */
+export const DELAY = 3 // seconds