@toa.io/extensions.exposition 1.0.0-alpha.245 → 1.0.0-alpha.246

package/documentation/tracing.md

@@ -0,0 +1,70 @@
+# Tracing
+
+Each incoming request is processed within a [trace](../../telemetry/readme.md#tracing) —
+a server span is created, and its trace context is propagated to component invocations and
+emitted events, so all log entries and spans across the application caused by the request
+share the same `trace_id`.
+
+## The `ray` response header
+
+Each response contains a `ray` header with the trace ID of the request.
+It can be used to find related log entries, or be attached to a support ticket.
+
+Log entries are stamped with the trace ID regardless of the sampling decision,
+while the trace itself is only recorded if the trace is sampled (see [Sampling](#sampling)).
+
+```
+HTTP/1.1 201 Created
+ray: 0af7651916cd43dd8448eb211c80319c
+```
+
+## Continuing a trace
+
+By default, the trace starts at the gateway. If the request carries a trace context in one of the
+following headers, the trace is continued, that is, the incoming trace ID is used.
+
+### `traceparent`
+
+The [W3C Trace Context](https://www.w3.org/TR/trace-context/#traceparent-header) header, sent by
+clients instrumented with OpenTelemetry-compatible tooling, or by upstream proxies.
+
+```
+traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
+```
+
+In addition to the trace ID, it carries the ID of the client span, which becomes the *parent* of
+the gateway's server span, linking client-side and server-side spans into a single trace tree.
+
+### `ray`
+
+A bare trace ID (32 lowercase hex digits), for clients without tracing instrumentation willing to
+correlate their requests with server-side traces.
+
+```
+ray: 4bf92f3577b34da6a3ce929d0e0e4736
+```
+
+Unlike `traceparent`, it carries no parent span, so the gateway's server span becomes the root of
+the trace.
+
+If both headers are present, `traceparent` takes precedence as it carries more information.
+Headers with invalid values are ignored, and a new trace is started.
+
+## Sampling
+
+The trace *context* (trace ID) is always created and propagated, so the `ray` header and
+`trace_id` in log entries are always present. The sampling decision only determines whether
+the *spans* of the trace are recorded.
+
+The decision is made once, by the process that starts the trace, and is propagated along with
+the trace context (the flags byte of `traceparent`), so a trace is either recorded as a whole
+or not at all. Sampling is configured with the
+[`telemetry.traces` annotation](../../telemetry/readme.md#sampling).
+
+Sending a trace context does not bypass sampling: an incoming `ray` header adopts the trace ID,
+while the recording decision is still made by the gateway. The `sampled` flag of an incoming
+`traceparent` is respected, as the decision belongs to the trace root.
+
+## See also
+
+- [Telemetry extension](../../telemetry/readme.md) — spans, structured logs, `context.span`

package/features/steps/Gateway.ts

@@ -24,15 +24,12 @@ export class Gateway {
       process.env.TOA_EXPOSITION = encode(tree)
     }
 
-    const { debug, trace, authorities } = annotation
+    const { debug, authorities } = annotation
     const properties = Object.assign({}, DEFAULT_PROPERTIES)
 
     if (debug !== undefined)
       properties.debug = debug
 
-    if (trace !== undefined)
-      properties.trace = trace
-
     if (authorities !== undefined)
       properties.authorities = authorities
 

package/features/steps/Parameters.ts

@@ -1,7 +1,6 @@
 import { join } from 'node:path'
 import * as dotenv from 'dotenv'
 import { setDefaultTimeout } from '@cucumber/cucumber'
-import { console } from 'openspan'
 import { encode } from '@toa.io/generic'
 
 dotenv.config({ path: join(__dirname, '.env') })
@@ -16,10 +15,17 @@ export class Parameters {
 
 setDefaultTimeout(60 * 1000)
 
-console.configure({ format: 'terminal' })
-
 process.env.TOA_DEV = '1'
 
+// export traces to the local Tempo (`docker compose up tempo grafana`),
+// unavailability of the endpoint is harmless
+process.env.TOA_TELEMETRY_TRACES ??= encode({
+  exporters: {
+    console: null,
+    otlp: { endpoint: 'http://localhost:4318' }
+  }
+})
+
 process.env.TOA_STORAGES = encode({
   octets: {
     provider: 'tmp',

package/features/steps/components/audit/manifest.toa.yaml

@@ -0,0 +1,13 @@
+name: audit
+
+operations:
+  record:
+    input:
+      id: string
+      title: string
+      volume: number
+      price: number
+    output: undefined
+
+receivers:
+  orders.created: record

package/features/steps/components/audit/operations/record.js

@@ -0,0 +1,9 @@
+'use strict'
+
+async function effect (input) {
+  await new Promise((resolve) => setTimeout(resolve, 500))
+
+  return null
+}
+
+exports.effect = effect

package/features/steps/components/notify/manifest.toa.yaml

@@ -0,0 +1,13 @@
+name: notify
+
+operations:
+  send:
+    input:
+      id: string
+      title: string
+      volume: number
+      price: number
+    output: undefined
+
+receivers:
+  orders.created: send

package/features/steps/components/notify/operations/send.js

@@ -0,0 +1,9 @@
+'use strict'
+
+async function effect (input) {
+  await new Promise((resolve) => setTimeout(resolve, 800))
+
+  return null
+}
+
+exports.effect = effect

package/features/steps/components/orders/manifest.toa.yaml

@@ -0,0 +1,15 @@
+name: orders
+
+entity:
+  schema:
+    title: string
+    volume: number
+    price?: number
+
+operations:
+  create:
+    concurrency: retry
+    query: false
+    input:
+      title*: .
+      volume*: .

package/features/steps/components/orders/operations/create.js

@@ -0,0 +1,9 @@
+'use strict'
+
+async function transition (input, object, context) {
+  const price = await context.remote.default.pricing.quote({ input: { volume: input.volume } })
+
+  return Object.assign(object, { ...input, price })
+}
+
+exports.transition = transition

package/features/steps/components/pricing/manifest.toa.yaml

@@ -0,0 +1,7 @@
+name: pricing
+
+operations:
+  quote:
+    input:
+      volume*: number
+    output: number

package/features/steps/components/pricing/operations/quote.js

@@ -0,0 +1,7 @@
+'use strict'
+
+async function computation (input) {
+  return input.volume * 42
+}
+
+exports.computation = computation

package/features/timing.feature

@@ -8,26 +8,7 @@ Feature: Server timing
           POST: create
       """
 
-  Scenario: Server timing is not available by default
-    When the following request is received:
-      """
-      POST /pots/ HTTP/1.1
-      host: nex.toa.io
-      content-type: application/yaml
-
-      title: Hello
-      volume: 1.5
-      """
-    Then the reply does not contain:
-      """
-      server-timing:
-      """
-
-  Scenario: Server timing is sent when `trace` is enabled
-    Given the annotation:
-      """
-      trace: true
-      """
+  Scenario: Server timing is sent
     When the following request is received:
       """
       POST /pots/ HTTP/1.1
@@ -47,13 +28,12 @@ Feature: Server timing
   Scenario: Octets timing
     Given the annotation:
       """yaml
-      trace: true
       /:
         io:output: true
         auth:anonymous: true
         octets:context: octets
         POST:
-          octets:store: ~
+          octets:put: ~
       """
     When the stream of `lenna.png` is received with the following headers:
       """

package/features/tracing.feature

@@ -0,0 +1,123 @@
+Feature: Distributed tracing
+
+  Background:
+    Given the `pots` is running with the following manifest:
+      """yaml
+      exposition:
+        /:
+          POST: create
+      """
+
+  Scenario: Response carries the `ray` header
+    When the following request is received:
+      """
+      POST /pots/ HTTP/1.1
+      host: nex.toa.io
+      content-type: application/yaml
+
+      title: Hello
+      volume: 1.5
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      ray:
+      """
+
+  Scenario: Trace is continued from the `traceparent` request header
+    When the following request is received:
+      """
+      POST /pots/ HTTP/1.1
+      host: nex.toa.io
+      content-type: application/yaml
+      traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
+
+      title: Hello
+      volume: 1.5
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      ray: 0af7651916cd43dd8448eb211c80319c
+      """
+
+  Scenario: Trace is continued from the `ray` request header
+    When the following request is received:
+      """
+      POST /pots/ HTTP/1.1
+      host: nex.toa.io
+      content-type: application/yaml
+      ray: 4bf92f3577b34da6a3ce929d0e0e4736
+
+      title: Hello
+      volume: 1.5
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      ray: 4bf92f3577b34da6a3ce929d0e0e4736
+      """
+
+  Scenario: Trace spans remote calls, storage and events
+    Given the `identity.basic` database is empty
+    And the `pricing` is running
+    And the `orders` is running with the following manifest:
+      """yaml
+      exposition:
+        /:id:
+          auth:id: id
+          POST: create
+      """
+    And the `audit` is running
+    And the `notify` is running
+    When the following request is received:
+      """
+      POST /identity/ HTTP/1.1
+      host: nex.toa.io
+      authorization: Basic dXNlcjpwYXNzMTIzNA==
+      accept: application/yaml
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      authorization: Token ${{ token }}
+
+      id: ${{ id }}
+      roles: []
+      """
+    When the following request is received:
+      """
+      POST /orders/${{ id }}/ HTTP/1.1
+      host: nex.toa.io
+      authorization: Token ${{ token }}
+      content-type: application/yaml
+
+      title: Kettle
+      volume: 2
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      ray:
+      """
+
+  Scenario: Invalid `traceparent` starts a new trace
+    When the following request is received:
+      """
+      POST /pots/ HTTP/1.1
+      host: nex.toa.io
+      content-type: application/yaml
+      traceparent: not-a-valid-traceparent
+
+      title: Hello
+      volume: 1.5
+      """
+    Then the following reply is sent:
+      """
+      201 Created
+      ray:
+      """
+    And the reply does not contain:
+      """
+      ray: 0af7651916cd43dd8448eb211c80319c
+      """

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.exposition",
-  "version": "1.0.0-alpha.245",
+  "version": "1.0.0-alpha.246",
   "description": "Toa Exposition",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,20 +19,20 @@
   "dependencies": {
     "@aws-sdk/protocol-http": "3.370.0",
     "@simplewebauthn/server": "13.1.1",
-    "@toa.io/core": "1.0.0-alpha.243",
+    "@toa.io/core": "1.0.0-alpha.246",
     "@toa.io/generic": "1.0.0-alpha.225",
-    "@toa.io/schemas": "1.0.0-alpha.225",
+    "@toa.io/schemas": "1.0.0-alpha.246",
     "bcryptjs": "2.4.3",
     "content-type": "1.0.5",
     "error-value": "0.4.2",
     "jose": "5.10.0",
-    "js-yaml": "4.3.0",
+    "js-yaml": "4.3.1",
     "lru-cache": "11.2.7",
     "matchacho": "0.3.5",
     "minimatch": "10.2.4",
     "msgpackr": "1.11.9",
     "negotiator": "1.0.0",
-    "openspan": "1.0.0-alpha.173",
+    "openspan": "1.0.0-alpha.246",
     "paseto": "3.1.4"
   },
   "scripts": {
@@ -64,5 +64,5 @@
     },
     "testEnvironment": "node"
   },
-  "gitHead": "053dee53258c9adf64cb038fe638b2badd75ad09"
+  "gitHead": "d6ee2a7246c64a44c50c28fadcfd1e27aaf137cb"
 }

package/readme.md

@@ -136,7 +136,13 @@ exposition:
 | `class`       | Ingress class.                                                                                                    |
 | `annotations` | Ingress annotations.                                                                                              |
 | `debug`       | Output server errors. Default `false`.                                                                            |
-| `trace`       | Output [server timing](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing). Default `false`. |
+
+### Observability
+
+Each response contains a `ray` header with the trace ID of the request, and
+[server timing](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing) headers.
+
+Incoming trace context is continued, see [Tracing](documentation/tracing.md).
 
 ### Context resources
 

package/schemas/annotation.cos.yaml

@@ -9,5 +9,4 @@ resources:
   memory: *constraint
 annotations: <string>
 debug: boolean
-trace: boolean
 /: ~

package/source/Annotation.ts

@@ -6,6 +6,5 @@ export interface Annotation {
   resources?: Resources
   annotations?: Record<string, string>
   debug?: boolean
-  trace?: boolean
   '/'?: object // parsed and validated by RTD.syntax.parse
 }

package/source/Directive.ts

@@ -1,3 +1,4 @@
+import { console, type SpanOptions } from 'openspan'
 import type { Context, OutgoingMessage } from './HTTP'
 import type { Remotes } from './Remotes'
 import type { Output } from './io'
@@ -17,7 +18,8 @@ export class Directives implements RTD.Directives {
       if (set.family.preflight === undefined)
         continue
 
-      const out = await set.family.preflight(set.directives, context, parameters)
+      const out = await console.span(options(set, 'preflight'),
+        async () => await set.family.preflight!(set.directives, context, parameters))
 
       if (out === null)
         continue
@@ -34,7 +36,8 @@ export class Directives implements RTD.Directives {
   public async settle (context: Context, response: OutgoingMessage): Promise<void> {
     for (const set of this.sets)
       if (set.family.settle !== undefined)
-        await set.family.settle(set.directives, context, response)
+        await console.span(options(set, 'settle'),
+          async () => await set.family.settle!(set.directives, context, response))
   }
 
   public dispose (): void {
@@ -67,6 +70,8 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
     declarations.sort((a, b) =>
       (mandatory.has(b.family) ? 1 : 0) - (mandatory.has(a.family) ? 1 : 0))
 
+    const names: Record<string, string[]> = {}
+
     for (const declaration of declarations) {
       const family = this.families[declaration.family]
 
@@ -77,6 +82,8 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
 
       groups[family.name] ??= []
       groups[family.name].push(directive)
+      names[family.name] ??= []
+      names[family.name].push(`${declaration.family}:${declaration.name}`)
       mandatory.delete(family.name)
     }
 
@@ -85,13 +92,15 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
     for (const family of mandatory)
       sets.push({
         family: this.families[family],
-        directives: []
+        directives: [],
+        names: []
       })
 
     for (const [family, directives] of Object.entries(groups))
       sets.push({
         family: this.families[family],
-        directives
+        directives,
+        names: names[family]
       })
 
     const directives = new Directives(sets)
@@ -107,6 +116,15 @@ export class DirectivesFactory implements RTD.DirectiveFactory {
   }
 }
 
+function options (set: RTD.DirectiveSet, stage: 'preflight' | 'settle'): SpanOptions {
+  const options: SpanOptions = { name: `${set.family.name} ${stage}` }
+
+  if (set.names !== undefined && set.names.length > 0)
+    options.attributes = { directives: Array.from(new Set(set.names)).join(' ') }
+
+  return options
+}
+
 export const shortcuts: RTD.syntax.Shortcuts = new Map([
   ['anonymous', 'auth:anonymous'],
   ['anyone', 'auth:anyone'],

package/source/Factory.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert'
 import { createHash } from 'node:crypto'
-import { console } from 'openspan'
+import { console, traces, type LevelName, type TracesOptions } from 'openspan'
 import { decode } from '@toa.io/generic'
 import { Tenant } from './Tenant'
 import { Gateway } from './Gateway'
@@ -17,7 +17,6 @@ import type { Branch } from './Branch'
 import type { syntax } from './RTD'
 import type { Broadcast } from './Gateway'
 import type { Connector, Locator, extensions } from '@toa.io/core'
-import type { Channel } from 'openspan'
 
 export class Factory implements extensions.Factory {
   private readonly boot: Bootloader
@@ -72,13 +71,18 @@ export class Factory implements extensions.Factory {
 
 const CHANNEL = 'exposition'
 const LOGS_PREFIX = 'TOA_TELEMETRY_LOGS'
+const TRACES_ENV = 'TOA_TELEMETRY_TRACES'
 
 function configureLogs (): void {
   const globEnv = process.env[LOGS_PREFIX]
-  const level: Channel = process.env.TOA_DEV === '1' ? 'debug' : 'info'
-  const options = globEnv === undefined ? { level } : decode<{ level?: Channel }>(globEnv)
+  const level: LevelName = process.env.TOA_DEV === '1' ? 'trace' : 'info'
+  const options = globEnv === undefined ? { level } : decode<{ level?: LevelName }>(globEnv)
 
   console.configure({ level: options.level ?? level })
+
+  const tracesEnv = process.env[TRACES_ENV]
+
+  traces(tracesEnv === undefined ? {} : decode<TracesOptions>(tracesEnv))
 }
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-imports

package/source/HTTP/Context.ts

@@ -31,7 +31,7 @@ export class Context {
 
     this.id = crypto.randomUUID()
     this.url = new URL(request.url, `https://${request.headers.host}`)
-    this.timing = new Timing(properties.trace)
+    this.timing = new Timing()
     this.debug = properties.debug
     this.log(request)
 
@@ -95,7 +95,6 @@ interface Pipelines {
 
 interface Properties {
   debug: boolean
-  trace: boolean
 }
 
 const SUBTYPE = /^(?<type>\w{1,32})\/(vnd\.toa\.(?<subtype>\S{1,32})\+)(?<suffix>\S{1,32})$/