@toa.io/extensions.introspection 1.0.0-alpha.259

package/features/steps/Map.ts

@@ -0,0 +1,151 @@
+import assert from 'node:assert'
+import { resolve } from 'node:path'
+import { readdirSync } from 'node:fs'
+import { setTimeout } from 'node:timers/promises'
+import { MongoClient } from 'mongodb'
+import { after, before, binding, then, when } from 'cucumber-tsflow'
+import { parse } from '@toa.io/yaml'
+import { match } from '@toa.io/generic'
+import * as boot from '@toa.io/boot'
+import * as stage from '@toa.io/userland/stage'
+import { Factory } from '../../source'
+import type { Component, Connector, Request } from '@toa.io/core'
+
+@binding()
+export class Map {
+  private service: Connector | null = null
+  private composition: Connector | null = null
+  private remotes: Record<string, Component> = {}
+
+  @when('the `{word}` is called with:')
+  public async call (endpoint: string, yaml: string): Promise<void> {
+    const request = parse<Request>(yaml)
+    const [operation, component, namespace = 'default'] = endpoint.split('.').reverse()
+
+    // an operation may throw, and that is exactly what one of the scenarios is about
+    await this.invoke(`${namespace}.${component}`, operation, request).catch(() => undefined)
+  }
+
+  @then('the map contains a node:')
+  public async node (yaml: string): Promise<void> {
+    await this.eventually('introspection.nodes', parse(yaml))
+  }
+
+  @then('the map contains an edge:')
+  public async edge (yaml: string): Promise<void> {
+    await this.eventually('introspection.edges', parse(yaml))
+  }
+
+  @then('the map contains no node:')
+  public async noNode (yaml: string): Promise<void> {
+    await this.never('introspection.nodes', parse(yaml))
+  }
+
+  @then('the map contains no edge:')
+  public async noEdge (yaml: string): Promise<void> {
+    await this.never('introspection.edges', parse(yaml))
+  }
+
+  @when('the components are stopped')
+  public async stop (): Promise<void> {
+    await this.composition?.disconnect()
+
+    this.composition = null
+  }
+
+  @before('not @ui')
+  public async run (): Promise<void> {
+    await clean()
+
+    this.service = new Factory(boot).service()!
+
+    await this.service.connect()
+
+    // booted here rather than through the stage, so that a scenario can stop it on its own
+    this.composition = await boot.composition(components())
+
+    await this.composition.connect()
+  }
+
+  @after('not @ui')
+  public async shutdown (): Promise<void> {
+    this.remotes = {}
+
+    await this.composition?.disconnect()
+    await stage.shutdown()
+    await this.service?.disconnect()
+
+    this.composition = null
+    this.service = null
+  }
+
+  private async invoke (id: string, operation: string, request: Request): Promise<unknown> {
+    this.remotes[id] ??= await stage.remote(id)
+
+    return await this.remotes[id].invoke(operation, request)
+  }
+
+  /** The map is eventually consistent: the collector buffers, flushes, and the task is queued. */
+  private async eventually (id: string, expected: object): Promise<void> {
+    const deadline = Date.now() + DEADLINE
+
+    let records: any[] = []
+
+    while (Date.now() < deadline) {
+      records = await this.list(id)
+
+      if (records.some((record) => match(record, expected)))
+        return
+
+      await setTimeout(POLL)
+    }
+
+    assert.fail(`No record in '${id}' matches\n${JSON.stringify(expected, null, 2)}\n\n` +
+      `Present:\n${JSON.stringify(records, null, 2)}`)
+  }
+
+  /** Absence has to outlive a flush, otherwise it only proves the map is slow. */
+  private async never (id: string, expected: object): Promise<void> {
+    await setTimeout(SETTLE)
+
+    const records = await this.list(id)
+    const found = records.find((record) => match(record, expected))
+
+    assert.equal(found, undefined,
+      `Unexpected record in '${id}':\n${JSON.stringify(found, null, 2)}`)
+  }
+
+  private async list (id: string): Promise<any[]> {
+    return await this.invoke(id, 'enumerate', { query: { limit: 100 } }) as any[]
+  }
+}
+
+async function clean (): Promise<void> {
+  const client = new MongoClient(URL)
+
+  await client.connect()
+
+  const db = client.db(DB)
+
+  await Promise.all([
+    db.collection('introspection_nodes').deleteMany({}),
+    db.collection('introspection_edges').deleteMany({})
+  ])
+
+  await client.close()
+}
+
+function components (): string[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => resolve(ROOT, entry.name))
+}
+
+const ROOT = resolve(__dirname, 'components')
+const URL = 'mongodb://developer:secret@localhost:27017'
+const DB = 'toa-dev'
+const DEADLINE = 20_000
+const POLL = 250
+const SETTLE = 4_000

package/features/steps/UI.ts

@@ -0,0 +1,88 @@
+import assert from 'node:assert'
+import * as http from 'node:http'
+import { resolve } from 'node:path'
+import { after, binding, given, then, when } from 'cucumber-tsflow'
+import { UI } from '../../source/UI'
+import { UI_PORT } from '../../source/const'
+
+/**
+ * The scenarios are about the server, not about the page: it is pointed at a fixture
+ * directory, so that running them never needs a UI build.
+ */
+@binding()
+export class Site {
+  private server: UI | null = null
+  private response: Response | null = null
+
+  @given('the UI is published')
+  public async publish (): Promise<void> {
+    this.server = new UI(UI_PORT, resolve(__dirname, '..', 'site'))
+
+    await this.server.connect()
+  }
+
+  @when('{string} is requested')
+  public async request (path: string): Promise<void> {
+    this.response = await get(path)
+  }
+
+  @then('the status is {int}')
+  public statusIs (status: number): void {
+    assert.equal(this.response?.status, status)
+  }
+
+  @then('the body contains {string}')
+  public bodyContains (text: string): void {
+    assert.ok(this.response?.body.includes(text),
+      `Expected the body to contain '${text}', got '${this.response?.body}'`)
+  }
+
+  @then('the body is empty')
+  public bodyIsEmpty (): void {
+    assert.equal(this.response?.body, '')
+  }
+
+  @then('the {string} header is {string}')
+  public headerIs (name: string, value: string): void {
+    assert.equal(this.response?.headers[name], value)
+  }
+
+  @after()
+  public async shutdown (): Promise<void> {
+    await this.server?.disconnect()
+
+    this.server = null
+    this.response = null
+  }
+}
+
+/**
+ * Raw `http`, not `fetch`: the URL parser normalizes `%2e%2e` away, and one of the
+ * scenarios is about exactly that segment reaching the server.
+ */
+async function get (path: string): Promise<Response> {
+  return await new Promise((resolve, reject) => {
+    // `agent: false` — a keep-alive socket outlives the scenario that opened it
+    const request = http.get({ port: UI_PORT, path, agent: false }, (response) => {
+      let body = ''
+
+      response.setEncoding('utf8')
+      response.on('data', (chunk: string) => (body += chunk))
+      response.on('end', () => {
+        resolve({
+          status: response.statusCode!,
+          headers: response.headers as Record<string, string>,
+          body
+        })
+      })
+    })
+
+    request.on('error', reject)
+  })
+}
+
+interface Response {
+  status: number
+  headers: Record<string, string>
+  body: string
+}

package/features/steps/components/probe.discreet/manifest.toa.yaml

@@ -0,0 +1,12 @@
+namespace: probe
+name: discreet
+
+# on the map, but its payloads are never captured
+introspection:
+  samples: false
+
+operations:
+  ping:
+    input:
+      a: number
+    output: number

package/features/steps/components/probe.discreet/operations/ping.js

@@ -0,0 +1,7 @@
+'use strict'
+
+async function computation (input) {
+  return input.a
+}
+
+exports.computation = computation

package/features/steps/components/probe.quiet/manifest.toa.yaml

@@ -0,0 +1,11 @@
+namespace: probe
+name: quiet
+
+# opted out entirely: neither described nor instrumented
+introspection: false
+
+operations:
+  ping:
+    input:
+      a: number
+    output: number

package/features/steps/components/probe.quiet/operations/ping.js

@@ -0,0 +1,7 @@
+'use strict'
+
+async function computation (input) {
+  return input.a
+}
+
+exports.computation = computation

package/features/steps/components/probe.source/manifest.toa.yaml

@@ -0,0 +1,27 @@
+namespace: probe
+name: source
+
+entity:
+  associated: true
+  storage: null
+  schema:
+    value:
+      type: integer
+      default: 0
+    secret: string
+
+operations:
+  transit:
+    concurrency: retry
+    input:
+      value: .
+      secret: .
+  relay:
+    input:
+      a: number
+      b: number
+    output: number
+  crash: {}
+  fail:
+    errors:
+      - NOPE

package/features/steps/components/probe.source/operations/crash.js

@@ -0,0 +1,7 @@
+'use strict'
+
+function computation (input) {
+  throw new Error('crashed on purpose')
+}
+
+exports.computation = computation

package/features/steps/components/probe.source/operations/fail.js

@@ -0,0 +1,12 @@
+'use strict'
+
+function computation (input) {
+  return NOPE
+}
+
+const NOPE = Object.create(Error.prototype, {
+  code: { value: 'NOPE' },
+  message: { value: 'declined on purpose', enumerable: true }
+})
+
+exports.computation = computation

package/features/steps/components/probe.source/operations/relay.js

@@ -0,0 +1,7 @@
+'use strict'
+
+async function computation (input, context) {
+  return context.remote.probe.target.compute({ input })
+}
+
+exports.computation = computation

package/features/steps/components/probe.target/manifest.toa.yaml

@@ -0,0 +1,24 @@
+namespace: probe
+name: target
+
+entity:
+  storage: null
+  schema:
+    counted:
+      type: integer
+      default: 0
+
+operations:
+  compute:
+    input:
+      a: number
+      b: number
+    output: number
+  count:
+    concurrency: retry
+    input:
+      type: object
+      additionalProperties: true
+
+receivers:
+  probe.source.created: count

package/features/steps/components/probe.target/operations/compute.js

@@ -0,0 +1,7 @@
+'use strict'
+
+async function computation (input) {
+  return input.a + input.b
+}
+
+exports.computation = computation

package/features/steps/components/probe.target/operations/count.js

@@ -0,0 +1,9 @@
+'use strict'
+
+function transition (input, object) {
+  object.counted++
+
+  return object
+}
+
+exports.transition = transition

package/features/steps/config.ts

@@ -0,0 +1,20 @@
+import { setDefaultTimeout } from '@cucumber/cucumber'
+import { encode } from '@toa.io/generic'
+
+process.env.TOA_DEV = '1'
+
+/*
+ * The extension factory reads the environment once, and the bootloader caches
+ * factories per process — so the context level of the annotation is fixed for
+ * a whole run. `npm run features` therefore makes a pass per configuration.
+ */
+process.env.TOA_INTROSPECTION = encode({
+  samples: process.env.TOA_INTROSPECTION_SAMPLES === '1',
+  interval: Number(process.env.TOA_INTROSPECTION_INTERVAL ?? 1),
+  threshold: 64,
+
+  // `ui.feature` starts its own server; the explorer must not take the port first
+  ui: false
+})
+
+setDefaultTimeout(60 * 1000)

package/features/steps/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "noEmit": true,
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true
+  }
+}

package/features/ui.feature

@@ -0,0 +1,85 @@
+@ui
+Feature: Publishing the UI
+
+  The explorer serves the UI on its own port, out of the directory `ui` builds. The
+  mount path is baked into the bundle and forwarded by the ingress, so the server
+  routes relative to it.
+
+  Scenario: The root is the way in
+    Given the UI is published
+    When "/" is requested
+    Then the status is 302
+    And the "location" header is "/.introspection/"
+
+  Scenario: The UI answers at its mount path
+    Given the UI is published
+    When "/.introspection" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A trailing slash is the same resource
+    Given the UI is published
+    When "/.introspection/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A query string does not change the resource
+    Given the UI is published
+    When "/.introspection?tab=edges" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: An unknown route is the page
+    The client router knows what routes there are; this server does not.
+
+    Given the UI is published
+    When "/.introspection/whatever" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A route that looks like a file is still a route
+    A component is named `namespace.component`, so the address of one carries a dot and
+    an extension is no way to tell an asset from a route.
+
+    Given the UI is published
+    When "/.introspection/identity.passkeys/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: And with no trailing slash to give it away
+    Given the UI is published
+    When "/.introspection/identity.passkeys" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A hashed asset is served forever
+    Given the UI is published
+    When "/.introspection/_app/immutable/asset.js" is requested
+    Then the status is 200
+    And the "content-type" header is "text/javascript; charset=utf-8"
+    And the "cache-control" header is "public, max-age=31536000, immutable"
+
+  Scenario: The page itself is never cached
+    Given the UI is published
+    When "/.introspection/" is requested
+    Then the "cache-control" header is "no-cache"
+
+  Scenario: A missing asset is missing
+    Given the UI is published
+    When "/.introspection/_app/immutable/absent.js" is requested
+    Then the status is 404
+
+  Scenario: The path of the map API is not served by the UI
+    Given the UI is published
+    When "/introspection/nodes/" is requested
+    Then the status is 404
+
+  Scenario: A path that only looks like the mount path is absent
+    Given the UI is published
+    When "/.introspectionable" is requested
+    Then the status is 404
+
+  Scenario: Nothing outside the site is reachable
+    Given the UI is published
+    When "/.introspection/%2e%2e/%2e%2e/package.json" is requested
+    Then the status is 404

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@toa.io/extensions.introspection",
+  "version": "1.0.0-alpha.259",
+  "description": "Toa Introspection",
+  "author": "temich <tema.gurtovoy@gmail.com>",
+  "homepage": "https://github.com/toa-io/toa#readme",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toa-io/toa.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toa-io/toa/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.259",
+    "@toa.io/generic": "1.0.0-alpha.254",
+    "@toa.io/schemas": "1.0.0-alpha.254",
+    "openspan": "1.0.0-alpha.254"
+  },
+  "scripts": {
+    "transpile": "npx tsc && npx tsc -p ./components/introspection.nodes && npx tsc -p ./components/introspection.edges",
+    "build:ui": "npm --prefix ui ci && npm --prefix ui run build",
+    "prepack": "npm run build:ui",
+    "features": "npx cucumber-js --tags 'not @samples and not @shutdown' && TOA_INTROSPECTION_SAMPLES=1 npx cucumber-js --tags @samples && TOA_INTROSPECTION_INTERVAL=3600 npx cucumber-js --tags @shutdown"
+  },
+  "gitHead": "78bb1ba551bea0579ed61a2270dbc37536b12cbb"
+}

package/readme.md

@@ -0,0 +1,135 @@
+# Toa Introspection
+
+Introspection collects information about a product's topology and presents it as a graph of
+components and their interactions.
+
+The topology includes:
+
+- components and their entities, operations, events, and receivers;
+- declared event relations between components;
+- calls observed between components and services at runtime;
+- input and outcome samples for observed calls, when sampling is enabled.
+
+This provides a single view of both the product's declared structure and the communication paths
+that are exercised at runtime. The topology is available through a web UI and an HTTP API.
+
+Introspection is enabled by default for every component in a composition.
+
+## Configuration
+
+Configure Introspection in `context.toa.yaml`:
+
+```yaml
+introspection:
+  samples: false
+  interval: 300
+  threshold: 1024
+  ui: true
+```
+
+All properties are optional.
+
+| Property | Default | Description |
+| --- | --- | --- |
+| `samples` | `false` | Enables collection of call input and outcome samples. |
+| `interval` | `300` | Interval between topology updates, in seconds. |
+| `threshold` | `1024` | Number of distinct observed interactions that triggers an update before the interval expires. |
+| `ui` | `true` | Publishes the web UI. |
+| `resources` | — | Resource requirements for the Introspection deployment. |
+
+Resource requirements can be declared specifically for Introspection:
+
+```yaml
+introspection:
+  resources:
+    cpu: [100m, 500m]
+    memory: [128Mi, 256Mi]
+```
+
+If omitted, Introspection uses the context-level `resources` declaration. Set `resources: null` to
+run without resource requirements.
+
+Disable Introspection for the entire context:
+
+```yaml
+introspection: false
+```
+
+Introspection requires the [Exposition](../exposition) extension to publish its API. A context that
+does not use Exposition must disable Introspection.
+
+### Component configuration
+
+Sampling can be prohibited for an individual component in `manifest.toa.yaml`:
+
+```yaml
+introspection:
+  samples: false
+```
+
+This restriction cannot be overridden by the context configuration.
+
+Exclude a component from the topology entirely:
+
+```yaml
+introspection: false
+```
+
+## Samples
+
+A sample contains the input and outcome of an observed call. Sampling is disabled by default
+because call inputs may contain production or personal data.
+
+Samples are collected only when they are enabled in the context and not prohibited by the target
+component. Payloads in the `identity` namespace are never collected. Properties whose names match
+known secret patterns are redacted, streams are excluded, and oversized values are replaced with a
+truncation marker.
+
+Components that process sensitive data should disable sampling explicitly in their manifests.
+
+## Web UI
+
+The web UI visualizes the product topology and is enabled by default. It is published at
+`/.introspection/` on the hosts declared in the context:
+
+```yaml
+ingress:
+  hosts:
+    - api.example.com
+```
+
+Open the following URL after deploying the composition:
+
+```text
+https://api.example.com/.introspection/
+```
+
+The configured host must also be served by Exposition because the UI reads topology data from the
+Introspection API on the same origin.
+
+Access to topology data requires the `system:introspection` role. The UI page itself is public, but
+it cannot display topology data without an authorized API session.
+
+To collect topology data without publishing the UI:
+
+```yaml
+introspection:
+  ui: false
+```
+
+## HTTP API
+
+The topology is available through the following endpoints:
+
+```text
+GET /introspection/nodes/
+GET /introspection/nodes/:id/
+GET /introspection/edges/
+GET /introspection/edges/:id/
+```
+
+Nodes describe components and their declared interfaces. Edges represent calls observed between
+components or services. Both endpoint groups support listing all records and retrieving an
+individual record by its identifier.
+
+API access requires the `system:introspection` role.

package/schemas/annotation.cos.yaml

@@ -0,0 +1,21 @@
+# context.toa.yaml introspection annotation
+samples: boolean
+interval:
+  type: integer
+  minimum: 1
+threshold:
+  type: integer
+  minimum: 1
+ui: boolean
+resources:
+  # `null` deploys the service without any, which every deployment has to state one way
+  # or the other — see `migrations/263.md`.
+  type: object
+  nullable: true
+  properties:
+    cpu: &constraint
+      type: array
+      items: { type: string }
+      minItems: 2
+      maxItems: 2
+    memory: *constraint