@toa.io/extensions.introspection 1.0.0-alpha.262

package/readme.md

@@ -0,0 +1,141 @@
+# Toa Introspection
+
+Collects and shows how the parts of an application talk to each other.
+
+Two things make that picture. Every component describes itself — what it can be asked to do, what it
+announces, what it reacts to — and that much is true before a single request is served. Every call
+between components is then recorded as it happens, along with what set it off. One says how the
+application is put together, the other what it actually does.
+
+It comes out as a page: a map of the components and the calls between them, and a list of what each
+one declares. Nothing has to be declared to get it — the extension takes part in every composition.
+
+## Configuration
+
+Both levels of the annotation are optional. Everything below shows the defaults.
+
+### Context
+
+```yaml
+# context.toa.yaml
+
+introspection:
+  samples: false      # capture the actual payloads of calls
+  interval: 15        # how often a component reports, seconds
+  threshold: 256      # report earlier once this many edges are pending
+  ui: true            # publish the UI
+  resources:          # explorer pod limits
+    cpu: [100m, 500m]
+    memory: [128Mi, 256Mi]
+```
+
+To turn introspection off completely — no explorer, no collection:
+
+```yaml
+introspection: false
+```
+
+Introspection exposes the map over HTTP and therefore requires the
+[exposition](../exposition) extension. An application that does not use exposition has to turn
+introspection off.
+
+### Component
+
+```yaml
+# manifest.toa.yaml
+
+introspection:
+  samples: false      # never capture this component's payloads
+```
+
+To keep a component off the map entirely:
+
+```yaml
+introspection: false
+```
+
+### Samples
+
+A sample is the actual input of a call, kept on the edge. It is production data, so it is off by
+default, and both the context and the component have to allow it — either can veto. A component that
+handles personal data should opt out permanently in its own manifest.
+
+Payloads of the `identity` namespace are never captured, keys that look like secrets are masked, and
+oversized payloads are dropped.
+
+## The UI
+
+The UI is published at `/.introspection`, on the hosts the context declares:
+
+```yaml
+# context.toa.yaml
+
+ingress:
+  hosts:
+    - api.example.com
+  class: alb
+  annotations:
+    alb.ingress.kubernetes.io/group.name: example
+```
+
+This section is what every service uses to reach the outside; without it the UI has nowhere to land
+and `toa export` says so. To collect the map without publishing anything, set `ui: false`.
+
+List the same hostnames Exposition serves — the page is static and reads the map from the API on its
+own origin, so a host Exposition does not serve gives a page that cannot load anything.
+
+Two services now share one host, which is a question for the ingress controller rather than for
+Toa: ingress-nginx merges them, while AWS ALB needs `alb.ingress.kubernetes.io/group.name` on both.
+The `annotations` above are applied to every service, which is the place to put it.
+
+Reading the map still needs the `system:introspection` role. The page itself is served without
+authentication — it is a page, and it displays nothing the API has not already granted.
+
+### Developing the UI
+
+The page is a Svelte application under `ui`, built into `ui/dist` — a directory of files the
+explorer serves, with no server rendering and nothing to run beside it. The mount path is baked
+into the bundle, because the ingress forwards it rather than rewriting it, and the client router
+matches against it.
+
+The build is git-ignored and produced on `npm publish`, so from a checkout it is a one-time step,
+like `npm run transpile`:
+
+```shell
+$ npm run build:ui                  # in this package
+```
+
+The explorer then answers on port `8002`, and `/` redirects to the mount path, so a local run
+opens at <http://localhost:8002>. To work on the page itself:
+
+```shell
+$ npm --prefix ui run dev           # http://localhost:5173/.introspection
+```
+
+Either way the API is the page's own origin — except on a local host, where the explorer and the
+gateway are separate ports and the page asks `:8000` instead. Nothing configures this; see
+`ui/src/config/index.ts`.
+
+## Reading the map
+
+The map lives in two ordinary components, `introspection.nodes` and `introspection.edges`. Both
+answer `enumerate` and `observe`, and both are exposed under the `system:introspection` role:
+
+```
+GET /introspection/nodes/
+GET /introspection/nodes/:id
+GET /introspection/edges/
+GET /introspection/edges/:id
+```
+
+Records carry `_updated`, which is when the node or edge was last observed — that is how a component
+that no longer exists is recognized.
+
+## Resources
+
+The explorer runs as the `introspection-explorer` service on port `8002`. It needs a database: the
+components use the context's `mongodb` annotation like any other.
+
+A port belongs to one service only — `toa export` refuses two claims on the same one, because
+`toa mono` and a local run put every service in one process. Taken so far: `8000` by the exposition
+gateway, `8001` by the telemetry readiness probe, `8002` here.

package/schemas/annotation.cos.yaml

@@ -0,0 +1,16 @@
+# context.toa.yaml introspection annotation
+samples: boolean
+interval:
+  type: integer
+  minimum: 1
+threshold:
+  type: integer
+  minimum: 1
+ui: boolean
+resources:
+  cpu: &constraint
+    type: array
+    items: { type: string }
+    minItems: 2
+    maxItems: 2
+  memory: *constraint

package/schemas/declaration.cos.yaml

@@ -0,0 +1,2 @@
+# manifest.toa.yaml introspection declaration
+samples: boolean

package/source/Composition.ts

@@ -0,0 +1,61 @@
+import { readdirSync, type Dirent } from 'node:fs'
+import { resolve } from 'node:path'
+import { Connector } from '@toa.io/core'
+import { type Bootloader } from './Factory'
+import type { Annotation } from './annotation'
+
+/** Hosts the introspection components in the explorer process. */
+export class Composition extends Connector {
+  private readonly boot: Bootloader
+
+  public constructor (boot: Bootloader) {
+    super()
+    this.boot = boot
+  }
+
+  protected override async open (): Promise<void> {
+    const paths = find()
+    const composition = await this.boot.composition(paths)
+
+    await composition.connect()
+
+    this.depends(composition)
+  }
+}
+
+export function find (): string[] {
+  return entries().map((entry) => resolve(ROOT, entry.name))
+}
+
+function entries (): Dirent[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries.filter((entry) => entry.isDirectory())
+}
+
+/**
+ * The extension is predefined, so an application that turns introspection off
+ * must not end up with the explorer components — nor with the exposition
+ * dependency they bring in.
+ */
+export function components (annotation?: Annotation): Components {
+  if (annotation === false)
+    return { labels: [], paths: [] }
+
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
+const ROOT = resolve(__dirname, '../components/')

package/source/Explorer.ts

@@ -0,0 +1,16 @@
+import { Connector } from '@toa.io/core'
+import { console } from 'openspan'
+
+/**
+ * The explorer process. It hosts the introspection components — the map is read
+ * through their own operations — and serves the UI.
+ */
+export class Explorer extends Connector {
+  protected override async open (): Promise<void> {
+    console.info('Introspection explorer started')
+  }
+
+  protected override dispose (): void {
+    console.info('Introspection explorer is closed')
+  }
+}

package/source/Factory.ts

@@ -0,0 +1,119 @@
+import { Connector } from '@toa.io/core'
+import { DISABLED, environment, component as declaration, settings } from './annotation'
+import { NAMESPACE, UI_PORT } from './const'
+import { describe } from './describe'
+import { Reporter } from './Reporter'
+import { Tenant } from './Tenant'
+import { Composition } from './Composition'
+import { Explorer } from './Explorer'
+import { UI } from './UI'
+import { capture, samplable } from './sample'
+import type { Declaration, Options, Settings } from './annotation'
+import type { Origin, Outcome, Target } from './model'
+import type { Manifest } from '@toa.io/norm'
+import type { Component, Locator, Reply, Request, extensions } from '@toa.io/core'
+
+export class Factory implements extensions.Factory {
+  private readonly boot: Bootloader
+  private readonly options: Options | null
+  private readonly settings: Record<string, Settings> = {}
+  private reporter: Reporter | null = null
+
+  public constructor (boot: Bootloader) {
+    this.boot = boot
+    this.options = environment()
+  }
+
+  public tenant (locator: Locator, decl: Declaration | null, manifest: Manifest): Connector {
+    const resolved = settings(locator.namespace, declaration(decl), this.options)
+
+    this.settings[locator.id] = resolved
+
+    if (!resolved.enabled || locator.namespace === NAMESPACE)
+      return new Connector()
+
+    return new Tenant(this.collector(), describe(manifest))
+  }
+
+  public component (component: Component): Component {
+    const locator = component.locator
+    const resolved = this.resolve(locator)
+
+    if (!resolved.enabled)
+      return component
+
+    const reporter = this.collector()
+    const invoke = component.invoke.bind(component)
+
+    component.invoke = async (endpoint: string, request: Request): Promise<any> => {
+      let outcome: Outcome = 'ok'
+      let reply: Reply | undefined
+
+      try {
+        reply = await invoke(endpoint, request)
+
+        if (reply?.exception !== undefined) outcome = 'exception'
+        else if (reply?.error !== undefined) outcome = 'error'
+
+        return reply
+      } catch (error) {
+        outcome = 'exception'
+
+        throw error
+      } finally {
+        // a call that failed is still a connection between two components
+        const src: Origin = request?.source ?? UNKNOWN
+        const dst: Target = { namespace: locator.namespace, component: locator.name, operation: endpoint }
+
+        const sample = resolved.samples && samplable(request?.input)
+          ? capture(request?.input, outcome)
+          : undefined
+
+        reporter.observe({ src, dst, sample })
+      }
+    }
+
+    component.depends(reporter)
+
+    return component
+  }
+
+  public service (): Connector | null {
+    if (this.options === null)
+      return null
+
+    const composition = new Composition(this.boot)
+    const explorer = new Explorer()
+
+    explorer.depends(composition)
+
+    if (this.options.ui)
+      explorer.depends(new UI(UI_PORT))
+
+    return explorer
+  }
+
+  /**
+   * `tenant()` runs before any component is created, so settings are warm.
+   * A component booted on its own (without a composition) falls back to
+   * the environment, with sampling off.
+   */
+  private resolve (locator: Locator): Settings {
+    if (locator.namespace === NAMESPACE)
+      return DISABLED
+
+    return this.settings[locator.id] ??
+      settings(locator.namespace, {}, this.options === null ? null : { ...this.options, samples: false })
+  }
+
+  private collector (): Reporter {
+    this.reporter ??= new Reporter(this.boot, this.options!)
+
+    return this.reporter
+  }
+}
+
+const UNKNOWN = { service: 'unknown' } as const
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+export type Bootloader = typeof import('@toa.io/boot')

package/source/Reporter.ts

@@ -0,0 +1,210 @@
+import { Connector, Locator } from '@toa.io/core'
+import { console } from 'openspan'
+import { EDGES, MAX_EDGES, NAMESPACE, NODES } from './const'
+import * as keys from './keys'
+import type { Bootloader } from './Factory'
+import type { Options } from './annotation'
+import type { Edge, Node } from './model'
+import type { Remote } from '@toa.io/core'
+
+/**
+ * Buffers what a process observes and flushes it into the introspection
+ * components.
+ *
+ * Nothing here is on the critical path. Reaching the explorer is a discovery,
+ * which waits as long as it takes, so the collector never holds up a component
+ * starting, running or stopping: it buffers until the connection is there, and
+ * when it has to choose it gives up on the data rather than on the application.
+ */
+export class Reporter extends Connector {
+  private readonly boot: Bootloader
+  private readonly options: Options
+  private readonly nodes = new Map<string, Node>()
+  private readonly edges = new Map<string, Edge>()
+
+  /** Holds a remote only once it is connected and usable. */
+  private readonly remotes: Record<string, Remote> = {}
+
+  private timer: NodeJS.Timeout | null = null
+  private flushing: Promise<void> | null = null
+  private acquiring = false
+  private dropped = 0
+
+  public constructor (boot: Bootloader, options: Options) {
+    super()
+
+    this.boot = boot
+    this.options = options
+  }
+
+  /** The static description of a component. */
+  public expose (node: Node): void {
+    this.nodes.set(keys.node(node.namespace, node.component), node)
+
+    void this.flush()
+  }
+
+  /** A call between two components. */
+  public observe (observed: Edge): void {
+    const id = keys.edge(observed.src, observed.dst)
+    const edge = this.edges.get(id)
+
+    if (edge === undefined) {
+      /*
+       * `source` arrives over the wire, so the number of distinct edges a process
+       * can hold has to be bounded regardless of what peers send — and of whether
+       * anyone is there to take them.
+       */
+      if (this.edges.size >= MAX_EDGES) {
+        this.dropped++
+
+        return
+      }
+
+      this.edges.set(id, observed)
+    } else if (observed.sample !== undefined)
+      edge.sample = observed.sample
+
+    if (this.edges.size >= this.options.threshold)
+      void this.flush()
+  }
+
+  protected override async open (): Promise<void> {
+    // deliberately not awaited: the explorer may not be there yet, or at all
+    this.acquire()
+
+    this.timer = setInterval(() => void this.flush(), this.options.interval * 1000)
+    this.timer.unref()
+  }
+
+  protected override async close (): Promise<void> {
+    if (this.timer !== null) {
+      clearInterval(this.timer)
+      this.timer = null
+    }
+
+    await this.flushing
+
+    if (!this.ready()) {
+      this.discard('the explorer was never reached')
+
+      return
+    }
+
+    // the remotes are still up: dependencies are disconnected after this returns
+    await this.dispatch().catch((error: Error) => {
+      console.debug('Introspection final flush failed', { message: error.message })
+    })
+  }
+
+  private ready (): boolean {
+    return NODES in this.remotes && EDGES in this.remotes
+  }
+
+  private async flush (): Promise<void> {
+    // a dispatch in flight keeps observations buffered, they join the next batch
+    if (this.flushing !== null)
+      return
+
+    if (this.nodes.size === 0 && this.edges.size === 0)
+      return
+
+    if (!this.ready()) {
+      this.acquire()
+
+      if (this.edges.size >= MAX_EDGES)
+        this.discard('the explorer is not reachable')
+
+      return
+    }
+
+    this.flushing = this.dispatch()
+      .catch((error: Error) => {
+        console.debug('Introspection flush failed', { message: error.message })
+      })
+      .finally(() => {
+        this.flushing = null
+      })
+
+    await this.flushing
+  }
+
+  private discard (reason: string): void {
+    const nodes = this.nodes.size
+    const edges = this.edges.size + this.dropped
+
+    this.nodes.clear()
+    this.edges.clear()
+    this.dropped = 0
+
+    if (nodes === 0 && edges === 0)
+      return
+
+    console.warn(`Introspection data discarded, ${reason}`, { nodes, edges })
+  }
+
+  private async dispatch (): Promise<void> {
+    const nodes = [...this.nodes.entries()]
+    const edges = [...this.edges.entries()]
+
+    this.nodes.clear()
+    this.edges.clear()
+
+    if (this.dropped > 0) {
+      console.warn('Introspection edges dropped', { dropped: this.dropped, limit: MAX_EDGES })
+      this.dropped = 0
+    }
+
+    await Promise.all([
+      this.merge(NODES, 'nodes', nodes),
+      this.merge(EDGES, 'edges', edges)
+    ])
+  }
+
+  /**
+   * A mass transition: every affected object is acquired and committed at once,
+   * so a flush is one call per component whatever it carries.
+   */
+  private async merge (name: string, property: string,
+    observed: Array<[string, Node | Edge]>): Promise<void> {
+    if (observed.length === 0)
+      return
+
+    const objects: Record<string, Node | Edge> = {}
+
+    for (const [id, object] of observed)
+      objects[id] = object
+
+    await this.remotes[name].invoke('merge', {
+      query: { ids: observed.map(([id]) => id) },
+      input: { [property]: objects },
+      task: true
+    })
+  }
+
+  /** Runs in the background: discovery waits for the explorer as long as it takes. */
+  private acquire (): void {
+    if (this.acquiring)
+      return
+
+    this.acquiring = true
+
+    void this.reach().catch((error: Error) => {
+      this.acquiring = false
+
+      console.error('Introspection cannot reach its explorer', { message: error.message })
+    })
+  }
+
+  private async reach (): Promise<void> {
+    await Promise.all([NODES, EDGES].map(async (name) => {
+      const remote = await this.boot.remote(new Locator(name, NAMESPACE))
+
+      this.depends(remote)
+
+      await remote.connect()
+
+      this.remotes[name] = remote
+    }))
+  }
+}

package/source/Tenant.ts

@@ -0,0 +1,47 @@
+import { setTimeout } from 'node:timers/promises'
+import { Connector } from '@toa.io/core'
+import { ANNOUNCE_INTERVAL } from './const'
+import type { Reporter } from './Reporter'
+import type { Node } from './model'
+
+/**
+ * Announces the static description of a component.
+ *
+ * Delivery is guaranteed, so the repeat is not about reliability — it keeps
+ * `_updated` fresh, which is how a removed component fades off the map.
+ */
+export class Tenant extends Connector {
+  private readonly reporter: Reporter
+  private readonly node: Node
+  private stopped = false
+
+  public constructor (reporter: Reporter, node: Node) {
+    super()
+
+    this.reporter = reporter
+    this.node = node
+
+    this.depends(reporter)
+  }
+
+  protected override async open (): Promise<void> {
+    this.reporter.expose(this.node)
+
+    void this.announce()
+  }
+
+  protected override dispose (): void {
+    this.stopped = true
+  }
+
+  private async announce (): Promise<void> {
+    while (!this.stopped) {
+      await setTimeout(ANNOUNCE_INTERVAL, undefined, { ref: false })
+
+      if (this.stopped)
+        break
+
+      this.reporter.expose(this.node)
+    }
+  }
+}