@toa.io/extensions.introspection 1.0.0-alpha.262

package/source/UI.ts

@@ -0,0 +1,197 @@
+import * as fs from 'node:fs'
+import * as http from 'node:http'
+import * as path from 'node:path'
+import { Connector } from '@toa.io/core'
+import { console } from 'openspan'
+import { UI_PATH } from './const'
+
+/**
+ * Serves the introspection UI: the directory `ui` builds, and nothing else.
+ *
+ * The page is a single-page application whose mount path is baked into the bundle,
+ * so this server routes relative to `UI_PATH` — which is also what the ingress
+ * forwards. `/introspection/*` is left alone: that is the components' own API.
+ */
+export class UI extends Connector {
+  private readonly server: http.Server = http.createServer()
+  private readonly port: number
+  private readonly root: string
+
+  public constructor (port: number, root: string = SITE) {
+    super()
+
+    this.port = port
+    this.root = root
+    this.server.on('request', (request, response) => {
+      this.listen(request, response)
+    })
+  }
+
+  protected override async open (): Promise<void> {
+    /*
+     * A taken port is a real error: uniqueness across services is settled at export
+     * time, so nothing here has to negotiate for one.
+     */
+    await new Promise<void>((resolve, reject) => {
+      const failed = (error: Error): void => {
+        this.server.off('listening', listening)
+        reject(error)
+      }
+
+      const listening = (): void => {
+        this.server.off('error', failed)
+        resolve()
+      }
+
+      this.server.once('error', failed)
+      this.server.once('listening', listening)
+      this.server.listen(this.port)
+    })
+
+    if (!isFile(path.join(this.root, 'index.html')))
+      console.warn('Introspection UI is not built', { root: this.root })
+
+    console.info('Introspection UI started', { port: this.port, path: UI_PATH + '/' })
+  }
+
+  protected override async close (): Promise<void> {
+    const closed = new Promise<void>((resolve) => this.server.once('close', () => {
+      resolve()
+    }))
+
+    this.server.close()
+    this.server.closeAllConnections()
+
+    await closed
+  }
+
+  private listen (request: http.IncomingMessage, response: http.ServerResponse): void {
+    void this.respond(request, response).catch((error: Error) => {
+      console.error('Introspection UI failure', { message: error.message })
+
+      if (!response.writableEnded)
+        response.writeHead(500).end()
+    })
+  }
+
+  private async respond (request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    if (request.method !== 'GET' && request.method !== 'HEAD') {
+      response.writeHead(405, { allow: 'GET, HEAD' }).end()
+
+      return
+    }
+
+    const pathname = decode(request.url ?? '/')
+
+    if (pathname === null) {
+      response.writeHead(400).end()
+
+      return
+    }
+
+    // the port is the explorer's alone, so the root is the way in
+    if (pathname === '/') {
+      response.writeHead(302, { location: UI_PATH + '/' }).end()
+
+      return
+    }
+
+    const file = this.resolve(pathname)
+
+    if (file === null)
+      response.writeHead(404).end()
+    else
+      await this.send(file, request, response)
+  }
+
+  /**
+   * The file a request lands on, or `null` when nothing does. A path that exists is
+   * served as it is; anything else that could be a route falls back to the page,
+   * because the client router — not this server — knows what routes there are.
+   */
+  private resolve (pathname: string): string | null {
+    if (!pathname.startsWith(UI_PATH))
+      return null
+
+    const relative = pathname.slice(UI_PATH.length)
+
+    if (relative !== '' && !relative.startsWith('/'))
+      return null
+
+    const file = path.join(this.root, relative)
+
+    if (file !== this.root && !file.startsWith(this.root + path.sep))
+      return null
+
+    if (isFile(file))
+      return file
+
+    /*
+     * A missing asset is missing, but a route can look like one: `identity.passkeys` is a
+     * component, not a file with an extension. What this server would have served is what
+     * it knows how to serve, so anything else is a route and falls back to the page — as
+     * does anything ending in a slash, which is no name for a file at all.
+     */
+    const asset = !relative.endsWith('/') && path.extname(relative) in TYPES
+
+    return asset ? null : path.join(this.root, 'index.html')
+  }
+
+  private async send (file: string, request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    const stats = await fs.promises.stat(file).catch(() => null)
+
+    if (stats === null) {
+      response.writeHead(503, { 'content-type': 'text/plain' })
+        .end('The introspection UI is not built. Run `npm run build:ui`.\n')
+
+      return
+    }
+
+    response.writeHead(200, {
+      'content-type': TYPES[path.extname(file)] ?? 'application/octet-stream',
+      'content-length': stats.size,
+      'cache-control': file.startsWith(path.join(this.root, IMMUTABLE)) ? FOREVER : 'no-cache'
+    })
+
+    if (request.method === 'HEAD')
+      response.end()
+    else
+      fs.createReadStream(file).pipe(response)
+  }
+}
+
+function decode (url: string): string | null {
+  try {
+    return decodeURIComponent(url.split('?')[0])
+  } catch {
+    return null
+  }
+}
+
+function isFile (file: string): boolean {
+  return fs.existsSync(file) && fs.statSync(file).isFile()
+}
+
+/** Where `npm run build:ui` puts the page, from both `source` and `transpiled`. */
+const SITE = path.resolve(__dirname, '..', 'ui', 'dist')
+
+/** Assets under this prefix carry their build hash in the name. */
+const IMMUTABLE = path.join('_app', 'immutable')
+const FOREVER = 'public, max-age=31536000, immutable'
+
+const TYPES: Record<string, string> = {
+  '.css': 'text/css; charset=utf-8',
+  '.html': 'text/html; charset=utf-8',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.map': 'application/json; charset=utf-8',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml',
+  '.txt': 'text/plain; charset=utf-8',
+  '.webmanifest': 'application/manifest+json',
+  '.webp': 'image/webp',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2'
+}

package/source/annotation.ts

@@ -0,0 +1,83 @@
+import { decode } from '@toa.io/generic'
+import { DEFAULT_INTERVAL, DEFAULT_THRESHOLD, DENIED, ENV } from './const'
+import type { Resources } from '@toa.io/operations'
+
+/** `context.toa.yaml` */
+export type Annotation = false | {
+  /** Capture real payloads. Off by default: this is production data. */
+  samples?: boolean
+  /** Flush period, seconds. */
+  interval?: number
+  /** Flush as soon as this many distinct edges are buffered. */
+  threshold?: number
+  /** Publish the UI. On by default. */
+  ui?: boolean
+  resources?: Resources
+}
+
+/** `manifest.toa.yaml` */
+export type Declaration = false | {
+  samples?: boolean
+}
+
+/** What `deployment()` encodes into the environment. */
+export interface Options {
+  samples: boolean
+  interval: number
+  threshold: number
+  ui: boolean
+}
+
+/** The effective per-component decision. */
+export interface Settings {
+  enabled: boolean
+  samples: boolean
+}
+
+export const DISABLED: Settings = { enabled: false, samples: false }
+
+export function options (annotation?: Annotation): Options {
+  const declaration = annotation === undefined || annotation === false ? {} : annotation
+
+  return {
+    samples: declaration.samples === true,
+    interval: declaration.interval ?? DEFAULT_INTERVAL,
+    threshold: declaration.threshold ?? DEFAULT_THRESHOLD,
+    ui: declaration.ui !== false
+  }
+}
+
+/** Reads what `deployment()` has put into the environment. */
+export function environment (): Options | null {
+  const value = process.env[ENV]
+
+  if (value === undefined)
+    return null
+
+  return decode<Options>(value)
+}
+
+export function component (declaration: Declaration | null | undefined): Declaration {
+  if (declaration === false)
+    return false
+
+  // predefined extensions arrive as null for components that say nothing
+  if (declaration === null || declaration === undefined)
+    return {}
+
+  return declaration.samples === undefined ? {} : { samples: declaration.samples }
+}
+
+/**
+ * Both levels must agree, and either can veto: the context is the environment
+ * ceiling, the manifest is the component's own call. A component handling
+ * personal data opts out for good, and no context flag overrides that.
+ */
+export function settings (namespace: string, declaration: Declaration, opts: Options | null): Settings {
+  if (opts === null || declaration === false)
+    return DISABLED
+
+  const samples = opts.samples && declaration.samples !== false && !DENIED.has(namespace)
+
+  return { enabled: true, samples }
+}

package/source/const.ts

@@ -0,0 +1,32 @@
+export const ID = 'introspection'
+export const NAMESPACE = 'introspection'
+
+export const ENV = 'TOA_INTROSPECTION'
+
+export const NODES = 'nodes'
+export const EDGES = 'edges'
+
+/** Where the UI is mounted; `/introspection/*` belongs to the components' own API. */
+export const UI_PATH = '/.introspection'
+export const UI_PORT = 8002
+
+export const DEFAULT_INTERVAL = 15
+export const DEFAULT_THRESHOLD = 256
+
+/** How often a component re-announces its description, so that removed components fade out. */
+export const ANNOUNCE_INTERVAL = 1_800_000
+
+/**
+ * `source` arrives over the wire, so the number of distinct edges a process
+ * can hold must be bounded regardless of what peers send.
+ */
+export const MAX_EDGES = 4096
+
+/** Never capture samples for these namespaces, whatever the annotation says. */
+export const DENIED = new Set(['identity', 'introspection'])
+
+/** Keys never stored in a sample, even when sampling is on. */
+export const REDACTED = /^(password|secret|token|credentials?|key|authorization|cookie)$/i
+
+/** Serialized size cap of a single sample. */
+export const SAMPLE_LIMIT = 4096

package/source/describe.ts

@@ -0,0 +1,68 @@
+import type { Node, Operation, Event, Receiver } from './model'
+import type { Manifest } from '@toa.io/norm'
+
+/**
+ * Turns a normalized component manifest into the node of the map.
+ *
+ * Everything here is already expanded by norm: concise schemas are resolved and
+ * the prototype chain is collapsed, so this is the shape the runtime actually runs.
+ */
+export function describe (manifest: Manifest): Node {
+  return {
+    namespace: manifest.namespace,
+    component: manifest.name,
+    version: manifest.version,
+    entity: entity(manifest),
+    operations: operations(manifest),
+    events: events(manifest),
+    receivers: receivers(manifest),
+
+    // names only: extension declarations carry configuration and secrets
+    extensions: Object.keys(manifest.extensions ?? {})
+  }
+}
+
+function entity (manifest: Manifest): Node['entity'] {
+  if (manifest.entity === undefined)
+    return null
+
+  return {
+    schema: manifest.entity.schema,
+    storage: manifest.entity.storage,
+    associated: manifest.entity.associated === true
+  }
+}
+
+function operations (manifest: Manifest): Operation[] {
+  return Object.entries(manifest.operations ?? {})
+    .map(([endpoint, definition]: [string, any]) => ({
+      endpoint,
+      type: definition.type,
+      scope: definition.scope,
+      query: definition.query,
+      input: definition.input ?? null,
+      output: definition.output ?? null,
+      errors: definition.errors ?? []
+    }))
+}
+
+function events (manifest: Manifest): Event[] {
+  return Object.entries(manifest.events ?? {})
+    .map(([label, definition]: [string, any]) => ({ label, binding: definition.binding }))
+}
+
+/*
+ * Receiver labels contain dots (`identity.bans.created`), which is why nodes keep
+ * these as arrays rather than maps — a dot in a document key is a hazard in Mongo.
+ */
+function receivers (manifest: Manifest): Receiver[] {
+  return Object.entries(manifest.receivers ?? {})
+    .map(([label, definition]: [string, any]) => ({
+      label,
+      source: definition.source ?? label.split('.').slice(0, 2).join('.'),
+      event: label.split('.').pop()!,
+      operation: definition.operation,
+      conditioned: definition.conditioned === true,
+      adaptive: definition.adaptive === true
+    }))
+}

package/source/extension.ts

@@ -0,0 +1,45 @@
+import { encode } from '@toa.io/generic'
+import { options } from './annotation'
+import { components } from './Composition'
+import { ENV, UI_PATH, UI_PORT } from './const'
+import * as schemas from './schemas'
+import type { Annotation } from './annotation'
+import type { Dependency, Instances, Service } from '@toa.io/operations'
+
+export const standalone = true
+
+/**
+ * The explorer hosts the introspection components, exactly as the exposition
+ * gateway hosts the identity ones. Collection is on unless the context says
+ * `introspection: false`, and the environment variable is emitted together
+ * with the service — never on its own, or tasks would pile up in a queue
+ * nothing consumes.
+ */
+export function deployment (_: Instances<unknown>, annotation?: Annotation): Dependency {
+  if (annotation === false)
+    return {}
+
+  if (annotation !== undefined)
+    schemas.annotation.validate(annotation)
+
+  const opts = options(annotation)
+
+  const service: Service = {
+    group: 'introspection',
+    name: 'explorer',
+    version: require('../package.json').version,
+    components: components().labels,
+    resources: annotation?.resources,
+    variables: []
+  }
+
+  if (opts.ui) {
+    service.port = UI_PORT
+    service.ingress = { path: UI_PATH }
+  }
+
+  return {
+    services: [service],
+    variables: { global: [{ name: ENV, value: encode(opts) }] }
+  }
+}

package/source/index.ts

@@ -0,0 +1,7 @@
+export { Factory } from './Factory'
+export { manifest } from './manifest'
+export { components } from './Composition'
+export * from './extension'
+
+export type { Annotation, Declaration, Options, Settings } from './annotation'
+export type * from './model'

package/source/manifest.ts

@@ -0,0 +1,19 @@
+import { component } from './annotation'
+import * as schemas from './schemas'
+import type { Declaration } from './annotation'
+
+/**
+ * The component level of the annotation.
+ *
+ * The extension is predefined, so most components say nothing and the
+ * declaration arrives as `null` — which still has to produce a value,
+ * or norm rejects the extension.
+ */
+export function manifest (declaration: Declaration | null | undefined): Declaration {
+  const normalized = component(declaration)
+
+  if (normalized !== false)
+    schemas.declaration.validate(normalized)
+
+  return normalized
+}

package/source/model.ts

@@ -0,0 +1,82 @@
+import type { Source } from '@toa.io/core'
+
+export type { Source }
+
+/**
+ * The caller side of an edge: the operation that made the call, the event that
+ * caused it, or the service the call came from.
+ */
+export type Origin = Source
+
+/** The callee side of an edge. */
+export interface Target {
+  namespace: string
+  component: string
+  operation: string
+}
+
+export interface Operation {
+  endpoint: string
+  type: string
+  scope?: string
+  query?: boolean
+  input?: object | null
+  output?: object | null
+  errors?: string[]
+}
+
+export interface Event {
+  label: string
+  binding?: string
+}
+
+export interface Receiver {
+  label: string
+  source: string
+  event: string
+  operation: string
+  conditioned: boolean
+  adaptive: boolean
+}
+
+export interface Entity {
+  schema: object
+  storage?: string
+  associated: boolean
+}
+
+/**
+ * The static description of a component, derived from its manifest.
+ * Sent to `introspection.nodes`.
+ */
+export interface Node {
+  namespace: string
+  component: string
+  version: string
+  entity: Entity | null
+  operations: Operation[]
+  events: Event[]
+  receivers: Receiver[]
+  extensions: string[]
+}
+
+export interface Sample {
+  at: number
+  input?: unknown
+  outcome: Outcome
+}
+
+export type Outcome = 'ok' | 'error' | 'exception'
+
+/**
+ * A call observed in a process between flushes. What the application declares —
+ * which events exist, and which receiver takes which — is a node's business;
+ * an edge is what actually happened.
+ *
+ * Sent to `introspection.edges`.
+ */
+export interface Edge {
+  src: Origin
+  dst: Target
+  sample?: Sample
+}

package/source/sample.ts

@@ -0,0 +1,49 @@
+import { Readable } from 'node:stream'
+import { REDACTED, SAMPLE_LIMIT } from './const'
+import type { Outcome, Sample } from './model'
+
+/**
+ * Captures the payload of a call. Only reached when both the context and the
+ * component have opted in, and never for a denied namespace.
+ */
+export function capture (input: unknown, outcome: Outcome): Sample {
+  return { at: Date.now(), input: redact(input), outcome }
+}
+
+export function samplable (input: unknown): boolean {
+  return !(input instanceof Readable)
+}
+
+function redact (value: unknown): unknown {
+  if (value === null || value === undefined || typeof value !== 'object')
+    return truncate(value)
+
+  if (Array.isArray(value))
+    return truncate(value.map(redact))
+
+  const result: Record<string, unknown> = {}
+
+  for (const [key, property] of Object.entries(value))
+    result[key] = REDACTED.test(key) ? '***' : redact(property)
+
+  return truncate(result)
+}
+
+/**
+ * The map holds one sample per edge indefinitely, so a single oversized
+ * payload must not become a permanent tenant of the collection.
+ */
+function truncate (value: unknown): unknown {
+  let serialized: string
+
+  try {
+    serialized = JSON.stringify(value) ?? ''
+  } catch {
+    return '[unserializable]'
+  }
+
+  if (serialized.length <= SAMPLE_LIMIT)
+    return value
+
+  return '[truncated]'
+}

package/source/schemas.ts

@@ -0,0 +1,10 @@
+import { resolve } from 'node:path'
+import schemas from '@toa.io/schemas'
+import type { Schema } from '@toa.io/schemas'
+import type { Annotation, Declaration } from './annotation'
+
+const path = resolve(__dirname, '../schemas')
+const namespace = schemas.namespace(path)
+
+export const annotation: Schema<Exclude<Annotation, false>> = namespace.schema('annotation')
+export const declaration: Schema<Exclude<Declaration, false>> = namespace.schema('declaration')

package/transpiled/Composition.d.ts

@@ -0,0 +1,21 @@
+import { Connector } from '@toa.io/core';
+import { type Bootloader } from './Factory';
+import type { Annotation } from './annotation';
+/** Hosts the introspection components in the explorer process. */
+export declare class Composition extends Connector {
+    private readonly boot;
+    constructor(boot: Bootloader);
+    protected open(): Promise<void>;
+}
+export declare function find(): string[];
+/**
+ * The extension is predefined, so an application that turns introspection off
+ * must not end up with the explorer components — nor with the exposition
+ * dependency they bring in.
+ */
+export declare function components(annotation?: Annotation): Components;
+interface Components {
+    labels: string[];
+    paths: string[];
+}
+export {};

package/transpiled/Composition.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.components = exports.find = exports.Composition = void 0;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const core_1 = require("@toa.io/core");
+/** Hosts the introspection components in the explorer process. */
+class Composition extends core_1.Connector {
+    boot;
+    constructor(boot) {
+        super();
+        this.boot = boot;
+    }
+    async open() {
+        const paths = find();
+        const composition = await this.boot.composition(paths);
+        await composition.connect();
+        this.depends(composition);
+    }
+}
+exports.Composition = Composition;
+function find() {
+    return entries().map((entry) => (0, node_path_1.resolve)(ROOT, entry.name));
+}
+exports.find = find;
+function entries() {
+    const entries = (0, node_fs_1.readdirSync)(ROOT, { withFileTypes: true });
+    return entries.filter((entry) => entry.isDirectory());
+}
+/**
+ * The extension is predefined, so an application that turns introspection off
+ * must not end up with the explorer components — nor with the exposition
+ * dependency they bring in.
+ */
+function components(annotation) {
+    if (annotation === false)
+        return { labels: [], paths: [] };
+    const labels = [];
+    const paths = [];
+    for (const entry of entries()) {
+        labels.push(entry.name.replace('.', '-'));
+        paths.push((0, node_path_1.resolve)(ROOT, entry.name));
+    }
+    return { labels, paths };
+}
+exports.components = components;
+const ROOT = (0, node_path_1.resolve)(__dirname, '../components/');
+//# sourceMappingURL=Composition.js.map

package/transpiled/Composition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Composition.js","sourceRoot":"","sources":["../source/Composition.ts"],"names":[],"mappings":";;;AAAA,qCAAkD;AAClD,yCAAmC;AACnC,uCAAwC;AAIxC,kEAAkE;AAClE,MAAa,WAAY,SAAQ,gBAAS;IACvB,IAAI,CAAY;IAEjC,YAAoB,IAAgB;QAClC,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IAClB,CAAC;IAEkB,KAAK,CAAC,IAAI;QAC3B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAA;QACpB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAEtD,MAAM,WAAW,CAAC,OAAO,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAC3B,CAAC;CACF;AAhBD,kCAgBC;AAED,SAAgB,IAAI;IAClB,OAAO,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,mBAAO,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;AAC5D,CAAC;AAFD,oBAEC;AAED,SAAS,OAAO;IACd,MAAM,OAAO,GAAG,IAAA,qBAAW,EAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;IAE1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;AACvD,CAAC;AAED;;;;GAIG;AACH,SAAgB,UAAU,CAAE,UAAuB;IACjD,IAAI,UAAU,KAAK,KAAK;QACtB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAElC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;QACzC,KAAK,CAAC,IAAI,CAAC,IAAA,mBAAO,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IACvC,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC1B,CAAC;AAbD,gCAaC;AAOD,MAAM,IAAI,GAAG,IAAA,mBAAO,EAAC,SAAS,EAAE,gBAAgB,CAAC,CAAA"}

package/transpiled/Explorer.d.ts

@@ -0,0 +1,9 @@
+import { Connector } from '@toa.io/core';
+/**
+ * The explorer process. It hosts the introspection components — the map is read
+ * through their own operations — and serves the UI.
+ */
+export declare class Explorer extends Connector {
+    protected open(): Promise<void>;
+    protected dispose(): void;
+}