@toa.io/definitions 1.0.0-alpha.288

package/source/extensions.cadence/manifest.ts

@@ -0,0 +1,64 @@
+import * as schemas from './schemas.js'
+import type { Declaration } from './types.js'
+import type { Manifest } from '@toa.io/norm'
+
+/**
+ * The `cadence:` block of a component manifest: the operations called on a cadence, by the
+ * operation each one calls. Norm hands over the whole manifest, so what the declaration refers
+ * to is checked here rather than left to fail at boot.
+ *
+ * A component that declares nothing still has a declaration. `cadence: ~` is how one that only
+ * delays calls says so, and it has to produce a value or norm rejects the extension.
+ */
+export function manifest(
+  declaration: Declaration | null | undefined,
+  component: Manifest
+): Declaration {
+  const normalized: Record<string, unknown> = expand(declaration)
+
+  schemas.declaration.validate<Declaration>(normalized, 'Invalid cadence declaration')
+
+  for (const [endpoint, pulse] of Object.entries(normalized)) {
+    const operation = component.operations?.[endpoint]
+
+    if (operation === undefined)
+      throw new Error(`Pulse refers to undefined operation '${endpoint}'`)
+
+    if (!TYPES.has(operation.type))
+      throw new Error(
+        `Pulse '${endpoint}' must refer to an operation of the allowed types: ` +
+          [...TYPES].join(', ')
+      )
+
+    if (pulse.intervals > pulse.cycle)
+      throw new Error(
+        `Pulse '${endpoint}' splits a cycle of ${pulse.cycle} seconds into ` +
+          `${pulse.intervals} intervals, which is less than a second each`
+      )
+  }
+
+  return normalized
+}
+
+/** A pulse whose cycle is not split has nothing to state but the cycle. */
+function expand(declaration: Declaration | null | undefined): Record<string, unknown> {
+  if (declaration === null || declaration === undefined) return {}
+
+  // a cycle nothing splits is one interval, which is what the shorthand declares
+  return Object.fromEntries(
+    Object.entries(declaration).map(([endpoint, pulse]) => {
+      const declared: Record<string, unknown> =
+        typeof pulse === 'number' ? { cycle: pulse } : { ...pulse }
+
+      declared.intervals ??= 1
+
+      return [endpoint, declared]
+    })
+  )
+}
+
+/**
+ * The types a receiver may refer to. An operation that produces no side effects has nothing to
+ * be called periodically for.
+ */
+const TYPES = new Set(['transition', 'assignment', 'effect', 'unmanaged'])

package/source/extensions.cadence/schemas.ts

@@ -0,0 +1,10 @@
+import { resolve } from 'node:path'
+import * as schemas from '@toa.io/schemas'
+import type { Schema } from '@toa.io/schemas'
+import type { Annotation, Declaration } from './types.js'
+
+const path = resolve(import.meta.dirname, '../../schemas/extensions.cadence')
+const namespace = schemas.namespace(path)
+
+export const declaration: Schema<Declaration> = namespace.schema('declaration')
+export const annotation: Schema<Annotation> = namespace.schema('annotation')

package/source/extensions.cadence/types.ts

@@ -0,0 +1,41 @@
+/** What one component declares under `cadence:`, by the operation each one calls. */
+export type Declaration = Record<string, Pulse>
+
+export interface Pulse {
+  /** seconds one whole cycle takes */
+  cycle: number
+
+  /** intervals the cycle is split into; the gap between calls is `cycle / intervals` */
+  intervals: number
+}
+
+/** `context.delay` — a call to be made later, answering the id that cancels it. */
+export interface Delay {
+  (endpoint: string, request: object | null, options: Options): Promise<string>
+
+  /** Raises where the id was never issued. */
+  cancel: (id: string) => Promise<void>
+}
+
+export interface Options {
+  /** milliseconds from now */
+  interval: number
+
+  /**
+   * Milliseconds the call may be late and still be made, or `null` for no bound at all.
+   *
+   * Stated rather than defaulted, because only the caller knows whether a late call is still
+   * the right call: an unpaid order expires on time or not at all, and a report is wanted
+   * whenever it can be had. Nothing else in the system can tell those apart.
+   *
+   * An upper bound on lateness, not a promise of timeliness. It is bounded below by how often
+   * a dispatcher scans, so a bound shorter than that is one nothing can honour.
+   */
+  overdue: number | null
+}
+
+/** What an application states under `cadence:` in its context. */
+export interface Annotation {
+  /** seconds between passes over the calls waiting to be made */
+  discreteness?: number
+}

package/source/extensions.configuration/components.ts

@@ -0,0 +1,6 @@
+import { components as digest, type Components } from '../digest/read.js'
+
+/** The values component, which runs in the service of its own. */
+export function components(): Components {
+  return digest('extensions.configuration')
+}

package/source/extensions.configuration/const.ts

@@ -0,0 +1,30 @@
+import { environment } from '@toa.io/generic'
+import type { Source } from '@toa.io/core/types'
+
+/** Per-component variables: the local override and the secrets. */
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+/** The map of every configured component, on the values service. */
+export const VALUES = 'TOA_CONFIGURATION_VALUES'
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+
+/** Where the UI is mounted; `/configuration/*` belongs to the component's own API. */
+export const UI_PATH = '/.configuration'
+export const UI_PORT = 8003
+
+/**
+ * The port the UI is served on. `UI_PORT` is what a deployment publishes and what the chart
+ * renders; the variable is for a machine that runs more than one Toa process — a Toa checkout
+ * beside an application, or two applications — where one of them has to give it up.
+ */
+export const UI_PORT_ENV = 'TOA_CONFIGURATION_UI_PORT'
+export function uiPort(): number {
+  const value = environment.get(UI_PORT_ENV)
+
+  return value === undefined ? UI_PORT : Number(value)
+}
+
+export const EVENT = 'configuration.values.created'
+
+export const SOURCE: Source = { service: 'configuration' }

package/source/extensions.configuration/context.ts

@@ -0,0 +1,13 @@
+import type { Manifest } from './manifest.js'
+import type { Contribution } from '@toa.io/core/types'
+
+/** What this extension puts on the context of a component that declares it. */
+export function context(declaration: Manifest): Contribution {
+  const contribution: Contribution = { name: 'configuration', schema: declaration.schema }
+
+  // a value the schema declares a secret is one on the context, not a string
+  if (JSON.stringify(declaration.schema).includes('"secret"'))
+    contribution.imports = { '@toa.io/extensions.configuration': ['Secret'] }
+
+  return contribution
+}

package/source/extensions.configuration/deployment.ts

@@ -0,0 +1,159 @@
+import assert from 'node:assert'
+import {
+  type Dependency,
+  type Resources,
+  type Service,
+  type Variable,
+  type Variables
+} from '@toa.io/operations'
+import { add } from '@toa.io/generic'
+import { components } from './components.js'
+import { version } from '../version.js'
+import { EVENT, PREFIX, SECRET_RX, UI_PATH, UI_PORT, VALUES } from './const.js'
+import { epoch } from './epoch.js'
+import * as validators from './schemas.js'
+import type { Manifest } from './manifest.js'
+import type { context } from '@toa.io/norm'
+
+/** Where Toa's release publishes this service's image. An application takes it
+ *  instead of building one when its context says `registry.services: published`. */
+export const image = 'ghcr.io/toa-io/extension-configuration-values'
+
+export function deployment(
+  instances: Instance[],
+  annotation: Annotation = {}
+): Dependency {
+  const { resources, values } = split(annotation)
+
+  annotation = prepare(values, instances)
+
+  const variables: Variables = {}
+
+  for (const instance of instances) {
+    const values = annotation[instance.locator.id]
+
+    if (values === undefined) continue
+
+    const secrets = createSecrets(values)
+
+    if (secrets.length > 0) variables[instance.locator.label] = secrets
+  }
+
+  const service: Service = {
+    group: 'configuration',
+    name: 'values',
+    image,
+    version,
+    components: components().labels,
+    resources,
+    // the service that holds the values also serves the page that reads them
+    port: UI_PORT,
+    ingress: { path: UI_PATH },
+    variables: [
+      {
+        name: VALUES,
+        value: JSON.stringify(describe(instances, annotation))
+      }
+    ]
+  }
+
+  return { services: [service], variables, events: [EVENT] }
+}
+
+/** What the values service is given: the epoch, the schema and the defaults of every component. */
+export function describe(instances: Instance[], annotation: Annotation = {}): Values {
+  annotation = prepare(split(annotation).values, instances)
+
+  const values: Values = {}
+
+  for (const { locator, manifest } of instances) {
+    const deployed = annotation[locator.id]
+
+    values[locator.id] = {
+      epoch: epoch(manifest.schema),
+      schema: manifest.schema,
+      // what the Context says stands over what the manifest declares, value by value: a
+      // deployment that names one of them does not thereby unset the rest
+      defaults:
+        deployed === undefined
+          ? manifest.defaults
+          : add(structuredClone(deployed), manifest.defaults ?? {})
+    }
+  }
+
+  return values
+}
+
+function createSecrets(values: object): Variable[] {
+  const secrets: Variable[] = []
+
+  for (const value of Object.values(values)) {
+    if (typeof value === 'object' && value !== null)
+      secrets.push(...createSecrets(value as object))
+
+    if (typeof value !== 'string') continue
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) continue
+
+    const name = match.groups?.variable
+
+    assert.ok(name !== undefined)
+
+    secrets.push({
+      name: PREFIX + '_' + name,
+      secret: {
+        name: 'toa-configuration',
+        key: name
+      }
+    })
+  }
+
+  return secrets
+}
+
+/**
+ * The service's own resources, and the component values that are the rest of the annotation.
+ *
+ * Every key here names a component, so the one option the service has of its own needs a
+ * name that cannot be mistaken for one. `resources` is reserved: a component actually
+ * called that is written with its namespace, `default.resources`, which is what an id is
+ * anyway — the bare form is the shorthand.
+ */
+function split(annotation: Annotation): { resources?: Resources; values: Annotation } {
+  validators.annotation.validate(annotation)
+
+  const { resources, ...values } = annotation as Annotation & { resources?: Resources }
+
+  return { resources, values }
+}
+
+/** Validated, keyed by full component ids, and checked against the components that ask. */
+function prepare(annotation: Annotation, instances: Instance[]): Annotation {
+  const normalized: Annotation = {}
+  const requested = instances.map((instance) => instance.locator.id)
+
+  for (const [key, values] of Object.entries(annotation)) {
+    const id = key.includes('.') ? key : 'default.' + key
+
+    assert.ok(
+      requested.includes(id),
+      `Component '${id}' does not request configuration or does not exist.`
+    )
+
+    normalized[id] = values
+  }
+
+  return normalized
+}
+
+export type Annotation = Record<string, object>
+export type Instance = context.Dependency<Manifest>
+export type Values = Record<string, Entry>
+
+export interface Entry {
+  epoch: string
+  schema: object
+  defaults?: object
+}

package/source/extensions.configuration/epoch.ts

@@ -0,0 +1,22 @@
+import { createHash } from 'node:crypto'
+
+/** The same for the same schema, whatever the key order it was written in. */
+export function epoch(schema: object): string {
+  return createHash('sha256').update(canonical(schema)).digest('hex')
+}
+
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return '[' + value.map(canonical).join(',') + ']'
+
+  if (value !== null && typeof value === 'object') {
+    const object = value as Record<string, unknown>
+    const keys = Object.keys(object)
+      .filter((key) => object[key] !== undefined)
+      .sort()
+    const entries = keys.map((key) => JSON.stringify(key) + ':' + canonical(object[key]))
+
+    return '{' + entries.join(',') + '}'
+  }
+
+  return JSON.stringify(value) ?? 'null'
+}

package/source/extensions.configuration/index.ts

@@ -0,0 +1,10 @@
+export { manifest } from './manifest.js'
+export { components } from './components.js'
+export { deployment, describe, image } from './deployment.js'
+export { epoch } from './epoch.js'
+export { context } from './context.js'
+export * from './const.js'
+export * as schemas from './schemas.js'
+
+export type { Manifest } from './manifest.js'
+export type { Annotation, Entry, Instance, Values } from './deployment.js'

package/source/extensions.configuration/manifest.ts

@@ -0,0 +1,12 @@
+import * as schemas from './schemas.js'
+
+export function manifest(manifest: Manifest): Manifest {
+  schemas.manifest.validate(manifest)
+
+  return manifest
+}
+
+export interface Manifest {
+  schema: object
+  defaults?: Record<string, unknown>
+}

package/source/extensions.configuration/schemas.ts

@@ -0,0 +1,9 @@
+import { resolve } from 'node:path'
+import * as schemas from '@toa.io/schemas'
+import type { Schema } from '@toa.io/schemas'
+
+const path = resolve(import.meta.dirname, '../../schemas/extensions.configuration')
+const namespace = schemas.namespace(path)
+
+export const manifest: Schema<any> = namespace.schema('manifest')
+export const annotation: Schema<any> = namespace.schema('annotation')

package/source/extensions.exposition/Annotation.ts

@@ -0,0 +1,91 @@
+import type { Resources } from '@toa.io/operations'
+
+/** What the gateway speaks upstream. */
+export type Protocol = 'h1' | 'h2c'
+
+export interface Annotation {
+  authorities: Record<string, string>
+
+  /**
+   * `h2c` requires an ingress controller that proxies cleartext HTTP/2 upstream.
+   * See `documentation/protocol.md`.
+   */
+  protocol?: Protocol
+  class?: string
+  resources?: Resources
+  annotations?: Record<string, string>
+
+  /** The Service, as opposed to the Ingress that `annotations` above describes. */
+  service?: { annotations?: Record<string, string> }
+
+  debug?: boolean
+
+  /** The header the client address is read from; the connection's without one. See `ip.md`. */
+  ip?: string
+
+  /** Failed authentications an address may make; none are metered unless set. See `identity.md`. */
+  bouncer?: Bouncer
+
+  /** The authorization server this context exposes; none is exposed without it. */
+  oauth?: OAuth
+
+  /** JSON-RPC at `/.rpc`; none is served without it. See `documentation/rpc.md`. */
+  rpc?: RPC
+
+  /** MCP at `/.mcp`; none is served without it. See `documentation/mcp.md`. */
+  mcp?: MCP
+  '/'?: object // parsed and validated by RTD.syntax.parse
+}
+
+export interface OAuth {
+  /**
+   * Where a client sends the user to consent. The one endpoint of the flow that needs a
+   * person, and so the one this extension does not serve: an application builds it.
+   */
+  authorize: string
+
+  /** Paths a token may be restricted to, each advertised as a protected resource. */
+  resources?: string[]
+
+  /** What a client may ask for. Each is a role, or a scope within one. */
+  scopes?: string[]
+
+  /** Origins whose Client ID Metadata Documents may be read. None are without it. */
+  trust?: string[]
+
+  /** Dynamic Client Registration, off unless opened. */
+  registration?: 'open' | 'closed'
+}
+
+export interface RPC {
+  /**
+   * Calls one request may carry, 32 by default. One request is one authentication and one
+   * reply however many calls it makes, so what a caller may ask for at once is bounded.
+   */
+  batch?: number
+}
+
+export interface MCP {
+  /** What the server calls itself, which a client shows a user. */
+  name: string
+
+  /** What a model is told before it calls anything here. */
+  instructions?: string
+
+  /**
+   * Browser origins the endpoint answers, which the revision requires against DNS rebinding.
+   * None are admitted without it; a request carrying no `Origin` is not a browser's.
+   */
+  origins?: string[]
+
+  /** Whether the endpoint answers without a credential. It does not unless it says so. */
+  anonymous?: boolean
+}
+
+export interface Bouncer {
+  /** what an address may fail at once, 20 by default */
+  attempts?: number
+
+  /** seconds it takes to earn them back, 60 by default */
+  interval?: number
+}

package/source/extensions.exposition/components.ts

@@ -0,0 +1,6 @@
+import { components as digest, type Components } from '../digest/read.js'
+
+/** The identity components, which run inside the gateway. */
+export function components(): Components {
+  return digest('extensions.exposition')
+}

package/source/extensions.exposition/const.ts

@@ -0,0 +1,45 @@
+export const BRANCH_TTL = 1_800_000
+
+/** The broadcast the gateway and its tenants share. */
+export const CHANNEL = 'exposition'
+
+/** The replicas of the gateway decide together, whatever context they serve. */
+export const ATOM_GROUP = 'exposition'
+
+/**
+ * Where JSON-RPC is served, for every procedure at once. Pinned rather than configured: a
+ * path an application could choose is a path it could collide with a route of its own.
+ */
+export const RPC = '/.rpc'
+
+/** Calls one request may carry where the annotation does not say. */
+export const BATCH = 32
+
+/**
+ * Where the Model Context Protocol is served. Pinned for the reason `/.rpc` is: a path an
+ * application could choose is a path it could collide with a route of its own.
+ */
+export const MCP = '/.mcp'
+
+/**
+ * Where the resource tree and the page that reads it are served. Pinned for the reason
+ * `/.rpc` is: a path an application could choose is a path it could collide with a route of
+ * its own. Unlike those two it is not annotated — every entry it carries is what `OPTIONS`
+ * on that path already answers to the same caller, and what it adds is the enumeration.
+ */
+export const DISCOVERY = '/.discovery'
+
+/** Where the gateway serves. */
+export const PORT = 8000
+
+/**
+ * Reserved for the readiness probe. `8001` is the Telemetry readiness probe's, and `toa export`
+ * refuses a port claimed twice — `toa mono` and a local run put every service in one process.
+ */
+export const PROBE = 8004
+
+/**
+ * The initial delay of the readiness probe. The server does not sleep for it: whoever
+ * probes is the one that waits, and doing it here as well only delayed the process twice.
+ */
+export const DELAY = 3 // seconds

package/source/extensions.exposition/deployment.ts

@@ -0,0 +1,95 @@
+import assert from 'node:assert'
+import { type Dependency, type Service } from '@toa.io/operations'
+import { type Annotation } from './Annotation.js'
+import * as schemas from './schemas.js'
+import { shortcuts } from './shortcuts.js'
+import { components } from './components.js'
+import { parse } from './syntax/index.js'
+import { DELAY, PORT, PROBE } from './const.js'
+import { version } from '../version.js'
+
+/** Where Toa's release publishes this service's image. An application takes it
+ *  instead of building one when its context says `registry.services: published`. */
+export const image = 'ghcr.io/toa-io/extension-exposition-gateway'
+
+export function deployment(_: unknown, annotation?: Annotation): Dependency {
+  assert.ok(annotation !== undefined, 'Exposition context annotation is required')
+  schemas.annotation.validate(annotation)
+
+  const labels = components().labels
+
+  const service: Service = {
+    group: 'exposition',
+    name: 'gateway',
+    image,
+    port: PORT,
+    version,
+    variables: [],
+    components: labels,
+    resources: annotation.resources,
+    ingress: { path: '/', hosts: [] },
+    probe: {
+      path: '/.ready',
+      port: PROBE,
+      delay: DELAY
+    }
+  }
+
+  if (annotation?.['/'] !== undefined) {
+    const tree = parse(annotation['/'], shortcuts)
+
+    service.variables!.push({
+      name: 'TOA_EXPOSITION',
+      value: JSON.stringify(tree)
+    })
+  }
+
+  const { debug, authorities } = annotation
+
+  service.ingress!.hosts = Object.values(authorities)
+
+  // leaving these undefined lets the context's own ingress section supply them
+  if (annotation.class !== undefined) service.ingress!.class = annotation.class
+
+  if (annotation.annotations !== undefined)
+    service.ingress!.annotations = annotation.annotations
+
+  if (annotation.service?.annotations !== undefined)
+    service.annotations = annotation.service.annotations
+
+  const properties: Properties = { authorities }
+
+  if (debug === true) properties.debug = true
+
+  if (annotation.protocol !== undefined) properties.protocol = annotation.protocol
+
+  if (annotation.ip !== undefined) properties.ip = annotation.ip
+
+  if (annotation.bouncer !== undefined) properties.bouncer = annotation.bouncer
+
+  if (annotation.oauth !== undefined) properties.oauth = annotation.oauth
+
+  if (annotation.rpc !== undefined) properties.rpc = annotation.rpc
+
+  if (annotation.mcp !== undefined) properties.mcp = annotation.mcp
+
+  service.variables!.push({
+    name: 'TOA_EXPOSITION_PROPERTIES',
+    value: JSON.stringify(properties)
+  })
+
+  // The identity composition nested in this process connects before route discovery settles,
+  // so telemetry's probe — which tracks that composition — would report ready too early.
+  // The gateway answers for itself, on the same port telemetry would have used.
+  service.variables!.push({
+    name: 'TOA_TELEMETRY_READY',
+    value: JSON.stringify(false)
+  })
+
+  return { services: [service] }
+}
+
+type Properties = Pick<
+  Annotation,
+  'authorities' | 'debug' | 'protocol' | 'bouncer' | 'ip' | 'oauth' | 'rpc' | 'mcp'
+>

package/source/extensions.exposition/index.ts

@@ -0,0 +1,16 @@
+export { manifest } from './manifest.js'
+export { components } from './components.js'
+export { deployment, image } from './deployment.js'
+export { shortcuts } from './shortcuts.js'
+export * from './const.js'
+export * as schemas from './schemas.js'
+export * as syntax from './syntax/index.js'
+
+export type {
+  Annotation,
+  Bouncer,
+  MCP as MCPAnnotation,
+  OAuth,
+  Protocol,
+  RPC as RPCAnnotation
+} from './Annotation.js'