@toa.io/operations 1.0.0-alpha.33 → 1.0.0-alpha.331

package/src/deployment/.deployment/.describe/compositions.js

@@ -1,11 +1,30 @@
-'use strict'
+import { addVariables } from './variables.js'
+import { addMounts } from './mounts.js'
+import { fold } from './fold.js'
 
-const { addVariables } = require('./variables')
-
-function compositions (compositions, variables) {
+export function compositions(compositions, dependency) {
   for (const composition of compositions) {
-    addVariables(composition, variables)
+    const claimed = (dependency.services ?? []).filter(
+      (service) => service.workload?.includes(composition.name) === true
+    )
+
+    // a service brings components of its own — the identity components inside the gateway —
+    // and their variables are keyed by their own labels, not by the composition's
+    const keys = composition.components.concat(
+      ...claimed.map((service) => service.components ?? [])
+    )
+
+    addVariables(composition, dependency.variables, keys)
+    addMounts(composition, dependency.mounts, keys)
+
+    if (claimed.length > 0) {
+      // the label each hosted service's own Service selects on
+      composition.hosted = claimed.map((service) => service.name)
+
+      fold(composition, claimed, dependency)
+    }
+
+    if (dependency.probe !== undefined && dependency.probe !== false)
+      composition.probe ??= dependency.probe
   }
 }
-
-exports.compositions = compositions

package/src/deployment/.deployment/.describe/dependencies.js

@@ -1,10 +1,8 @@
-'use strict'
-
 /**
  * @param {toa.deployment.dependency.Reference[]} references
  * @returns {*}
  */
-const dependencies = (references) => {
+export const dependencies = (references) => {
   return references?.reduce((map, reference) => {
     const { name, alias, values } = reference
 
@@ -13,6 +11,3 @@ const dependencies = (references) => {
     return map
   }, {})
 }
-
-
-exports.dependencies = dependencies

package/src/deployment/.deployment/.describe/events.js

@@ -0,0 +1,63 @@
+/**
+ * Tells each component which of its events something consumes. An event nobody consumes gets
+ * no emitter, no exchange and no outbox row, and a component none of whose events are consumed
+ * gets no outbox at all.
+ *
+ * A component that declares events always gets the variable, empty when nothing is consumed —
+ * an absent variable means every event, which is what a run without a deployment gets.
+ *
+ * @param {toa.norm.Context} context
+ * @param {toa.deployment.Dependency} dependency
+ */
+export function events(context, dependency) {
+  const components = deployed(context)
+  const consumed = collect(components, context.events, dependency.events)
+
+  for (const { locator, events } of components) {
+    if (events === undefined) continue
+
+    const labels = Object.keys(events).filter((label) =>
+      consumed.has(locator.id + '.' + label)
+    )
+
+    dependency.variables[locator.label] ??= []
+
+    dependency.variables[locator.label].push({
+      name: VARIABLE + locator.uppercase,
+      value: labels.join(' ')
+    })
+  }
+}
+
+/**
+ * Every component this context deploys, its own and those its extensions bring. `components`
+ * holds only the former; the latter reach a deployment as instances of what they depend on,
+ * which is also how they are given the rest of their variables.
+ */
+function deployed(context) {
+  const components = new Map()
+
+  for (const instances of Object.values(context.dependencies ?? {}))
+    for (const { component } of instances) components.set(component.locator.id, component)
+
+  return [...components.values()]
+}
+
+/**
+ * Everything that consumes an event of this context: the receivers of its own components, what
+ * a dependency declares it consumes, and what the context says is consumed outside it.
+ */
+function collect(components, declared, contributed) {
+  const consumed = new Set(declared ?? [])
+
+  for (const label of contributed ?? []) consumed.add(label)
+
+  for (const { receivers } of components)
+    for (const [label, receiver] of Object.entries(receivers ?? {}))
+      // a receiver with a source consumes from a foreign broker, not from this context
+      if (receiver.source === undefined) consumed.add(label)
+
+  return consumed
+}
+
+const VARIABLE = 'TOA_EVENTS_'

package/src/deployment/.deployment/.describe/fold.js

@@ -0,0 +1,42 @@
+/**
+ * What a workload takes on by running services in its own pod: their variables, the ports
+ * they bind, and a readiness probe answered by one of them rather than by the workload.
+ *
+ * What it does *not* take on is how they are reached. A composition-hosted service keeps
+ * its own Service and Ingress — only the pods behind them change — so the ingress union
+ * belongs to `mono`, the one workload that fronts every service under a single name.
+ *
+ * @param {object} workload
+ * @param {toa.deployment.dependency.Service[]} services
+ * @param {toa.deployment.Dependency} dependency
+ */
+export function fold(workload, services, dependency) {
+  workload.variables ??= []
+
+  for (const service of services) {
+    if (service.variables !== undefined)
+      for (const variable of service.variables)
+        if (!workload.variables.some((item) => item.name === variable.name))
+          workload.variables.push(variable)
+
+    // every declared port is bound by the single process, none is primary
+    if (service.port !== undefined)
+      (workload.backends ??= []).push({
+        port: service.port,
+        path: service.ingress?.path ?? '/'
+      })
+
+    if (service.probe !== undefined && service.probe !== false)
+      workload.probe = service.probe
+  }
+
+  if (
+    workload.probe === undefined &&
+    dependency.probe !== undefined &&
+    dependency.probe !== false
+  )
+    workload.probe = dependency.probe
+
+  // the more specific prefix must come first, whatever the controller's tie-break
+  workload.backends?.sort((a, b) => b.path.length - a.path.length)
+}

package/src/deployment/.deployment/.describe/fold.test.js

@@ -0,0 +1,108 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { fold } from './fold.js'
+
+const service = (name, extra = {}) => ({ group: 'group', name, version: '0', ...extra })
+
+it('should collect the ports of every service', () => {
+  const workload = {}
+
+  fold(workload, [service('one', { port: 8000 }), service('two', { port: 8002 })], {})
+
+  assert.deepStrictEqual(workload.backends, [
+    { port: 8000, path: '/' },
+    { port: 8002, path: '/' }
+  ])
+})
+
+it('should take the path a service declares', () => {
+  const workload = {}
+
+  fold(workload, [service('one', { port: 8002, ingress: { path: '/explorer' } })], {})
+
+  assert.deepStrictEqual(workload.backends, [{ port: 8002, path: '/explorer' }])
+})
+
+it('should put the more specific prefix first', () => {
+  const workload = {}
+
+  fold(
+    workload,
+    [
+      service('one', { port: 8000, ingress: { path: '/' } }),
+      service('two', { port: 8002, ingress: { path: '/explorer' } })
+    ],
+    {}
+  )
+
+  assert.deepStrictEqual(
+    workload.backends.map((backend) => backend.path),
+    ['/explorer', '/']
+  )
+})
+
+it('should leave a service without a port out of the backends', () => {
+  const workload = {}
+
+  fold(workload, [service('one')], {})
+
+  assert.strictEqual(workload.backends, undefined)
+})
+
+describe('variables', () => {
+  it('should take those of every service', () => {
+    const workload = {}
+
+    fold(
+      workload,
+      [
+        service('one', { variables: [{ name: 'A', value: '1' }] }),
+        service('two', { variables: [{ name: 'B', value: '2' }] })
+      ],
+      {}
+    )
+
+    assert.deepStrictEqual(workload.variables, [
+      { name: 'A', value: '1' },
+      { name: 'B', value: '2' }
+    ])
+  })
+
+  it('should keep the first of a name', () => {
+    const workload = { variables: [{ name: 'A', value: 'own' }] }
+
+    fold(workload, [service('one', { variables: [{ name: 'A', value: 'theirs' }] })], {})
+
+    assert.deepStrictEqual(workload.variables, [{ name: 'A', value: 'own' }])
+  })
+})
+
+describe('probe', () => {
+  const probe = { path: '/.ready', port: 8004 }
+  const fallback = { path: '/.ready', port: 8001 }
+
+  it('should take the probe of a service over the default', () => {
+    const workload = {}
+
+    fold(workload, [service('one', { port: 8000, probe })], { probe: fallback })
+
+    assert.deepStrictEqual(workload.probe, probe)
+  })
+
+  it('should fall back to the default', () => {
+    const workload = {}
+
+    fold(workload, [service('one', { port: 8000 })], { probe: fallback })
+
+    assert.deepStrictEqual(workload.probe, fallback)
+  })
+
+  it('should ignore a disabled probe', () => {
+    const workload = {}
+
+    fold(workload, [service('one', { port: 8000, probe: false })], { probe: false })
+
+    assert.strictEqual(workload.probe, undefined)
+  })
+})

package/src/deployment/.deployment/.describe/index.js

@@ -1,13 +1,4 @@
-'use strict'
-
-const { dependencies } = require('./dependencies')
-const { components } = require('./components')
-const { compositions } = require('./compositions')
-const { variables } = require('./variables')
-const { services } = require('./services')
-
-exports.dependencies = dependencies
-exports.components = components
-exports.compositions = compositions
-exports.variables = variables
-exports.services = services
+export { dependencies } from './dependencies.js'
+export { components } from './components.js'
+export { compositions } from './compositions.js'
+export { services } from './services.js'

package/src/deployment/.deployment/.describe/mounts.js

@@ -0,0 +1,17 @@
+export function addMounts(composition, mounts, keys = composition.components) {
+  if (mounts === undefined) return
+
+  const used = new Set()
+
+  for (const [key, mount] of Object.entries(mounts)) {
+    if (key !== 'global' && !keys?.includes(key)) continue
+
+    for (const { name, path, claim } of mount) {
+      if (used.has(name)) continue
+
+      composition.mounts ??= []
+      composition.mounts.push({ name, path, claim })
+      used.add(name)
+    }
+  }
+}

package/src/deployment/.deployment/.describe/resources.js

@@ -0,0 +1,129 @@
+/**
+ * Every deployment states what it may take. One that states nothing is `BestEffort`: the
+ * first evicted under memory pressure and the last given CPU under contention — which on
+ * a busy node is slow enough for its own startup probe to kill it, in a loop, while the
+ * chart reports nothing wrong. That is not a thing to find out in production.
+ *
+ * A deployment takes its own declaration, or the context's `resources` where it has none.
+ * Deploying without any is a decision rather than an omission, and it is spelled
+ * `resources: null` — at either place.
+ */
+export function resources(context, values) {
+  for (const unit of units(values, evicted(context))) {
+    // `null` is an answer and `undefined` is not one, so the fallback reads only the latter
+    if (unit.deployment.resources === undefined)
+      unit.deployment.resources = context.resources
+
+    if (unit.deployment.resources === undefined)
+      throw new Error(
+        `${unit.subject} declares no resources. ` +
+          "Declare them on it or as the context's 'resources', " +
+          "or 'resources: null' to deploy it without any."
+      )
+
+    heap(unit.deployment)
+  }
+}
+
+/**
+ * The memory limit sizes the heap. Node reads the machine's memory rather than the
+ * container's, so without this the heap grows past the limit and the pod is killed instead
+ * of collected. What is left of the limit is the process itself: its code, its buffers and
+ * its threads. A deployment that states `NODE_OPTIONS` of its own keeps them.
+ */
+function heap(deployment) {
+  const limit = deployment.resources?.memory?.[1]
+
+  if (limit === undefined) return
+
+  deployment.variables ??= []
+
+  if (deployment.variables.some((variable) => variable.name === NODE_OPTIONS)) return
+
+  const megabytes = Math.floor((quantity(limit) * HEAP_SHARE) / 2 ** 20)
+
+  deployment.variables.push({
+    name: NODE_OPTIONS,
+    value: `--max-old-space-size=${megabytes}`
+  })
+}
+
+/**
+ * A Kubernetes quantity, in bytes.
+ *
+ * @param {string | number} value
+ * @returns {number}
+ */
+export function quantity(value) {
+  const match = String(value).match(/^(\d+(?:\.\d+)?)([KMGTPE]i?|[kmun])?$/)
+
+  if (match === null) throw new Error(`'${value}' is not a quantity`)
+
+  const [, number, suffix = ''] = match
+  const scale = SUFFIXES[suffix]
+
+  if (scale === undefined) throw new Error(`'${value}' is not a memory quantity`)
+
+  return Number(number) * scale
+}
+
+const NODE_OPTIONS = 'NODE_OPTIONS'
+const HEAP_SHARE = 0.75
+
+const SUFFIXES = {
+  '': 1,
+  K: 1e3,
+  M: 1e6,
+  G: 1e9,
+  T: 1e12,
+  P: 1e15,
+  E: 1e18,
+  Ki: 2 ** 10,
+  Mi: 2 ** 20,
+  Gi: 2 ** 30,
+  Ti: 2 ** 40,
+  Pi: 2 ** 50,
+  Ei: 2 ** 60
+}
+
+/**
+ * The labels of the components the context evicts, whether or not the eviction was applied:
+ * `toa env -c` does not apply it, since a local run of an evicted component still needs its
+ * variables.
+ *
+ * @returns {Set<string>}
+ */
+function evicted(context) {
+  const ids = new Set(context.evicted?.components ?? [])
+
+  return new Set(
+    (context.components ?? [])
+      .filter(({ locator }) => ids.has(locator.id))
+      .map(({ locator }) => locator.label)
+  )
+}
+
+function* units(values, evicted) {
+  if (values.mono !== undefined)
+    yield { deployment: values.mono, subject: 'The mono deployment' }
+
+  for (const composition of values.compositions ?? []) {
+    // a composition of what the context evicts is here only for a local run of it, and Toa
+    // deploys none of it, so nothing asks what it may take
+    if (
+      composition.components?.length > 0 &&
+      composition.components.every((label) => evicted.has(label))
+    )
+      continue
+
+    yield { deployment: composition, subject: `Composition '${composition.name}'` }
+  }
+
+  for (const service of values.services ?? []) {
+    // a service a composition runs has no deployment of its own to size; the composition
+    // it runs in states what the pod may take
+    if (service.workload !== undefined) continue
+
+    yield { deployment: service, subject: `Service '${service.name}'` }
+  }
+}

package/src/deployment/.deployment/.describe/resources.test.js

@@ -0,0 +1,129 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { resources, quantity } from './resources.js'
+
+const declared = { cpu: ['100m', '1'], memory: ['100Mi', '1Gi'] }
+
+it('should take the declaration of a deployment', () => {
+  const values = { compositions: [{ name: 'edge', resources: declared }] }
+
+  resources({}, values)
+
+  assert.deepStrictEqual(values.compositions[0].resources, declared)
+})
+
+it('should fall back to the context', () => {
+  const values = { compositions: [{ name: 'edge' }] }
+
+  resources({ resources: declared }, values)
+
+  assert.deepStrictEqual(values.compositions[0].resources, declared)
+})
+
+it('should take `null` as an answer', () => {
+  const values = { compositions: [{ name: 'edge', resources: null }] }
+
+  assert.doesNotThrow(() => resources({ resources: declared }, values))
+  assert.strictEqual(values.compositions[0].resources, null)
+})
+
+describe('refusal', () => {
+  it('should refuse a composition that declares none', () => {
+    const values = { compositions: [{ name: 'edge' }] }
+
+    assert.throws(
+      () => resources({}, values),
+      (error) => /^Composition 'edge' declares no resources\./.test(error.message)
+    )
+  })
+
+  it('should refuse a service that declares none', () => {
+    const values = { services: [{ name: 'exposition-gateway' }] }
+
+    assert.throws(
+      () => resources({}, values),
+      (error) =>
+        /^Service 'exposition-gateway' declares no resources\./.test(error.message)
+    )
+  })
+
+  it('should refuse the mono deployment when it declares none', () => {
+    const values = { mono: {} }
+
+    assert.throws(
+      () => resources({}, values),
+      (error) => /^The mono deployment declares no resources\./.test(error.message)
+    )
+  })
+
+  it('should not ask a service a composition runs', () => {
+    // it has no deployment of its own to size; the composition running it states the pod's
+    const values = {
+      compositions: [{ name: 'edge', resources: declared }],
+      services: [{ name: 'exposition-gateway', workload: ['edge'] }]
+    }
+
+    assert.doesNotThrow(() => resources({}, values))
+    assert.strictEqual(values.services[0].resources, undefined)
+  })
+})
+
+describe('heap', () => {
+  it('should size the heap by the memory limit', () => {
+    const values = { compositions: [{ name: 'edge', resources: declared }] }
+
+    resources({}, values)
+
+    assert.deepStrictEqual(values.compositions[0].variables, [
+      { name: 'NODE_OPTIONS', value: '--max-old-space-size=768' }
+    ])
+  })
+
+  it('should size the heap of a service and of the mono deployment', () => {
+    const values = {
+      services: [
+        { name: 'exposition-gateway', resources: { memory: ['200Mi', '500Mi'] } }
+      ],
+      mono: { variables: [{ name: 'TOA_ENV', value: 'test' }] }
+    }
+
+    resources({ resources: declared }, values)
+
+    assert.deepStrictEqual(values.services[0].variables, [
+      { name: 'NODE_OPTIONS', value: '--max-old-space-size=375' }
+    ])
+
+    assert.deepStrictEqual(values.mono.variables, [
+      { name: 'TOA_ENV', value: 'test' },
+      { name: 'NODE_OPTIONS', value: '--max-old-space-size=768' }
+    ])
+  })
+
+  it('should keep the options a deployment states', () => {
+    const variables = [{ name: 'NODE_OPTIONS', value: '--max-old-space-size=100' }]
+    const values = { compositions: [{ name: 'edge', resources: declared, variables }] }
+
+    resources({}, values)
+
+    assert.deepStrictEqual(values.compositions[0].variables, variables)
+  })
+
+  it('should not size the heap without a memory limit', () => {
+    const values = { compositions: [{ name: 'edge', resources: { cpu: ['100m', '1'] } }] }
+
+    resources({}, values)
+
+    assert.strictEqual(values.compositions[0].variables, undefined)
+  })
+
+  it('should read a quantity', () => {
+    assert.strictEqual(quantity('512Mi'), 512 * 2 ** 20)
+    assert.strictEqual(quantity('1Gi'), 2 ** 30)
+    assert.strictEqual(quantity('500M'), 500e6)
+    assert.strictEqual(quantity('1.5Gi'), 1.5 * 2 ** 30)
+    assert.strictEqual(quantity(1024), 1024)
+    assert.throws(() => quantity('500m'), /not a memory quantity/)
+    assert.throws(() => quantity('big'), /not a quantity/)
+  })
+})

package/src/deployment/.deployment/.describe/services.js

@@ -1,11 +1,37 @@
-'use strict'
+import { addVariables } from './variables.js'
 
-const { addVariables } = require('./variables')
-
-function services (services, variables) {
+export function services(services, variables, probe, ingress) {
   for (const service of services) {
-    addVariables(service, variables)
+    // one a composition runs has no deployment of its own to carry them; the composition
+    // it runs in states them, and repeating them here is dead weight in the values
+    if (service.workload === undefined) addVariables(service, variables)
+
+    if (service.probe === false) delete service.probe
+    else if (service.probe === undefined && probe !== undefined && probe !== false)
+      service.probe = probe
+
+    if (service.ingress !== undefined) expose(service, ingress)
   }
 }
 
-exports.services = services
+/**
+ * A service declares only its own intent — a path, a port. Where that lands is
+ * the context's business, so everything else comes from its `ingress` section.
+ * What the service did declare wins.
+ */
+function expose(service, ingress = {}) {
+  const declared = Object.fromEntries(
+    Object.entries(service.ingress).filter(([, value]) => value !== undefined)
+  )
+
+  service.ingress = Object.assign({}, ingress, declared)
+
+  if (service.ingress.hosts === undefined)
+    throw new Error(
+      `Service '${service.name}' declares an ingress, but no hosts are defined. ` +
+        "Declare them in the context's 'ingress' section."
+    )
+
+  if (service.port === undefined)
+    throw new Error(`Service '${service.name}' declares an ingress, but no port.`)
+}

package/src/deployment/.deployment/.describe/services.test.js

@@ -0,0 +1,78 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { services } from './services.js'
+
+const service = (extra = {}) => ({
+  group: 'group',
+  name: 'group-one',
+  version: '0',
+  ...extra
+})
+
+it('should leave a service without an ingress alone', () => {
+  const list = [service()]
+
+  services(list, {}, undefined, { hosts: ['api.dev'] })
+
+  assert.strictEqual(list[0].ingress, undefined)
+})
+
+/*
+ * A service declares only where it wants to land; the cluster-level plumbing
+ * belongs to the context.
+ */
+it('should supply hosts, class and annotations from the context', () => {
+  const list = [service({ port: 8002, ingress: { path: '/.introspection' } })]
+
+  services(list, {}, undefined, {
+    hosts: ['api.dev'],
+    class: 'alb',
+    annotations: { a: 'b' }
+  })
+
+  assert.deepStrictEqual(list[0].ingress, {
+    path: '/.introspection',
+    hosts: ['api.dev'],
+    class: 'alb',
+    annotations: { a: 'b' }
+  })
+})
+
+it('should keep what the service declared itself', () => {
+  const list = [service({ port: 8000, ingress: { path: '/', hosts: ['own.dev'] } })]
+
+  services(list, {}, undefined, { hosts: ['api.dev'], class: 'alb' })
+
+  assert.deepStrictEqual(list[0].ingress.hosts, ['own.dev'])
+  assert.deepStrictEqual(list[0].ingress.class, 'alb')
+})
+
+it('should ignore properties the service left undefined', () => {
+  const list = [service({ port: 8000, ingress: { path: '/', class: undefined } })]
+
+  services(list, {}, undefined, { hosts: ['api.dev'], class: 'alb' })
+
+  assert.deepStrictEqual(list[0].ingress.class, 'alb')
+})
+
+it('should reject an ingress with nowhere to land', () => {
+  const list = [service({ port: 8002, ingress: { path: '/.introspection' } })]
+
+  assert.throws(
+    () => services(list, {}, undefined, undefined),
+    (error) =>
+      /Service 'group-one' declares an ingress, but no hosts are defined/.test(
+        error.message
+      )
+  )
+})
+
+it('should reject an ingress without a port', () => {
+  const list = [service({ ingress: { path: '/.introspection' } })]
+
+  assert.throws(
+    () => services(list, {}, undefined, { hosts: ['api.dev'] }),
+    (error) => /Service 'group-one' declares an ingress, but no port/.test(error.message)
+  )
+})