@toa.io/operations 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/deployment/drain.js

@@ -0,0 +1,70 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+
+/**
+ * Waits until no pod of the application is still terminating. `helm --wait` answers once the
+ * replacements are ready, while the replicas they replace are still draining, and a request
+ * made then may land on either.
+ *
+ * @param {toa.operations.Process} process
+ * @param {toa.deployment.installation.Options} options
+ * @returns {Promise<void>}
+ */
+export async function drain(process, options) {
+  const timeout = options.timeout ?? DEFAULT_TIMEOUT
+  const deadline = Date.now() + duration(timeout)
+  const args = ['get', 'pods', '-o', 'json']
+
+  if (options.namespace !== undefined) args.push('-n', options.namespace)
+
+  let announced = false
+
+  for (;;) {
+    const output = await process.execute('kubectl', args, { silently: true })
+    const pods = JSON.parse(output).items.filter(terminating)
+
+    if (pods.length === 0) return
+
+    if (Date.now() >= deadline)
+      throw new Error(`${pods.length} replaced pod(s) still terminating after ${timeout}`)
+
+    if (!announced) {
+      console.log(`Waiting for ${pods.length} replaced pod(s) to terminate`)
+      announced = true
+    }
+
+    await sleep(INTERVAL)
+  }
+}
+
+/**
+ * A pod of the application on its way out. The chart labels every pod it renders under `toa/`,
+ * and nothing else in the namespace is waited for.
+ */
+const terminating = (pod) =>
+  pod.metadata.deletionTimestamp !== undefined &&
+  Object.keys(pod.metadata.labels ?? {}).some((label) => label.startsWith('toa/'))
+
+/**
+ * A helm duration, `12m` or `1h30m` or `90s`, in milliseconds.
+ *
+ * @param {string} value
+ * @returns {number}
+ */
+export function duration(value) {
+  const matches = value.matchAll(/(\d+)(h|ms|m|s)/g)
+
+  let total = 0
+
+  for (const [, amount, unit] of matches) total += Number(amount) * UNITS[unit]
+
+  if (total === 0) throw new Error(`'${value}' is not a duration`)
+
+  return total
+}
+
+const UNITS = { h: 3_600_000, m: 60_000, s: 1_000, ms: 1 }
+
+/** helm's own default for `--timeout` */
+const DEFAULT_TIMEOUT = '5m'
+
+const INTERVAL = 3_000

package/src/deployment/drain.test.js

@@ -0,0 +1,64 @@
+import { it, beforeEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { drain, duration } from './drain.js'
+
+/** @type {Array<object[]>} what each poll answers */
+let polls
+
+/** @type {toa.operations.Process} */
+let process
+
+beforeEach(() => {
+  polls = []
+  process = /** @type {toa.operations.Process} */ {
+    execute: mock.fn(async () => JSON.stringify({ items: polls.shift() ?? [] }))
+  }
+})
+
+it('should answer once no pod of the application is terminating', async () => {
+  polls.push([pod('composition-a', true), pod('composition-b')], [pod('composition-b')])
+
+  await drain(process, { namespace: 'acme', timeout: '1m' })
+
+  assert.strictEqual(process.execute.mock.callCount(), 2)
+
+  const [, args, options] = process.execute.mock.calls[0].arguments
+
+  assert.deepStrictEqual(args, ['get', 'pods', '-o', 'json', '-n', 'acme'])
+  assert.deepStrictEqual(options, { silently: true })
+})
+
+it('should not wait for a pod that is not the application', async () => {
+  polls.push([pod('redis-0', true, {})])
+
+  await drain(process, {})
+
+  assert.strictEqual(process.execute.mock.callCount(), 1)
+})
+
+it('should give up at the timeout', async () => {
+  polls.push([pod('composition-a', true)], [pod('composition-a', true)])
+
+  await assert.rejects(drain(process, { timeout: '1ms' }), /still terminating after 1ms/)
+})
+
+it('should read a helm duration', () => {
+  assert.strictEqual(duration('12m'), 720_000)
+  assert.strictEqual(duration('1h30m'), 5_400_000)
+  assert.strictEqual(duration('90s'), 90_000)
+  assert.throws(() => duration('soon'), /not a duration/)
+})
+
+/**
+ * @param {string} name
+ * @param {boolean} [terminating]
+ * @param {object} [labels]
+ */
+function pod(name, terminating = false, labels = { 'toa/composition': 'a' }) {
+  const metadata = { name, labels }
+
+  if (terminating) metadata.deletionTimestamp = '2026-01-01T00:00:00Z'
+
+  return { metadata }
+}

package/src/deployment/factory.js

@@ -1,77 +1,153 @@
-'use strict'
-
-const { context: load } = require('@toa.io/norm')
-const { Process } = require('../process')
-const { Operator } = require('./operator')
-const { Factory: ImagesFactory } = require('./images')
-const { Deployment } = require('./deployment')
-const { Registry } = require('./registry')
-const { Composition } = require('./composition')
-const { Service } = require('./service')
-
-class Factory {
+import { context as load, definition } from '@toa.io/norm'
+import { Process } from '../process.js'
+import { Operator } from './operator.js'
+import { Factory as ImagesFactory } from './images/index.js'
+import { Deployment } from './deployment.js'
+
+import { Registry } from './registry.js'
+import { Composition } from './composition.js'
+import { Service } from './service.js'
+
+export class Factory {
   #context
+  #mono
   #compositions
   #dependencies
   #registry
   #process
-  #extensionComponents = []
+  #image
+
+  /** Which compositions run a given extension's service, rather than deploying it on its
+   *  own. A service is stateless and already runs several replicas, so several compositions
+   *  running one are replicas of it, behind the one Service that selects them all.
+   *  @type {Map<string, string[]>} */
+  #claims
+
+  /** what this context does not deploy, by package reference
+   *  @type {Set<string>} */
+  #evicted
 
-  constructor (context) {
+  /** the extensions some composition lists, whether or not that composition is deployed
+   *  @type {Set<string>} */
+  #listed
+
+  constructor(context, options = {}) {
     this.#context = context
+    this.#mono = options.mono === true
     this.#process = new Process()
 
-    const imagesFactory = new ImagesFactory(context.name, context.runtime, context.registry)
+    const imagesFactory = new ImagesFactory(
+      context.name,
+      context.runtime,
+      context.registry
+    )
+
+    const compositions = deployed(context.compositions)
 
     this.#registry = new Registry(context.registry, imagesFactory, this.#process)
-    this.#compositions = context.compositions.map((composition) => this.#composition(composition))
+    this.#claims = claims(compositions)
+    this.#listed = listed(context.compositions)
+    this.#evicted = new Set(context.evicted?.services ?? [])
     this.#dependencies = this.#getDependencies()
+    this.#compositions = []
+
+    if (this.#mono)
+      this.#image = this.#registry.mono({
+        components: context.components.filter(managed),
+        // mono runs every service, so it installs what every one of them brings
+        packages: context.packages
+      })
+    else
+      this.#compositions = compositions.map((composition) => this.#composition(composition))
   }
 
-  operator () {
-    const deployment = new Deployment(this.#context, this.#compositions, this.#dependencies, this.#process)
-
-    return new Operator(deployment, this.#registry)
+  async operator() {
+    const deployment = new Deployment(
+      this.#context,
+      this.#compositions,
+      // the constructor cannot await, so what it started is settled here
+      await this.#dependencies,
+      this.#process,
+      this.#image
+    )
+
+    return new Operator(deployment, this.#registry, this.#context.environment)
   }
 
-  registry () {
+  registry() {
     return this.#registry
   }
 
-  #composition (composition) {
+  #composition(composition) {
     const image = this.#registry.composition(composition)
 
     return new Composition(composition, image)
   }
 
-  #getDependencies () {
+  async #getDependencies() {
     /** @type {toa.deployment.Dependency[]} */
     const dependencies = []
 
     if (this.#context.dependencies === undefined) return dependencies
 
+    /** the claimed references that turned out to contribute a service */
+    const contributing = new Set()
+
     for (const [reference, instances] of Object.entries(this.#context.dependencies)) {
-      const dependency = this.#getDependency(reference, instances)
+      const dependency = await this.#getDependency(reference, instances)
+
+      if (dependency === undefined) continue
 
-      if (dependency !== undefined) dependencies.push(dependency)
+      if (dependency.services?.length > 0) contributing.add(reference)
+
+      dependencies.push(dependency)
     }
 
+    // this is the first point where every `deployment()` has run, so it is the first
+    // point where a reference that yields nothing can be told from one that yields a service
+    for (const [reference, compositions] of this.#claims)
+      if (!contributing.has(reference))
+        throw new Error(
+          `Composition '${compositions[0]}' lists '${reference}', ` +
+            'which contributes no service.'
+        )
+
     return dependencies
   }
 
-  #getDependency (path, instances) {
-    const module = require(path)
-    const pkg = require(path + '/package.json')
+  async #getDependency(reference, instances) {
+    const { name, module } = await definition(reference)
 
     if (module.deployment === undefined) return
 
-    const annotation = this.#context.annotations?.[pkg.name]
+    // an evicted component is deployed by other means, and so is whatever only evicted
+    // components require; a service a composition lists is deployed for being listed
+    const managed = instances.filter(({ component }) => component.evicted !== true)
 
+    if (managed.length === 0 && instances.length > 0 && !this.#listed.has(reference)) return
+
+    const annotation = this.#context.annotations?.[name]
+
+    // every instance as well: what a definition keeps for all of its components, such as the
+    // configuration the values service serves, holds for an evicted one too
     /** @type {toa.deployment.dependency.Declaration} */
-    const dependency = module.deployment(instances, annotation)
+    const dependency = module.deployment(managed, annotation, instances)
+
+    // mono claims every service, including one an extension added since this context was
+    // written, so its claim is the wildcard rather than a list
+    const workload = this.#mono ? [MONO] : this.#claims.get(reference)
+
+    // an evicted service is deployed nowhere, so nothing is prepared for it: no image, and
+    // no Deployment, Service or Ingress downstream
+    const declared = this.#evicted.has(reference) ? undefined : dependency.services
 
     /** @type {toa.deployment.Service[]} */
-    const services = dependency.services?.map((service) => this.#service(path, service))
+    const services = declared?.map((service) =>
+      workload === undefined
+        ? this.#service(reference, service) // its own deployment, its own image
+        : // named the way `Service` would name it, since it skips that wrapper
+          { ...service, name: `${service.group}-${service.name}`, workload }
+    )
 
     return { ...dependency, services }
   }
@@ -81,17 +157,73 @@ class Factory {
    * @param service {toa.deployment.dependency.Service}
    * @returns {Service}
    */
-  #service (path, service) {
-    const image = this.#registry.service(path, service)
+  #service(path, service) {
+    // an extension that publishes its service's image is taken at its word: nothing is
+    // added to the registry, so nothing is prepared, probed, built or pushed for it
+    const published =
+      service.image !== undefined && this.#context.registry?.services === PUBLISHED
+
+    const image = published
+      ? { reference: `${service.image}:${this.#context.runtime.version}` }
+      : this.#registry.service(path, service)
 
     return new Service(service, image)
   }
 
-  static async create (path, environment) {
-    const context = await load(path, environment)
+  static async create(path, environment, options = {}) {
+    const context = await load(path, environment, options)
 
-    return new Factory(context)
+    return new Factory(context, options)
   }
 }
 
-exports.Factory = Factory
+const MONO = 'mono'
+
+const PUBLISHED = 'published'
+
+/**
+ * The compositions this context deploys: each without its evicted members, and none left with no
+ * members at all — the name it held is free again, and a service it listed falls back to a
+ * deployment of its own.
+ *
+ * @param {toa.norm.Composition[]} [compositions]
+ * @returns {toa.norm.Composition[]}
+ */
+function deployed(compositions = []) {
+  return compositions
+    .map((composition) => ({
+      ...composition,
+      components: composition.components.filter(managed)
+    }))
+    .filter((composition) => composition.components.length > 0)
+}
+
+/** An evicted component is deployed by other means. */
+function managed(component) {
+  return component.evicted !== true
+}
+
+/**
+ * @param {toa.norm.Composition[]} [compositions]
+ * @returns {Set<string>}
+ */
+function listed(compositions = []) {
+  return new Set(compositions.flatMap((composition) => composition.services ?? []))
+}
+
+/**
+ * @param {toa.norm.Composition[]} compositions
+ * @returns {Map<string, string[]>}
+ */
+function claims(compositions) {
+  const map = new Map()
+
+  for (const composition of compositions)
+    for (const reference of composition.services ?? []) {
+      if (!map.has(reference)) map.set(reference, [])
+
+      map.get(reference).push(composition.name)
+    }
+
+  return map
+}

package/src/deployment/images/bundle.js

@@ -0,0 +1,88 @@
+import { basename, join } from 'node:path'
+import { cp } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+
+import { Image } from './image.js'
+import { Dependencies } from './dependencies.js'
+import { declare, normalized } from './format.js'
+
+/**
+ * Components in one image: their sources laid over their dependencies, which are an image
+ * of their own (see `Dependencies`). What is built here is the sources layer alone.
+ *
+ * @implements {toa.deployment.images.Bundle}
+ * @abstract
+ */
+export class Bundle extends Image {
+  /** @type {Dependencies} */
+  dependencies
+
+  /** @type {string | undefined} */
+  image
+
+  /** @type {toa.norm.Component[]} */
+  components
+
+  /** What the services this workload runs install, which no component of it declares.
+   *  @type {Record<string, string> | undefined} */
+  packages
+
+  constructor(scope, runtime, registry, composition) {
+    super(scope, runtime, registry)
+
+    this.image = composition.image
+    this.components = composition.components
+    this.packages = composition.packages
+    this.dependencies = new Dependencies(scope, runtime, registry, this)
+  }
+
+  tag() {
+    this.dependencies.tag()
+    super.tag()
+  }
+
+  get version() {
+    const hash = createHash('sha256')
+
+    for (const component of this.components) {
+      hash.update(component.locator.id)
+      hash.update(component.version)
+    }
+
+    // laid over a different base is a different image, whatever the sources
+    hash.update(this.dependencies.version)
+
+    return hash.digest('hex').slice(0, 8)
+  }
+
+  get base() {
+    return this.dependencies.reference
+  }
+
+  /**
+   * What to say when the components ask for different base images.
+   *
+   * @abstract
+   * @returns {string}
+   */
+  conflict() {
+    throw new Error('Not implemented')
+  }
+
+  async prepare(root) {
+    const context = await super.prepare(root)
+
+    for (const component of this.components) {
+      const target = join(context, component.locator.label)
+
+      // what was installed in the workspace is not what the image installs
+      await cp(component.path, target, { recursive: true, filter: sources })
+      await declare(component.path, target, component.locator.label)
+      await normalized(component, target)
+    }
+
+    return context
+  }
+}
+
+const sources = (path) => basename(path) !== 'node_modules'

package/src/deployment/images/composition.Dockerfile

@@ -1,19 +1,11 @@
 FROM {{build.image}}
 
-{{build.arguments}}
-
-ENV NODE_ENV=production
-RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
-RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-RUN npm i -g @toa.io/runtime@{{runtime.version}} --omit=dev
-
-WORKDIR /composition
-COPY --chown=node:node . /composition
-
-{{build.run}}
-
-# run 'npm i' in each component
-RUN for entry in *; do if [ -f "$entry/package.json" ]; then (cd $entry && npm i --omit=dev); fi; done
-
-USER node
-CMD toa compose *
+# the one instruction that touches the filesystem, and linked: the base is not pulled to lay
+# the sources over it, and a push carries this layer and a manifest, not the base's layers.
+# Owned by root like the dependencies beside them: a linked layer knows no user by name,
+# and the runtime reads its sources, it does not write them
+COPY --link . /composition
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa compose * --map {{map.file}}

package/src/deployment/images/composition.js

@@ -1,70 +1,23 @@
-'use strict'
+import { join } from 'node:path'
 
-const { join } = require('node:path')
-const fs = require('fs-extra')
-const { createHash } = require('node:crypto')
+import { Bundle } from './bundle.js'
 
-const { Image } = require('./image')
-
-class Composition extends Image {
-  dockerfile = join(__dirname, 'composition.Dockerfile')
+export class Composition extends Bundle {
+  dockerfile = join(import.meta.dirname, 'composition.Dockerfile')
 
   #name
-  #image
-  #components
 
-  constructor (scope, runtime, registry, composition) {
-    super(scope, runtime, registry)
+  constructor(scope, runtime, registry, composition) {
+    super(scope, runtime, registry, composition)
 
     this.#name = composition.name
-    this.#image = composition.image
-    this.#components = composition.components
   }
 
-  get name () {
+  get name() {
     return 'composition-' + this.#name
   }
 
-  get version () {
-    const hash = createHash('sha256')
-
-    for (const component of this.#components) {
-      hash.update(component.locator.id)
-      hash.update(component.version)
-    }
-
-    return hash.digest('hex').slice(0, 8)
-  }
-
-  get base () {
-    if (this.#image !== undefined) {
-      return this.#image
-    }
-
-    let image = null
-
-    for (const component of this.#components) {
-      const value = component.build?.image
-
-      if (image !== null && image !== value) {
-        throw new Error(`Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`)
-      }
-
-      image = value
-    }
-
-    return image ?? undefined
-  }
-
-  async prepare (root) {
-    const context = await super.prepare(root)
-
-    for (const component of this.#components) {
-      await fs.copy(component.path, join(context, component.locator.label))
-    }
-
-    return context
+  conflict() {
+    return `Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`
   }
 }
-
-exports.Composition = Composition

package/src/deployment/images/dependencies.Dockerfile

@@ -0,0 +1,23 @@
+FROM {{build.image}}
+
+{{build.arguments}}
+
+ENV NODE_ENV=production
+RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
+RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
+
+WORKDIR /composition
+COPY --chown=node:node . /composition
+
+{{build.run}}
+
+# the context holds the manifests alone, so this is every dependency and none of the sources.
+# `.packages` is a dotfile, so the glob does not reach it
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  for entry in *; do if grep -qs '"dependencies"' "$entry/package.json"; then (cd $entry && npm i --omit=dev); fi; done
+
+# what the extensions need for what these components declare: installed beside the extension
+# that reads the declaration, in Toa's own install, because that is where Node resolves an
+# extension's imports from — and one copy for the composition rather than one per component
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  if [ -s .packages ]; then npm i --prefix /toa --omit=dev $(cat .packages); fi