@toa.io/operations 1.0.0-alpha.34 → 1.0.0-alpha.340

package/src/deployment/factory.js

@@ -1,77 +1,162 @@
-'use strict'
-
-const { context: load } = require('@toa.io/norm')
-const { Process } = require('../process')
-const { Operator } = require('./operator')
-const { Factory: ImagesFactory } = require('./images')
-const { Deployment } = require('./deployment')
-const { Registry } = require('./registry')
-const { Composition } = require('./composition')
-const { Service } = require('./service')
-
-class Factory {
+import { context as load, definition } from '@toa.io/norm'
+import { Process } from '../process.js'
+import { Operator } from './operator.js'
+import { Factory as ImagesFactory } from './images/index.js'
+import { Deployment } from './deployment.js'
+
+import { Registry } from './registry.js'
+import { Composition } from './composition.js'
+import { Service } from './service.js'
+
+export class Factory {
   #context
+  #mono
   #compositions
   #dependencies
   #registry
   #process
-  #extensionComponents = []
+  #image
+
+  /** Which compositions run a given extension's service, rather than deploying it on its
+   *  own. A service is stateless and already runs several replicas, so several compositions
+   *  running one are replicas of it, behind the one Service that selects them all.
+   *  @type {Map<string, string[]>} */
+  #claims
+
+  /** what this context does not deploy, by package reference
+   *  @type {Set<string>} */
+  #evicted
 
-  constructor (context) {
+  /** the extensions some composition lists, whether or not that composition is deployed
+   *  @type {Set<string>} */
+  #listed
+
+  constructor(context, options = {}) {
     this.#context = context
+    this.#mono = options.mono === true
     this.#process = new Process()
 
-    const imagesFactory = new ImagesFactory(context.name, context.runtime, context.registry)
+    const imagesFactory = new ImagesFactory(
+      context.name,
+      context.runtime,
+      context.registry
+    )
+
+    const compositions = deployed(context.compositions)
 
     this.#registry = new Registry(context.registry, imagesFactory, this.#process)
-    this.#compositions = context.compositions.map((composition) => this.#composition(composition))
+    this.#claims = claims(compositions)
+    this.#listed = listed(context.compositions)
+    this.#evicted = new Set(context.evicted?.services ?? [])
     this.#dependencies = this.#getDependencies()
+    this.#compositions = []
+
+    if (this.#mono)
+      this.#image = this.#registry.mono({
+        components: context.components.filter(managed),
+        // mono runs every service, so it installs what every one of them brings
+        packages: context.packages
+      })
+    else
+      this.#compositions = compositions.map((composition) =>
+        this.#composition(composition)
+      )
   }
 
-  operator () {
-    const deployment = new Deployment(this.#context, this.#compositions, this.#dependencies, this.#process)
-
-    return new Operator(deployment, this.#registry)
+  async operator() {
+    const deployment = new Deployment(
+      this.#context,
+      this.#compositions,
+      // the constructor cannot await, so what it started is settled here
+      await this.#dependencies,
+      this.#process,
+      this.#image
+    )
+
+    return new Operator(deployment, this.#registry, this.#context.environment)
   }
 
-  registry () {
+  registry() {
     return this.#registry
   }
 
-  #composition (composition) {
+  #composition(composition) {
     const image = this.#registry.composition(composition)
 
     return new Composition(composition, image)
   }
 
-  #getDependencies () {
+  async #getDependencies() {
     /** @type {toa.deployment.Dependency[]} */
     const dependencies = []
 
     if (this.#context.dependencies === undefined) return dependencies
 
+    /** the claimed references that turned out to contribute a service */
+    const contributing = new Set()
+
     for (const [reference, instances] of Object.entries(this.#context.dependencies)) {
-      const dependency = this.#getDependency(reference, instances)
+      const dependency = await this.#getDependency(reference, instances)
+
+      if (dependency === undefined) continue
 
-      if (dependency !== undefined) dependencies.push(dependency)
+      if (dependency.services?.length > 0) contributing.add(reference)
+
+      dependencies.push(dependency)
     }
 
+    // this is the first point where every `deployment()` has run, so it is the first
+    // point where a reference that yields nothing can be told from one that yields a service
+    for (const [reference, compositions] of this.#claims)
+      if (!contributing.has(reference))
+        throw new Error(
+          `Composition '${compositions[0]}' lists '${reference}', ` +
+            'which contributes no service.'
+        )
+
     return dependencies
   }
 
-  #getDependency (path, instances) {
-    const module = require(path)
-    const pkg = require(path + '/package.json')
+  async #getDependency(reference, instances) {
+    const { name, module } = await definition(reference)
 
     if (module.deployment === undefined) return
 
-    const annotation = this.#context.annotations?.[pkg.name]
+    // an evicted component is deployed by other means, and so is whatever only evicted
+    // components require; a service a composition lists is deployed for being listed
+    const managed = instances.filter(({ component }) => component.evicted !== true)
 
+    if (managed.length === 0 && instances.length > 0 && !this.#listed.has(reference))
+      return
+
+    const annotation = this.#context.annotations?.[name]
+
+    // every instance as well: what a definition keeps for all of its components, such as the
+    // configuration the values service serves, holds for an evicted one too
     /** @type {toa.deployment.dependency.Declaration} */
-    const dependency = module.deployment(instances, annotation)
+    const dependency = module.deployment(managed, annotation, instances)
+
+    // mono claims every service, including one an extension added since this context was
+    // written, so its claim is the wildcard rather than a list
+    const workload = this.#mono ? [MONO] : this.#claims.get(reference)
+
+    // an evicted service is deployed nowhere, so nothing is prepared for it: no image, and
+    // no Deployment, Service or Ingress downstream
+    const declared = this.#evicted.has(reference) ? undefined : dependency.services
 
     /** @type {toa.deployment.Service[]} */
-    const services = dependency.services?.map((service) => this.#service(path, service))
+    const services = declared?.map((service) =>
+      workload === undefined
+        ? this.#service(reference, service) // its own deployment, its own image
+        : // named the way `Service` would name it, since it skips that wrapper; and by the
+          // extension it is of, which the workload names to its process to be run
+          {
+            ...service,
+            name: `${service.group}-${service.name}`,
+            workload,
+            extension: reference
+          }
+    )
 
     return { ...dependency, services }
   }
@@ -81,17 +166,73 @@ class Factory {
    * @param service {toa.deployment.dependency.Service}
    * @returns {Service}
    */
-  #service (path, service) {
-    const image = this.#registry.service(path, service)
+  #service(path, service) {
+    // an extension that publishes its service's image is taken at its word: nothing is
+    // added to the registry, so nothing is prepared, probed, built or pushed for it
+    const published =
+      service.image !== undefined && this.#context.registry?.services === PUBLISHED
+
+    const image = published
+      ? { reference: `${service.image}:${this.#context.runtime.version}` }
+      : this.#registry.service(path, service)
 
     return new Service(service, image)
   }
 
-  static async create (path, environment) {
-    const context = await load(path, environment)
+  static async create(path, environment, options = {}) {
+    const context = await load(path, environment, options)
 
-    return new Factory(context)
+    return new Factory(context, options)
   }
 }
 
-exports.Factory = Factory
+const MONO = 'mono'
+
+const PUBLISHED = 'published'
+
+/**
+ * The compositions this context deploys: each without its evicted members, and none left with no
+ * members at all — the name it held is free again, and a service it listed falls back to a
+ * deployment of its own.
+ *
+ * @param {toa.norm.Composition[]} [compositions]
+ * @returns {toa.norm.Composition[]}
+ */
+function deployed(compositions = []) {
+  return compositions
+    .map((composition) => ({
+      ...composition,
+      components: composition.components.filter(managed)
+    }))
+    .filter((composition) => composition.components.length > 0)
+}
+
+/** An evicted component is deployed by other means. */
+function managed(component) {
+  return component.evicted !== true
+}
+
+/**
+ * @param {toa.norm.Composition[]} [compositions]
+ * @returns {Set<string>}
+ */
+function listed(compositions = []) {
+  return new Set(compositions.flatMap((composition) => composition.services ?? []))
+}
+
+/**
+ * @param {toa.norm.Composition[]} compositions
+ * @returns {Map<string, string[]>}
+ */
+function claims(compositions) {
+  const map = new Map()
+
+  for (const composition of compositions)
+    for (const reference of composition.services ?? []) {
+      if (!map.has(reference)) map.set(reference, [])
+
+      map.get(reference).push(composition.name)
+    }
+
+  return map
+}

package/src/deployment/images/bundle.js

@@ -0,0 +1,93 @@
+import { basename, join } from 'node:path'
+import { cp } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+
+import { Image } from './image.js'
+import { Dependencies } from './dependencies.js'
+import { declare, normalized } from './format.js'
+
+/**
+ * Components in one image: their sources laid over their dependencies, which are an image
+ * of their own (see `Dependencies`). What is built here is the sources layer alone.
+ *
+ * @implements {toa.deployment.images.Bundle}
+ * @abstract
+ */
+export class Bundle extends Image {
+  /** @type {Dependencies} */
+  dependencies
+
+  /** @type {string | undefined} */
+  image
+
+  /** @type {toa.norm.Component[]} */
+  components
+
+  /** What the services this workload runs install, which no component of it declares.
+   *  @type {Record<string, string> | undefined} */
+  packages
+
+  constructor(scope, runtime, registry, composition) {
+    super(scope, runtime, registry)
+
+    this.image = composition.image
+    this.components = composition.components
+    this.packages = composition.packages
+    this.dependencies = new Dependencies(scope, runtime, registry, this)
+  }
+
+  tag() {
+    this.dependencies.tag()
+    super.tag()
+  }
+
+  get version() {
+    const hash = createHash('sha256')
+
+    for (const component of this.components) {
+      hash.update(component.locator.id)
+      hash.update(component.version)
+    }
+
+    // laid over a different base is a different image, whatever the sources
+    hash.update(this.dependencies.version)
+
+    return hash.digest('hex').slice(0, 8)
+  }
+
+  get base() {
+    return this.dependencies.reference
+  }
+
+  /**
+   * What to say when the components ask for different base images.
+   *
+   * @abstract
+   * @returns {string}
+   */
+  conflict() {
+    throw new Error('Not implemented')
+  }
+
+  async prepare(root) {
+    const context = await super.prepare(root)
+
+    for (const component of this.components) {
+      const target = join(context, component.locator.label)
+
+      // what was installed in the workspace is not what the image installs; a link is copied as
+      // the files it points to, which is the only way what it points to reaches the image at all
+      await cp(component.path, target, {
+        recursive: true,
+        filter: sources,
+        dereference: true
+      })
+      await declare(component.path, target, component.locator.label)
+      await normalized(component, target)
+    }
+
+    return context
+  }
+}
+
+const sources = (path) => basename(path) !== 'node_modules'

package/src/deployment/images/composition.Dockerfile

@@ -1,19 +1,11 @@
 FROM {{build.image}}
 
-{{build.arguments}}
-
-ENV NODE_ENV=production
-RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
-RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-RUN npm i -g @toa.io/runtime@{{runtime.version}} --omit=dev
-
-WORKDIR /composition
-COPY --chown=node:node . /composition
-
-{{build.run}}
-
-# run 'npm i' in each component
-RUN for entry in *; do if [ -f "$entry/package.json" ]; then (cd $entry && npm i --omit=dev); fi; done
-
-USER node
-CMD toa compose *
+# the one instruction that touches the filesystem, and linked: the base is not pulled to lay
+# the sources over it, and a push carries this layer and a manifest, not the base's layers.
+# Owned by root like the dependencies beside them: a linked layer knows no user by name,
+# and the runtime reads its sources, it does not write them
+COPY --link . /composition
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa compose * --map {{map.file}}

package/src/deployment/images/composition.js

@@ -1,70 +1,23 @@
-'use strict'
+import { join } from 'node:path'
 
-const { join } = require('node:path')
-const fs = require('fs-extra')
-const { createHash } = require('node:crypto')
+import { Bundle } from './bundle.js'
 
-const { Image } = require('./image')
-
-class Composition extends Image {
-  dockerfile = join(__dirname, 'composition.Dockerfile')
+export class Composition extends Bundle {
+  dockerfile = join(import.meta.dirname, 'composition.Dockerfile')
 
   #name
-  #image
-  #components
 
-  constructor (scope, runtime, registry, composition) {
-    super(scope, runtime, registry)
+  constructor(scope, runtime, registry, composition) {
+    super(scope, runtime, registry, composition)
 
     this.#name = composition.name
-    this.#image = composition.image
-    this.#components = composition.components
   }
 
-  get name () {
+  get name() {
     return 'composition-' + this.#name
   }
 
-  get version () {
-    const hash = createHash('sha256')
-
-    for (const component of this.#components) {
-      hash.update(component.locator.id)
-      hash.update(component.version)
-    }
-
-    return hash.digest('hex').slice(0, 8)
-  }
-
-  get base () {
-    if (this.#image !== undefined) {
-      return this.#image
-    }
-
-    let image = null
-
-    for (const component of this.#components) {
-      const value = component.build?.image
-
-      if (image !== null && image !== value) {
-        throw new Error(`Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`)
-      }
-
-      image = value
-    }
-
-    return image ?? undefined
-  }
-
-  async prepare (root) {
-    const context = await super.prepare(root)
-
-    for (const component of this.#components) {
-      await fs.copy(component.path, join(context, component.locator.label))
-    }
-
-    return context
+  conflict() {
+    return `Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`
   }
 }
-
-exports.Composition = Composition

package/src/deployment/images/dependencies.Dockerfile

@@ -0,0 +1,23 @@
+FROM {{build.image}}
+
+{{build.arguments}}
+
+ENV NODE_ENV=production
+RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
+RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
+
+WORKDIR /composition
+COPY --chown=node:node . /composition
+
+{{build.run}}
+
+# the context holds the manifests alone, so this is every dependency and none of the sources.
+# `.packages` is a dotfile, so the glob does not reach it
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  for entry in *; do if grep -qs '"dependencies"' "$entry/package.json"; then (cd $entry && npm i --omit=dev); fi; done
+
+# what the extensions need for what these components declare: installed beside the extension
+# that reads the declaration, in Toa's own install, because that is where Node resolves an
+# extension's imports from — and one copy for the composition rather than one per component
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  if [ -s .packages ]; then npm i --prefix /toa --omit=dev $(cat .packages); fi

package/src/deployment/images/dependencies.js

@@ -0,0 +1,171 @@
+import { basename, join } from 'node:path'
+import { existsSync, readFileSync } from 'node:fs'
+import { copyFile, mkdir, writeFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+
+import { Image } from './image.js'
+
+/**
+ * What a bundle's components depend on, installed on the base: the layers that weigh the
+ * most and change the least, so they are an image of their own. Its tag digests everything
+ * the install reads, and nothing else: a change to the sources leaves it where it is, and a
+ * bundle is laid over it without building or moving it again.
+ *
+ * @implements {toa.deployment.images.Image}
+ */
+export class Dependencies extends Image {
+  dockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
+  arguments = true
+
+  /** @type {toa.deployment.images.Bundle} */
+  #owner
+
+  #runtime
+  #registry
+
+  /** @type {string | undefined} */
+  #version
+
+  /**
+   * @param {string} scope
+   * @param {toa.norm.context.Runtime} runtime
+   * @param {toa.norm.context.Registry} registry
+   * @param {toa.deployment.images.Bundle} owner the bundle laid over this image
+   */
+  constructor(scope, runtime, registry, owner) {
+    super(scope, runtime, registry)
+
+    this.#owner = owner
+    this.#runtime = runtime
+    this.#registry = registry
+  }
+
+  /** Same repository as the bundle: a tag apart, not a name apart. */
+  get name() {
+    return this.#owner.name
+  }
+
+  get version() {
+    if (this.#version !== undefined) return this.#version
+
+    const hash = createHash('sha256')
+    const build = this.#registry.build ?? {}
+
+    const inputs = [
+      this.#runtime.version,
+      this.#runtime.registry,
+      this.#runtime.proxy,
+      this.base,
+      build.image,
+      build.run,
+      this.run,
+      ...(build.arguments ?? [])
+    ]
+
+    for (const input of inputs) hash.update(input ?? '').update('\0')
+
+    for (const [label, files] of this.#manifests())
+      for (const file of files)
+        hash.update(label).update(basename(file)).update(readFileSync(file))
+
+    hash.update(this.#packages().join('\n'))
+
+    this.#version = hash.digest('hex').slice(0, 8)
+
+    return this.#version
+  }
+
+  digest() {
+    return PREFIX + this.version
+  }
+
+  get base() {
+    if (this.#owner.image !== undefined) return this.#owner.image
+
+    const images = new Set(
+      this.#owner.components.map((component) => component.build?.image)
+    )
+
+    if (images.size > 1) throw new Error(this.#owner.conflict())
+
+    return images.values().next().value
+  }
+
+  get run() {
+    const commands = []
+
+    for (const component of this.#owner.components) {
+      const run = component.build?.run
+
+      if (run !== undefined) commands.push(run)
+    }
+
+    return commands.join('\n')
+  }
+
+  async prepare(root) {
+    const context = await super.prepare(join(root, DIRECTORY))
+
+    for (const [label, files] of this.#manifests()) {
+      const target = join(context, label)
+
+      await mkdir(target, { recursive: true })
+
+      for (const file of files) await copyFile(file, join(target, basename(file)))
+    }
+
+    const packages = this.#packages()
+
+    // written whether or not there are any, so the file is one thing the Dockerfile reads
+    await writeFile(join(context, PACKAGES), packages.join('\n'))
+
+    return context
+  }
+
+  /**
+   * What the extensions install for what this workload runs, as `name@version`: what its
+   * components declare, and what the services it hosts bring. A component states its own
+   * dependencies in its manifest; this is what the packages that read its declaration need,
+   * which its manifest has no way to say.
+   *
+   * @returns {string[]}
+   */
+  #packages() {
+    /** @type {Record<string, string>} */
+    const packages = { ...this.#owner.packages }
+
+    for (const component of this.#owner.components)
+      Object.assign(packages, component.packages)
+
+    return Object.entries(packages)
+      .map(([name, version]) => `${name}@${version}`)
+      .sort()
+  }
+
+  /**
+   * The files the install reads, by component label, in a fixed order.
+   *
+   * @returns {Array<[string, string[]]>}
+   */
+  #manifests() {
+    const manifests = []
+
+    for (const component of this.#owner.components) {
+      const files = MANIFESTS.map((name) => join(component.path, name)).filter(existsSync)
+
+      if (files.length > 0) manifests.push([component.locator.label, files])
+    }
+
+    return manifests.sort(([a], [b]) => a.localeCompare(b))
+  }
+}
+
+const PREFIX = 'deps-'
+
+/** Where the contexts land, apart from the bundles', whose names they share. */
+const DIRECTORY = 'dependencies'
+
+const MANIFESTS = ['package.json', 'package-lock.json']
+
+/** Where the install reads what the extensions need, one `name@version` per line. */
+const PACKAGES = '.packages'