@toa.io/operations 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/deployment/images/service.js

@@ -1,12 +1,15 @@
-'use strict'
+import { createRequire } from 'node:module'
+import { join, dirname } from 'node:path'
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
 
-const { join, dirname } = require('node:path')
+import { Image } from './image.js'
+import { cp, writeFile } from 'node:fs/promises'
 
-const { Image } = require('./image')
-const fs = require('fs-extra')
+// a service is named the way a package is, and its directory is where it lives
+const require = createRequire(import.meta.url)
 
-class Service extends Image {
-  dockerfile = join(__dirname, 'service.Dockerfile')
+export class Service extends Image {
+  dockerfile = join(import.meta.dirname, 'service.Dockerfile')
 
   /**
    * Used by Dockerfile
@@ -35,7 +38,7 @@ class Service extends Image {
    * @param {string} reference
    * @param {toa.deployment.dependency.Service} service
    */
-  constructor (scope, runtime, registry, reference, service) {
+  constructor(scope, runtime, registry, reference, service) {
     super(scope, runtime, registry)
 
     this.service = service.name
@@ -46,29 +49,75 @@ class Service extends Image {
     this.#version = service.version
   }
 
-  get name () {
+  get name() {
     return 'extension-' + this.#group + '-' + this.#name
   }
 
-  get version () {
+  get version() {
     return this.#version
   }
 
-  async prepare (root) {
+  async prepare(root) {
     const context = await super.prepare(root)
 
-    await fs.copy(this.#path, context)
+    await cp(this.#path, context, { recursive: true })
+    await writeFile(join(context, PACKAGES), this.#packages().join('\n'))
 
     return context
   }
+
+  /**
+   * What the components this service runs declare, as `name@version`.
+   *
+   * Each is installed beside its component, where the component's own modules import it, and
+   * beside Toa, where an extension that imports on a component's behalf is — a storage
+   * provider's SDK is imported by `@toa.io/extensions.storages`, not by the component that
+   * declares the storage.
+   *
+   * @returns {string[]}
+   */
+  #packages() {
+    const directory = join(this.#path, 'components')
+
+    if (!existsSync(directory)) return []
+
+    /** @type {Record<string, string>} */
+    const packages = {}
+
+    for (const label of readdirSync(directory)) {
+      const manifest = join(directory, label, 'package.json')
+
+      if (!existsSync(manifest)) continue
+
+      Object.assign(packages, JSON.parse(readFileSync(manifest, 'utf8')).dependencies)
+    }
+
+    return Object.entries(packages)
+      .map(([name, version]) => `${name}@${version}`)
+      .sort()
+  }
 }
 
+/** Where the install reads what the components need, one `name@version` per line. */
+const PACKAGES = '.packages'
+
 /**
+ * Where the extension is installed, which is what its image is built from. A deploy install
+ * carries what an extension declares and not the extension, so a service is built only where
+ * the runtime is installed beside the deployment library; elsewhere its image is published.
+ *
  * @param {string} reference
  * @returns {string}
  */
 const find = (reference) => {
-  return dirname(require.resolve(join(reference, 'package.json')))
+  try {
+    return dirname(require.resolve(join(reference, 'package.json')))
+  } catch (error) {
+    throw new Error(
+      `'${reference}' is not installed, and \`registry.services: build\` builds its image ` +
+        'from where it is: install the runtime beside @toa.io/operations, or take the ' +
+        'image the release publishes with `registry.services: published`',
+      { cause: error }
+    )
+  }
 }
-
-exports.Service = Service

package/src/deployment/index.js

@@ -1,5 +1 @@
-'use strict'
-
-const { Factory } = require('./factory')
-
-exports.Factory = Factory
+export { Factory } from './factory.js'

package/src/deployment/operator.js

@@ -1,24 +1,27 @@
-'use strict'
+import * as workspace from './workspace.js'
 
-const workspace = require('./workspace')
-
-class Operator {
+export class Operator {
   /** @type {toa.deployment.Deployment} */
   #deployment
 
   /** @type {toa.deployment.Registry} */
   #registry
 
+  /** @type {string | undefined} */
+  #environment
+
   /**
-   * @param deployment {toa.deployment.Deployment}
-   * @param registry {toa.deployment.Registry}
+   * @param {toa.deployment.Deployment} deployment
+   * @param {toa.deployment.Registry} registry
+   * @param {string} [environment]
    */
-  constructor (deployment, registry) {
+  constructor(deployment, registry, environment) {
     this.#deployment = deployment
     this.#registry = registry
+    this.#environment = environment
   }
 
-  async export (path) {
+  async export(path) {
     const target = await workspace.create('deployment', path)
 
     await this.#deployment.export(target)
@@ -26,33 +29,36 @@ class Operator {
     return target
   }
 
-  async prepare (path) {
+  async prepare(path) {
     return await this.#registry.prepare(path)
   }
 
-  async push () {
+  async push() {
     await this.#registry.push()
   }
 
-  async install (options = {}) {
+  async install(options = {}) {
     options = Object.assign({}, OPTIONS, options)
 
     await Promise.all([this.export(), this.push()])
+    await this.#registry.alias(this.#environment)
     await this.#deployment.install(options)
   }
 
-  async template (options = {}) {
+  async template(options = {}) {
     await this.export()
 
     return await this.#deployment.template(options)
   }
 
-  variables () {
-    return this.#deployment.variables()
+  variables(options) {
+    return this.#deployment.variables(options)
+  }
+
+  tags() {
+    return this.#registry.tags()
   }
 }
 
 /** @type {toa.deployment.installation.Options} */
 const OPTIONS = { wait: false }
-
-exports.Operator = Operator

package/src/deployment/operator.test.js

@@ -1,10 +1,11 @@
-'use strict'
+import { describe, it, beforeEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
 
-const { Operator } = require('../../src/deployment/operator')
-const { generate } = require('randomstring')
+import { Operator } from '../../src/deployment/operator.js'
+import { generate } from 'randomstring'
 
 it('should be', async () => {
-  expect(Operator).toBeInstanceOf(Function)
+  assert.ok(Operator instanceof Function)
 })
 
 /** @type {toa.deployment.Operator} */
@@ -25,7 +26,7 @@ describe('env', () => {
   it('should be', async () => {
     operator = new Operator(deployment, registry)
 
-    expect(operator.variables).toBeInstanceOf(Function)
+    assert.ok(operator.variables instanceof Function)
   })
 
   it('should return variables', async () => {
@@ -33,12 +34,61 @@ describe('env', () => {
 
     deployment.variables =
       /** @type {typeof toa.deployment.Operator.variables} */
-      jest.fn(() => variables)
+      mock.fn(() => variables)
 
     operator = new Operator(deployment, registry)
 
     const output = operator.variables()
 
-    expect(output).toStrictEqual(variables)
+    assert.deepStrictEqual(output, variables)
+  })
+})
+
+describe('install', () => {
+  /** @type {toa.deployment.Deployment} */
+  let deployment
+
+  /** @type {toa.deployment.Registry} */
+  let registry
+
+  beforeEach(() => {
+    deployment = /** @type {toa.deployment.Deployment} */ {
+      export: mock.fn(async () => {}),
+      install: mock.fn(async () => {})
+    }
+    registry = /** @type {toa.deployment.Registry} */ {
+      push: mock.fn(async () => {}),
+      alias: mock.fn(async () => {})
+    }
+  })
+
+  it('should tag the environment after a push', async () => {
+    const order = []
+
+    registry.push = mock.fn(async () => {
+      order.push('push')
+    })
+    registry.alias = mock.fn(async () => {
+      order.push('alias')
+    })
+    deployment.install = mock.fn(async () => {
+      order.push('install')
+    })
+
+    operator = new Operator(deployment, registry, 'production')
+
+    await operator.install()
+
+    assert.deepStrictEqual(order, ['push', 'alias', 'install'])
+    assert.deepStrictEqual(registry.alias.mock.calls[0].arguments, ['production'])
+  })
+
+  it('should not tag the environment on push', async () => {
+    operator = new Operator(deployment, registry, 'production')
+
+    await operator.push()
+
+    assert.strictEqual(registry.push.mock.callCount(), 1)
+    assert.strictEqual(registry.alias.mock.callCount(), 0)
   })
 })

package/src/deployment/registry.js

@@ -1,62 +1,83 @@
-'use strict'
-
-const workspace = require('./workspace')
-const { newid } = require('@toa.io/generic')
+import * as workspace from './workspace.js'
 
 /**
  * @implements {toa.deployment.Registry}
  */
-class Registry {
+export class Registry {
   #registry
 
   #factory
 
   #process
 
+  /** @type {toa.deployment.images.Image[]} */
   #images = []
 
-  constructor (registry, factory, process) {
+  /** @type {Promise<string> | undefined} */
+  #builder
+
+  constructor(registry, factory, process) {
     this.#registry = registry
     this.#factory = factory
     this.#process = process
   }
 
-  composition (composition) {
+  composition(composition) {
     return this.#create('composition', composition)
   }
 
-  service (path, service) {
+  service(path, service) {
     return this.#create('service', path, service)
   }
 
-  async prepare (root) {
+  mono(composition) {
+    return this.#create('mono', composition)
+  }
+
+  async prepare(root) {
     const path = await workspace.create('images', root)
 
-    await Promise.all(this.#images.map((image) => image.prepare(path)))
+    await Promise.all(this.#all().map((image) => image.prepare(path)))
 
     return path
   }
 
-  async build () {
-    await this.prepare()
+  async build() {
+    await this.#run(false)
+  }
 
-    for (const image of this.#images) {
-      await this.#build(image)
-    }
+  async push() {
+    await this.#run(true)
   }
 
-  async push () {
-    await this.prepare()
+  /**
+   * A second tag on each workload image, the environment, so a retention job can see what
+   * a deploy left running. `deps-*` is not in `#images`.
+   *
+   * @param {string} [environment]
+   * @returns {Promise<void>}
+   */
+  async alias(environment) {
+    if (
+      typeof environment !== 'string' ||
+      environment === '' ||
+      CONTENT_TAG.test(environment)
+    )
+      return
+
+    await Promise.all(this.#images.map((image) => this.#alias(image, environment)))
+  }
 
-    for (const image of this.#images) await this.#push(image)
+  tags() {
+    return this.#images.map((image) => image.reference)
   }
 
   /**
-   * @param {'composition' | 'service'} type
+   * @param {'composition' | 'service' | 'mono'} type
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
-  #create (type, ...args) {
+  #create(type, ...args) {
     const image = this.#factory[type](...args)
 
     this.#images.push(image)
@@ -64,58 +85,201 @@ class Registry {
     return image
   }
 
+  /**
+   * Every image and what it is laid over.
+   *
+   * @returns {toa.deployment.images.Image[]}
+   */
+  #all() {
+    return this.#images.flatMap((image) =>
+      image.dependencies === undefined ? [image] : [image.dependencies, image]
+    )
+  }
+
+  /**
+   * Every image is probed at once, and what is missing is prepared and built concurrently:
+   * a build is bound by the registry it uploads to, not by the runner, so one at a time
+   * leaves both idle. What an image is laid over is built before it.
+   *
+   * @param {boolean} push
+   * @returns {Promise<void>}
+   */
+  async #run(push) {
+    // a push builds on the container driver, whose bootstrap is spent while the probes are out
+    if (push) this.#ensureBuilder().catch(() => undefined)
+
+    const images = this.#all()
+
+    const existing = await Promise.all(
+      images.map((image) => this.exists(image.reference))
+    )
+
+    const missing = images.filter((image, index) => {
+      if (existing[index]) console.log('Image already exists, skipping:', image.reference)
+
+      return !existing[index]
+    })
+
+    if (missing.length === 0) return
+
+    const path = await workspace.create('images')
+
+    await Promise.all(missing.map((image) => image.prepare(path)))
+
+    const bases = new Set(this.#images.map((image) => image.dependencies))
+    const build = (image) => this.#build(image, push)
+
+    await pool(
+      missing.filter((image) => bases.has(image)),
+      build
+    )
+    await pool(
+      missing.filter((image) => !bases.has(image)),
+      build
+    )
+  }
+
   /**
    * @param {toa.deployment.images.Image} image
    * @param {boolean} [push]
    * @returns {Promise<void>}
    */
-  async #build (image, push = false) {
+  async #build(image, push = false) {
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push) {
-      args.push('--push')
-    } else {
-      args.push('--load')
-    }
+    if (push) args.push('--push')
+    else args.push('--load')
 
     args.push('--tag', image.reference, image.context)
 
-    const multiarch = this.#registry.platforms !== null
-
-    if (this.#registry.build?.arguments !== undefined) {
-      for (const arg of this.#registry.build.arguments) args.push('--build-arg', `${arg}=${process.env[arg]}`)
+    if (image.arguments && this.#registry.build?.arguments !== undefined) {
+      for (const arg of this.#registry.build.arguments)
+        args.push('--build-arg', `${arg}=${process.env[arg]}`)
     }
 
-    if (multiarch) {
-      const platform = this.#registry.platforms.join(',')
-      const builder = await this.#createBuilder()
+    const platforms = this.#registry.platforms
 
-      args.push('--platform', platform)
-      args.push('--builder', builder)
+    if (platforms !== null) args.push('--platform', platforms.join(','))
 
-    } else {
-      args.push('--builder', 'default')
-    }
+    // a push takes the container driver: its registry exporter is what lays an image over a
+    // base it never pulled. A load takes the daemon's own, unless it builds for a platform
+    // the runner is not, which the daemon's cannot
+    if (push || (platforms !== null && platforms.length > 1)) {
+      args.push('--builder', await this.#ensureBuilder())
+
+      // the driver that produces attestations is this one, and nothing reads them;
+      // each is a manifest list to export and push per image
+      args.push('--provenance=false')
+    } else args.push('--builder', 'default')
 
     args.push('--progress', 'plain')
 
     await this.#process.execute('docker', args)
   }
 
-  async #push (image) {
-    await this.#build(image, true)
+  /**
+   * @param {toa.deployment.images.Image} image
+   * @param {string} environment
+   * @returns {Promise<void>}
+   */
+  async #alias(image, environment) {
+    const target = retag(image.reference, environment)
+    const args = ['buildx', 'imagetools', 'create', '--tag', target]
+
+    if (LOCAL.test(image.reference)) args.push('--insecure')
+
+    args.push(image.reference)
+
+    await this.#process.execute('docker', args)
+  }
+
+  async exists(tag) {
+    const args = ['manifest', 'inspect']
+
+    // a registry on this machine speaks plain HTTP, which the probe has to be told
+    if (LOCAL.test(tag)) args.push('--insecure')
+
+    args.push(tag)
+
+    try {
+      await this.#process.execute('docker', args, { silently: true })
+    } catch (error) {
+      console.log(error.message)
+
+      return false
+    }
+
+    return true
   }
 
-  async #createBuilder () {
-    const name = `toa-${newid()}`
-    const create = `buildx create --name ${name} --bootstrap --use`.split(' ')
+  /** @returns {Promise<string>} */
+  #ensureBuilder() {
+    // the promise is what is memoized, not its value: concurrent builds ask before any answers
+    this.#builder ??= this.#createBuilder()
 
-    await this.#process.execute('docker', create)
+    return this.#builder
+  }
+
+  /** @returns {Promise<string>} */
+  async #createBuilder() {
+    try {
+      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], {
+        silently: true
+      })
+    } catch {
+      // on the host's network, a registry the host reaches as `localhost` is reached
+      await this.#process.execute('docker', [
+        'buildx',
+        'create',
+        '--name',
+        BUILDER,
+        '--driver',
+        'docker-container',
+        '--driver-opt',
+        'network=host',
+        '--bootstrap'
+      ])
+    }
 
-    return name
+    return BUILDER
   }
 }
 
+/**
+ * @template T
+ * @param {T[]} items
+ * @param {(item: T) => Promise<void>} work
+ * @returns {Promise<void>}
+ */
+async function pool(items, work) {
+  const queue = items.slice()
+
+  const workers = Array.from(
+    { length: Math.min(CONCURRENCY, queue.length) },
+    async () => {
+      while (queue.length > 0) await work(queue.shift())
+    }
+  )
+
+  await Promise.all(workers)
+}
+
+/**
+ * @param {string} reference
+ * @param {string} tag
+ * @returns {string}
+ */
+function retag(reference, tag) {
+  return reference.slice(0, reference.lastIndexOf(':') + 1) + tag
+}
+
 const BUILDER = 'toa'
 
-exports.Registry = Registry
+/** An environment named this way is a content tag, and must not be moved as one. */
+const CONTENT_TAG = /^(deps-)?[0-9a-f]{8}$/
+
+/** A reference into a registry on this machine, by any of the names it goes by. */
+const LOCAL = /^(localhost|127\.\d+\.\d+\.\d+|host\.docker\.internal)(:\d+)?\//
+
+/** How many builds run at once. Past this the uploads contend for the same uplink. */
+const CONCURRENCY = 3