@toa.io/operations 1.0.0-alpha.29 → 1.0.0-alpha.290

package/src/deployment/registry.js

@@ -1,62 +1,65 @@
-'use strict'
-
-const workspace = require('./workspace')
-const { newid } = require('@toa.io/generic')
+import * as workspace from './workspace.js'
 
 /**
  * @implements {toa.deployment.Registry}
  */
-class Registry {
+export class Registry {
   #registry
 
   #factory
 
   #process
 
+  /** @type {toa.deployment.images.Image[]} */
   #images = []
 
-  constructor (registry, factory, process) {
+  /** @type {Promise<string> | undefined} */
+  #builder
+
+  constructor(registry, factory, process) {
     this.#registry = registry
     this.#factory = factory
     this.#process = process
   }
 
-  composition (composition) {
+  composition(composition) {
     return this.#create('composition', composition)
   }
 
-  service (path, service) {
+  service(path, service) {
     return this.#create('service', path, service)
   }
 
-  async prepare (root) {
+  mono(composition) {
+    return this.#create('mono', composition)
+  }
+
+  async prepare(root) {
     const path = await workspace.create('images', root)
 
-    await Promise.all(this.#images.map((image) => image.prepare(path)))
+    await Promise.all(this.#all().map((image) => image.prepare(path)))
 
     return path
   }
 
-  async build () {
-    await this.prepare()
-
-    for (const image of this.#images) {
-      await this.#build(image)
-    }
+  async build() {
+    await this.#run(false)
   }
 
-  async push () {
-    await this.prepare()
+  async push() {
+    await this.#run(true)
+  }
 
-    for (const image of this.#images) await this.#push(image)
+  tags() {
+    return this.#images.map((image) => image.reference)
   }
 
   /**
-   * @param {'composition' | 'service'} type
+   * @param {'composition' | 'service' | 'mono'} type
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
-  #create (type, ...args) {
+  #create(type, ...args) {
     const image = this.#factory[type](...args)
 
     this.#images.push(image)
@@ -64,58 +67,173 @@ class Registry {
     return image
   }
 
+  /**
+   * Every image and what it is laid over.
+   *
+   * @returns {toa.deployment.images.Image[]}
+   */
+  #all() {
+    return this.#images.flatMap((image) =>
+      image.dependencies === undefined ? [image] : [image.dependencies, image]
+    )
+  }
+
+  /**
+   * Every image is probed at once, and what is missing is prepared and built concurrently:
+   * a build is bound by the registry it uploads to, not by the runner, so one at a time
+   * leaves both idle. What an image is laid over is built before it.
+   *
+   * @param {boolean} push
+   * @returns {Promise<void>}
+   */
+  async #run(push) {
+    // a push builds on the container driver, whose bootstrap is spent while the probes are out
+    if (push) this.#ensureBuilder().catch(() => undefined)
+
+    const images = this.#all()
+
+    const existing = await Promise.all(
+      images.map((image) => this.exists(image.reference))
+    )
+
+    const missing = images.filter((image, index) => {
+      if (existing[index]) console.log('Image already exists, skipping:', image.reference)
+
+      return !existing[index]
+    })
+
+    if (missing.length === 0) return
+
+    const path = await workspace.create('images')
+
+    await Promise.all(missing.map((image) => image.prepare(path)))
+
+    const bases = new Set(this.#images.map((image) => image.dependencies))
+    const build = (image) => this.#build(image, push)
+
+    await pool(
+      missing.filter((image) => bases.has(image)),
+      build
+    )
+    await pool(
+      missing.filter((image) => !bases.has(image)),
+      build
+    )
+  }
+
   /**
    * @param {toa.deployment.images.Image} image
    * @param {boolean} [push]
    * @returns {Promise<void>}
    */
-  async #build (image, push = false) {
+  async #build(image, push = false) {
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push) {
-      args.push('--push')
-    } else {
-      args.push('--load')
-    }
+    if (push) args.push('--push')
+    else args.push('--load')
 
     args.push('--tag', image.reference, image.context)
 
-    const multiarch = this.#registry.platforms !== null
-
-    if (this.#registry.build?.arguments !== undefined) {
-      for (const arg of this.#registry.build.arguments) args.push('--build-arg', `${arg}=${process.env[arg]}`)
+    if (image.arguments && this.#registry.build?.arguments !== undefined) {
+      for (const arg of this.#registry.build.arguments)
+        args.push('--build-arg', `${arg}=${process.env[arg]}`)
     }
 
-    if (multiarch) {
-      const platform = this.#registry.platforms.join(',')
-      const builder = await this.#createBuilder()
+    const platforms = this.#registry.platforms
 
-      args.push('--platform', platform)
-      args.push('--builder', builder)
+    if (platforms !== null) args.push('--platform', platforms.join(','))
 
-    } else {
-      args.push('--builder', 'default')
-    }
+    // a push takes the container driver: its registry exporter is what lays an image over a
+    // base it never pulled. A load takes the daemon's own, unless it builds for a platform
+    // the runner is not, which the daemon's cannot
+    if (push || (platforms !== null && platforms.length > 1)) {
+      args.push('--builder', await this.#ensureBuilder())
+
+      // the driver that produces attestations is this one, and nothing reads them;
+      // each is a manifest list to export and push per image
+      args.push('--provenance=false')
+    } else args.push('--builder', 'default')
 
     args.push('--progress', 'plain')
 
     await this.#process.execute('docker', args)
   }
 
-  async #push (image) {
-    await this.#build(image, true)
+  async exists(tag) {
+    const args = ['manifest', 'inspect']
+
+    // a registry on this machine speaks plain HTTP, which the probe has to be told
+    if (LOCAL.test(tag)) args.push('--insecure')
+
+    args.push(tag)
+
+    try {
+      await this.#process.execute('docker', args, { silently: true })
+    } catch (error) {
+      console.log(error.message)
+
+      return false
+    }
+
+    return true
   }
 
-  async #createBuilder () {
-    const name = `toa-${newid()}`
-    const create = `buildx create --name ${name} --bootstrap --use`.split(' ')
+  /** @returns {Promise<string>} */
+  #ensureBuilder() {
+    // the promise is what is memoized, not its value: concurrent builds ask before any answers
+    this.#builder ??= this.#createBuilder()
 
-    await this.#process.execute('docker', create)
+    return this.#builder
+  }
+
+  /** @returns {Promise<string>} */
+  async #createBuilder() {
+    try {
+      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], {
+        silently: true
+      })
+    } catch {
+      // on the host's network, a registry the host reaches as `localhost` is reached
+      await this.#process.execute('docker', [
+        'buildx',
+        'create',
+        '--name',
+        BUILDER,
+        '--driver',
+        'docker-container',
+        '--driver-opt',
+        'network=host',
+        '--bootstrap'
+      ])
+    }
 
-    return name
+    return BUILDER
   }
 }
 
+/**
+ * @template T
+ * @param {T[]} items
+ * @param {(item: T) => Promise<void>} work
+ * @returns {Promise<void>}
+ */
+async function pool(items, work) {
+  const queue = items.slice()
+
+  const workers = Array.from(
+    { length: Math.min(CONCURRENCY, queue.length) },
+    async () => {
+      while (queue.length > 0) await work(queue.shift())
+    }
+  )
+
+  await Promise.all(workers)
+}
+
 const BUILDER = 'toa'
 
-exports.Registry = Registry
+/** A reference into a registry on this machine, by any of the names it goes by. */
+const LOCAL = /^(localhost|127\.\d+\.\d+\.\d+|host\.docker\.internal)(:\d+)?\//
+
+/** How many builds run at once. Past this the uploads contend for the same uplink. */
+const CONCURRENCY = 3

package/src/deployment/registry.test.js

@@ -0,0 +1,407 @@
+import { it, beforeEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+import { isDeepStrictEqual } from 'node:util'
+
+import { Registry } from './registry.js'
+
+/** @type {toa.operations.Process} */
+let process
+
+/** @type {toa.deployment.images.Factory} */
+let factory
+
+/** @type {toa.deployment.images.Image[]} */
+let images
+
+beforeEach(() => {
+  images = []
+  process = /** @type {toa.operations.Process} */ {
+    execute: mock.fn(async (cmd, args) => {
+      if (args[0] === 'manifest') throw new Error('manifest unknown')
+
+      if (args[0] === 'buildx' && args[1] === 'inspect')
+        throw new Error('builder not found')
+
+      return ''
+    })
+  }
+
+  factory = /** @type {toa.deployment.images.Factory} */ {
+    composition: () => createImage('composition-mono', true),
+    service: () => createImage('extension-realtime'),
+    mono: () => createImage('mono', true)
+  }
+})
+
+it('should reuse named builder across images', async () => {
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
+
+  assert.strictEqual(creates.length, 1)
+  assert.deepStrictEqual(creates[0].arguments[1], [
+    'buildx',
+    'create',
+    '--name',
+    'toa',
+    '--driver',
+    'docker-container',
+    '--driver-opt',
+    'network=host',
+    '--bootstrap'
+  ])
+
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) =>
+      args[0] === '--context=default' && args[1] === 'buildx' && args[2] === 'build'
+  )
+
+  assert.strictEqual(builds.length, 3)
+
+  for (const {
+    arguments: [, args]
+  } of builds) {
+    assert.ok(args.includes('--builder'))
+    assert.strictEqual(args[args.indexOf('--builder') + 1], 'toa')
+  }
+})
+
+it('should not create builder when it already exists', async () => {
+  process.execute = mock.fn(async (cmd, args) => {
+    if (args[0] === 'manifest') throw new Error('manifest unknown')
+
+    return ''
+  })
+
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
+
+  assert.strictEqual(creates.length, 0)
+})
+
+it('should not export a build cache', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
+
+  assert.strictEqual(builds.length, 3)
+
+  for (const {
+    arguments: [, args]
+  } of builds) {
+    assert.ok(!args.includes('--cache-from'))
+    assert.ok(!args.includes('--cache-to'))
+  }
+})
+
+it('should use the default builder for a single platform', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
+
+  assert.strictEqual(creates.length, 0)
+
+  const {
+    arguments: [, args]
+  } = process.execute.mock.calls.find(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
+
+  assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
+  assert.ok(args.includes('--platform'))
+  assert.strictEqual(args[args.indexOf('--platform') + 1], 'linux/amd64')
+})
+
+it('should create the builder once when images build concurrently', async () => {
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+  registry.mono(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
+
+  assert.strictEqual(creates.length, 1)
+})
+
+it('should probe every image before building any', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const calls = process.execute.mock.calls.map(({ arguments: [, args] }) => args[0])
+  const build = calls.indexOf('--context=default')
+  const probed = calls.slice(0, build).filter((argument) => argument === 'manifest')
+
+  // every probe is spent before the first build starts, rather than one before each:
+  // the composition, what it is laid over, and the service
+  assert.strictEqual(probed.length, 3)
+})
+
+it('should build what an image is laid over before the image', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const tags = builds().map((args) => args[args.indexOf('--tag') + 1])
+  const [composition] = images
+
+  assert.strictEqual(tags.length, 3)
+  assert.ok(
+    tags.indexOf(composition.dependencies.reference) < tags.indexOf(composition.reference)
+  )
+})
+
+it('should lay the image over what already exists', async () => {
+  process.execute = mock.fn(async (cmd, args) => {
+    if (args[0] === 'manifest' && !args[2].includes(':deps-'))
+      throw new Error('manifest unknown')
+
+    return ''
+  })
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const tags = builds().map((args) => args[args.indexOf('--tag') + 1])
+  const [composition] = images
+
+  assert.deepStrictEqual(tags, [composition.reference])
+  assert.strictEqual(composition.dependencies.prepare.mock.callCount(), 0)
+  assert.strictEqual(composition.prepare.mock.callCount(), 1)
+})
+
+it('should prepare only what is missing', async () => {
+  process.execute = mock.fn(async () => '')
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  for (const image of images) assert.strictEqual(image.prepare.mock.callCount(), 0)
+})
+
+it('should pass build arguments to the images that read them', async () => {
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64'],
+    build: { arguments: ['TOKEN'] }
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const [composition] = images
+
+  for (const args of builds()) {
+    const tag = args[args.indexOf('--tag') + 1]
+    const reads = tag === composition.dependencies.reference
+
+    assert.strictEqual(args.includes('--build-arg'), reads, tag)
+  }
+})
+
+it('should push on the container builder', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.push()
+
+  for (const args of builds()) {
+    assert.ok(args.includes('--push'))
+    assert.strictEqual(args[args.indexOf('--builder') + 1], 'toa')
+    assert.ok(args.includes('--provenance=false'))
+  }
+})
+
+it('should not create the builder for a push when nothing is missing', async () => {
+  process.execute = mock.fn(async (cmd, args) => {
+    if (args[0] === 'buildx' && args[1] === 'inspect')
+      throw new Error('builder not found')
+
+    return ''
+  })
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.push()
+
+  assert.strictEqual(builds().length, 0)
+})
+
+it('should use default builder when platforms is null', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: null })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
+
+  assert.strictEqual(builds.length, 2)
+
+  for (const {
+    arguments: [, args]
+  } of builds) {
+    assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
+    assert.ok(!args.includes('--platform'))
+  }
+})
+
+it('should skip build when image already exists', async () => {
+  process.execute = mock.fn(async () => '')
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
+
+  assert.strictEqual(builds.length, 0)
+  assert.ok(
+    process.execute.mock.calls.some(
+      (call) =>
+        call.arguments.length === 3 &&
+        isDeepStrictEqual(call.arguments[0], 'docker') &&
+        isDeepStrictEqual(call.arguments[1], [
+          'manifest',
+          'inspect',
+          images[0].reference
+        ]) &&
+        isDeepStrictEqual(call.arguments[2], { silently: true })
+    )
+  )
+})
+
+it('should probe a local registry as insecure', async () => {
+  process.execute = mock.fn(async () => '')
+
+  const registry = createRegistry({ base: 'localhost:5000', platforms: null })
+
+  factory.composition = () => {
+    const image = createImage('composition-mono', true)
+
+    image.reference = 'localhost:5000/acme/composition-mono:abcdef12'
+    image.dependencies.reference = 'localhost:5000/acme/composition-mono:deps-12345678'
+
+    return image
+  }
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const probes = process.execute.mock.calls
+    .filter(({ arguments: [, args] }) => args[0] === 'manifest')
+    .map(({ arguments: [, args] }) => args)
+
+  assert.strictEqual(probes.length, 2)
+
+  for (const args of probes)
+    assert.deepStrictEqual(args.slice(0, 3), ['manifest', 'inspect', '--insecure'])
+})
+
+/**
+ * @param {object} [registry]
+ * @returns {Registry}
+ */
+function createRegistry(registry = {}) {
+  return new Registry(registry, factory, process)
+}
+
+/**
+ * @param {string} name
+ * @param {boolean} [bundle] laid over an image of its dependencies
+ * @returns {toa.deployment.images.Image}
+ */
+function createImage(name, bundle = false) {
+  const image = /** @type {toa.deployment.images.Image} */ {
+    reference: `example.com/reg/acme/${name}:abcdef12`,
+    context: `/tmp/${name}`,
+    prepare: mock.fn(async () => `/tmp/${name}`)
+  }
+
+  if (bundle)
+    image.dependencies = /** @type {toa.deployment.images.Image} */ {
+      reference: `example.com/reg/acme/${name}:deps-12345678`,
+      context: `/tmp/dependencies/${name}`,
+      arguments: true,
+      prepare: mock.fn(async () => `/tmp/dependencies/${name}`)
+    }
+
+  images.push(image)
+
+  return image
+}
+
+/** @returns {string[][]} the arguments of every build run */
+function builds() {
+  return process.execute.mock.calls
+    .filter(
+      ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+    )
+    .map(({ arguments: [, args] }) => args)
+}