@toa.io/operations 1.0.0-alpha.287 → 1.0.0-alpha.289

package/src/deployment/registry.js

@@ -10,6 +10,7 @@ export class Registry {
 
   #process
 
+  /** @type {toa.deployment.images.Image[]} */
   #images = []
 
   /** @type {Promise<string> | undefined} */
@@ -36,18 +37,16 @@ export class Registry {
   async prepare(root) {
     const path = await workspace.create('images', root)
 
-    await Promise.all(this.#images.map((image) => image.prepare(path)))
+    await Promise.all(this.#all().map((image) => image.prepare(path)))
 
     return path
   }
 
   async build() {
-    await this.prepare()
     await this.#run(false)
   }
 
   async push() {
-    await this.prepare()
     await this.#run(true)
   }
 
@@ -69,24 +68,57 @@ export class Registry {
   }
 
   /**
-   * Every image is probed at once, and what is missing is built concurrently: a build is
-   * bound by the registry it uploads to, not by the runner, so one at a time leaves both idle.
+   * Every image and what it is laid over.
+   *
+   * @returns {toa.deployment.images.Image[]}
+   */
+  #all() {
+    return this.#images.flatMap((image) =>
+      image.dependencies === undefined ? [image] : [image.dependencies, image]
+    )
+  }
+
+  /**
+   * Every image is probed at once, and what is missing is prepared and built concurrently:
+   * a build is bound by the registry it uploads to, not by the runner, so one at a time
+   * leaves both idle. What an image is laid over is built before it.
    *
    * @param {boolean} push
    * @returns {Promise<void>}
    */
   async #run(push) {
+    // a push builds on the container driver, whose bootstrap is spent while the probes are out
+    if (push) this.#ensureBuilder().catch(() => undefined)
+
+    const images = this.#all()
+
     const existing = await Promise.all(
-      this.#images.map((image) => this.exists(image.reference))
+      images.map((image) => this.exists(image.reference))
     )
 
-    const missing = this.#images.filter((image, index) => {
+    const missing = images.filter((image, index) => {
       if (existing[index]) console.log('Image already exists, skipping:', image.reference)
 
       return !existing[index]
     })
 
-    await pool(missing, (image) => this.#build(image, push))
+    if (missing.length === 0) return
+
+    const path = await workspace.create('images')
+
+    await Promise.all(missing.map((image) => image.prepare(path)))
+
+    const bases = new Set(this.#images.map((image) => image.dependencies))
+    const build = (image) => this.#build(image, push)
+
+    await pool(
+      missing.filter((image) => bases.has(image)),
+      build
+    )
+    await pool(
+      missing.filter((image) => !bases.has(image)),
+      build
+    )
   }
 
   /**
@@ -102,7 +134,7 @@ export class Registry {
 
     args.push('--tag', image.reference, image.context)
 
-    if (this.#registry.build?.arguments !== undefined) {
+    if (image.arguments && this.#registry.build?.arguments !== undefined) {
       for (const arg of this.#registry.build.arguments)
         args.push('--build-arg', `${arg}=${process.env[arg]}`)
     }
@@ -111,9 +143,10 @@ export class Registry {
 
     if (platforms !== null) args.push('--platform', platforms.join(','))
 
-    // the container driver is what builds for a platform the runner is not; for a single
-    // platform it costs a buildkit image to pull and buys nothing
-    if (platforms !== null && platforms.length > 1) {
+    // a push takes the container driver: its registry exporter is what lays an image over a
+    // base it never pulled. A load takes the daemon's own, unless it builds for a platform
+    // the runner is not, which the daemon's cannot
+    if (push || (platforms !== null && platforms.length > 1)) {
       args.push('--builder', await this.#ensureBuilder())
 
       // the driver that produces attestations is this one, and nothing reads them;
@@ -127,7 +160,12 @@ export class Registry {
   }
 
   async exists(tag) {
-    const args = ['manifest', 'inspect', tag]
+    const args = ['manifest', 'inspect']
+
+    // a registry on this machine speaks plain HTTP, which the probe has to be told
+    if (LOCAL.test(tag)) args.push('--insecure')
+
+    args.push(tag)
 
     try {
       await this.#process.execute('docker', args, { silently: true })
@@ -155,11 +193,16 @@ export class Registry {
         silently: true
       })
     } catch {
+      // on the host's network, a registry the host reaches as `localhost` is reached
       await this.#process.execute('docker', [
         'buildx',
         'create',
         '--name',
         BUILDER,
+        '--driver',
+        'docker-container',
+        '--driver-opt',
+        'network=host',
         '--bootstrap'
       ])
     }
@@ -189,5 +232,8 @@ async function pool(items, work) {
 
 const BUILDER = 'toa'
 
+/** A reference into a registry on this machine, by any of the names it goes by. */
+const LOCAL = /^(localhost|127\.\d+\.\d+\.\d+|host\.docker\.internal)(:\d+)?\//
+
 /** How many builds run at once. Past this the uploads contend for the same uplink. */
 const CONCURRENCY = 3

package/src/deployment/registry.test.js

@@ -27,9 +27,9 @@ beforeEach(() => {
   }
 
   factory = /** @type {toa.deployment.images.Factory} */ {
-    composition: () => createImage('composition-mono'),
+    composition: () => createImage('composition-mono', true),
     service: () => createImage('extension-realtime'),
-    mono: () => createImage('mono')
+    mono: () => createImage('mono', true)
   }
 })
 
@@ -54,6 +54,10 @@ it('should reuse named builder across images', async () => {
     'create',
     '--name',
     'toa',
+    '--driver',
+    'docker-container',
+    '--driver-opt',
+    'network=host',
     '--bootstrap'
   ])
 
@@ -62,7 +66,7 @@ it('should reuse named builder across images', async () => {
       args[0] === '--context=default' && args[1] === 'buildx' && args[2] === 'build'
   )
 
-  assert.strictEqual(builds.length, 2)
+  assert.strictEqual(builds.length, 3)
 
   for (const {
     arguments: [, args]
@@ -107,7 +111,7 @@ it('should not export a build cache', async () => {
     ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
   )
 
-  assert.strictEqual(builds.length, 2)
+  assert.strictEqual(builds.length, 3)
 
   for (const {
     arguments: [, args]
@@ -172,8 +176,114 @@ it('should probe every image before building any', async () => {
   const build = calls.indexOf('--context=default')
   const probed = calls.slice(0, build).filter((argument) => argument === 'manifest')
 
-  // both probes are spent before the first build starts, rather than one before each
-  assert.strictEqual(probed.length, 2)
+  // every probe is spent before the first build starts, rather than one before each:
+  // the composition, what it is laid over, and the service
+  assert.strictEqual(probed.length, 3)
+})
+
+it('should build what an image is laid over before the image', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const tags = builds().map((args) => args[args.indexOf('--tag') + 1])
+  const [composition] = images
+
+  assert.strictEqual(tags.length, 3)
+  assert.ok(
+    tags.indexOf(composition.dependencies.reference) < tags.indexOf(composition.reference)
+  )
+})
+
+it('should lay the image over what already exists', async () => {
+  process.execute = mock.fn(async (cmd, args) => {
+    if (args[0] === 'manifest' && !args[2].includes(':deps-'))
+      throw new Error('manifest unknown')
+
+    return ''
+  })
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const tags = builds().map((args) => args[args.indexOf('--tag') + 1])
+  const [composition] = images
+
+  assert.deepStrictEqual(tags, [composition.reference])
+  assert.strictEqual(composition.dependencies.prepare.mock.callCount(), 0)
+  assert.strictEqual(composition.prepare.mock.callCount(), 1)
+})
+
+it('should prepare only what is missing', async () => {
+  process.execute = mock.fn(async () => '')
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  for (const image of images) assert.strictEqual(image.prepare.mock.callCount(), 0)
+})
+
+it('should pass build arguments to the images that read them', async () => {
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64'],
+    build: { arguments: ['TOKEN'] }
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const [composition] = images
+
+  for (const args of builds()) {
+    const tag = args[args.indexOf('--tag') + 1]
+    const reads = tag === composition.dependencies.reference
+
+    assert.strictEqual(args.includes('--build-arg'), reads, tag)
+  }
+})
+
+it('should push on the container builder', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.push()
+
+  for (const args of builds()) {
+    assert.ok(args.includes('--push'))
+    assert.strictEqual(args[args.indexOf('--builder') + 1], 'toa')
+    assert.ok(args.includes('--provenance=false'))
+  }
+})
+
+it('should not create the builder for a push when nothing is missing', async () => {
+  process.execute = mock.fn(async (cmd, args) => {
+    if (args[0] === 'buildx' && args[1] === 'inspect')
+      throw new Error('builder not found')
+
+    return ''
+  })
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.push()
+
+  assert.strictEqual(builds().length, 0)
 })
 
 it('should use default builder when platforms is null', async () => {
@@ -187,14 +297,14 @@ it('should use default builder when platforms is null', async () => {
     ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
   )
 
-  assert.strictEqual(builds.length, 1)
+  assert.strictEqual(builds.length, 2)
 
-  const {
+  for (const {
     arguments: [, args]
-  } = builds[0]
-
-  assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
-  assert.ok(!args.includes('--platform'))
+  } of builds) {
+    assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
+    assert.ok(!args.includes('--platform'))
+  }
 })
 
 it('should skip build when image already exists', async () => {
@@ -226,6 +336,34 @@ it('should skip build when image already exists', async () => {
   )
 })
 
+it('should probe a local registry as insecure', async () => {
+  process.execute = mock.fn(async () => '')
+
+  const registry = createRegistry({ base: 'localhost:5000', platforms: null })
+
+  factory.composition = () => {
+    const image = createImage('composition-mono', true)
+
+    image.reference = 'localhost:5000/acme/composition-mono:abcdef12'
+    image.dependencies.reference = 'localhost:5000/acme/composition-mono:deps-12345678'
+
+    return image
+  }
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const probes = process.execute.mock.calls
+    .filter(({ arguments: [, args] }) => args[0] === 'manifest')
+    .map(({ arguments: [, args] }) => args)
+
+  assert.strictEqual(probes.length, 2)
+
+  for (const args of probes)
+    assert.deepStrictEqual(args.slice(0, 3), ['manifest', 'inspect', '--insecure'])
+})
+
 /**
  * @param {object} [registry]
  * @returns {Registry}
@@ -236,16 +374,34 @@ function createRegistry(registry = {}) {
 
 /**
  * @param {string} name
+ * @param {boolean} [bundle] laid over an image of its dependencies
  * @returns {toa.deployment.images.Image}
  */
-function createImage(name) {
+function createImage(name, bundle = false) {
   const image = /** @type {toa.deployment.images.Image} */ {
     reference: `example.com/reg/acme/${name}:abcdef12`,
     context: `/tmp/${name}`,
     prepare: mock.fn(async () => `/tmp/${name}`)
   }
 
+  if (bundle)
+    image.dependencies = /** @type {toa.deployment.images.Image} */ {
+      reference: `example.com/reg/acme/${name}:deps-12345678`,
+      context: `/tmp/dependencies/${name}`,
+      arguments: true,
+      prepare: mock.fn(async () => `/tmp/dependencies/${name}`)
+    }
+
   images.push(image)
 
   return image
 }
+
+/** @returns {string[][]} the arguments of every build run */
+function builds() {
+  return process.execute.mock.calls
+    .filter(
+      ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+    )
+    .map(({ arguments: [, args] }) => args)
+}

package/src/process.js

@@ -1,4 +1,4 @@
-import { execa } from 'execa'
+import { spawn } from 'node:child_process'
 
 /**
  * @implements {toa.operations.Process}
@@ -7,15 +7,59 @@ export class Process {
   async execute(cmd, args, options = {}) {
     console.log('toa>', cmd, args.join(' '))
 
-    const command = execa(cmd, args)
+    const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+    const stdout = []
+    const stderr = []
+
+    child.stdout.on('data', (chunk) => stdout.push(chunk))
+    child.stderr.on('data', (chunk) => stderr.push(chunk))
 
     if (options.silently !== true) {
-      command.stdout.pipe(process.stdout)
-      command.stderr.pipe(process.stderr)
+      child.stdout.pipe(process.stdout)
+      child.stderr.pipe(process.stderr)
     }
 
-    const result = await command
+    await exit(child, cmd, args, () => Buffer.concat(stderr).toString())
 
-    return result.stdout
+    return Buffer.concat(stdout).toString().replace(/\n$/, '')
   }
 }
+
+/**
+ * Runs a command on this process's own streams, and answers once it has exited.
+ *
+ * @param {string} cmd
+ * @param {string[]} args
+ * @returns {Promise<void>}
+ */
+export async function run(cmd, args) {
+  console.log('toa>', cmd, args.join(' '))
+
+  await exit(spawn(cmd, args, { stdio: 'inherit' }), cmd, args)
+}
+
+/**
+ * @param {import('node:child_process').ChildProcess} child
+ * @param {string} cmd
+ * @param {string[]} args
+ * @param {() => string} stderr
+ * @returns {Promise<void>}
+ */
+const exit = (child, cmd, args, stderr = () => '') =>
+  new Promise((resolve, reject) => {
+    child.once('error', reject)
+
+    child.once('close', (code, signal) => {
+      if (code === 0) return resolve()
+
+      const output = stderr()
+      const reason = signal === null ? `exit code ${code}` : signal
+
+      reject(
+        new Error(
+          `Command failed with ${reason}: ${cmd} ${args.join(' ')}` +
+            (output === '' ? '' : '\n' + output.trimEnd())
+        )
+      )
+    })
+  })

package/src/process.test.js

@@ -5,8 +5,6 @@ import { Process } from './process.js'
 
 describe('execute', () => {
   it('should answer what the command wrote', async () => {
-    // `execa` is a named export, and calling the module namespace instead answers
-    // `execa is not a function` — where only a deployment would have found out
     const output = await new Process().execute(
       'node',
       ['-e', 'process.stdout.write("ok")'],
@@ -16,9 +14,20 @@ describe('execute', () => {
     assert.strictEqual(output, 'ok')
   })
 
-  it('should reject what the command failed at', async () => {
+  it('should reject what the command failed at, with what it wrote', async () => {
     await assert.rejects(
-      new Process().execute('node', ['-e', 'process.exit(1)'], { silently: true })
+      new Process().execute(
+        'node',
+        ['-e', 'process.stderr.write("broken"); process.exit(2)'],
+        { silently: true }
+      ),
+      (error) => /exit code 2: node/.test(error.message) && /broken/.test(error.message)
+    )
+  })
+
+  it('should reject a command that does not exist', async () => {
+    await assert.rejects(
+      new Process().execute('toa-no-such-command', [], { silently: true })
     )
   })
 })

package/types/_deployment/images/image.d.ts

@@ -3,12 +3,27 @@ declare namespace toa.deployment.images {
     readonly reference: string
     readonly context: string
 
+    /** The image this one is laid over, built and pushed as an image of its own. */
+    readonly dependencies?: Image
+
+    /** Whether the build reads `registry.build.arguments`. */
+    readonly arguments?: boolean
+
     name: string
 
     tag(): void
 
     prepare(root: string): Promise<string>
   }
+
+  /** Components in one image, laid over their dependencies. */
+  export interface Bundle extends Image {
+    readonly image?: string
+    readonly components: any[]
+
+    conflict(): string
+  }
 }
 
 export type Image = toa.deployment.images.Image
+export type Bundle = toa.deployment.images.Bundle