@toa.io/operations 1.0.0-alpha.284 → 1.0.0-alpha.287

package/src/deployment/registry.js

@@ -1,12 +1,9 @@
-import { posix } from 'node:path'
 import * as workspace from './workspace.js'
 
 /**
  * @implements {toa.deployment.Registry}
  */
 export class Registry {
-  #scope
-
   #registry
 
   #factory
@@ -15,29 +12,28 @@ export class Registry {
 
   #images = []
 
-  /** @type {string | undefined} */
+  /** @type {Promise<string> | undefined} */
   #builder
 
-  constructor (scope, registry, factory, process) {
-    this.#scope = scope
+  constructor(registry, factory, process) {
     this.#registry = registry
     this.#factory = factory
     this.#process = process
   }
 
-  composition (composition) {
+  composition(composition) {
     return this.#create('composition', composition)
   }
 
-  service (path, service) {
+  service(path, service) {
     return this.#create('service', path, service)
   }
 
-  mono (composition) {
+  mono(composition) {
     return this.#create('mono', composition)
   }
 
-  async prepare (root) {
+  async prepare(root) {
     const path = await workspace.create('images', root)
 
     await Promise.all(this.#images.map((image) => image.prepare(path)))
@@ -45,20 +41,17 @@ export class Registry {
     return path
   }
 
-  async build () {
+  async build() {
     await this.prepare()
-
-    for (const image of this.#images)
-      await this.#build(image)
+    await this.#run(false)
   }
 
-  async push () {
+  async push() {
     await this.prepare()
-
-    for (const image of this.#images) await this.#push(image)
+    await this.#run(true)
   }
 
-  tags () {
+  tags() {
     return this.#images.map((image) => image.reference)
   }
 
@@ -67,7 +60,7 @@ export class Registry {
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
-  #create (type, ...args) {
+  #create(type, ...args) {
     const image = this.#factory[type](...args)
 
     this.#images.push(image)
@@ -75,54 +68,65 @@ export class Registry {
     return image
   }
 
+  /**
+   * Every image is probed at once, and what is missing is built concurrently: a build is
+   * bound by the registry it uploads to, not by the runner, so one at a time leaves both idle.
+   *
+   * @param {boolean} push
+   * @returns {Promise<void>}
+   */
+  async #run(push) {
+    const existing = await Promise.all(
+      this.#images.map((image) => this.exists(image.reference))
+    )
+
+    const missing = this.#images.filter((image, index) => {
+      if (existing[index]) console.log('Image already exists, skipping:', image.reference)
+
+      return !existing[index]
+    })
+
+    await pool(missing, (image) => this.#build(image, push))
+  }
+
   /**
    * @param {toa.deployment.images.Image} image
    * @param {boolean} [push]
    * @returns {Promise<void>}
    */
-  async #build (image, push = false) {
-    if (await this.exists(image.reference)) {
-      console.log('Image already exists, skipping:', image.reference)
-      return
-    }
-
+  async #build(image, push = false) {
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push)
-      args.push('--push')
-    else
-      args.push('--load')
+    if (push) args.push('--push')
+    else args.push('--load')
 
     args.push('--tag', image.reference, image.context)
 
-    const multiarch = this.#registry.platforms !== null
-
     if (this.#registry.build?.arguments !== undefined) {
-      for (const arg of this.#registry.build.arguments) args.push('--build-arg', `${arg}=${process.env[arg]}`)
+      for (const arg of this.#registry.build.arguments)
+        args.push('--build-arg', `${arg}=${process.env[arg]}`)
     }
 
-    if (multiarch) {
-      const platform = this.#registry.platforms.join(',')
-      const builder = await this.#ensureBuilder()
+    const platforms = this.#registry.platforms
+
+    if (platforms !== null) args.push('--platform', platforms.join(','))
 
-      args.push('--platform', platform)
-      args.push('--builder', builder)
+    // the container driver is what builds for a platform the runner is not; for a single
+    // platform it costs a buildkit image to pull and buys nothing
+    if (platforms !== null && platforms.length > 1) {
+      args.push('--builder', await this.#ensureBuilder())
 
-      if (this.#registry.base !== undefined)
-        this.#appendCache(args)
-    } else
-      args.push('--builder', 'default')
+      // the driver that produces attestations is this one, and nothing reads them;
+      // each is a manifest list to export and push per image
+      args.push('--provenance=false')
+    } else args.push('--builder', 'default')
 
     args.push('--progress', 'plain')
 
     await this.#process.execute('docker', args)
   }
 
-  async #push (image) {
-    await this.#build(image, true)
-  }
-
-  async exists (tag) {
+  async exists(tag) {
     const args = ['manifest', 'inspect', tag]
 
     try {
@@ -136,30 +140,54 @@ export class Registry {
     return true
   }
 
-  async #ensureBuilder () {
-    if (this.#builder !== undefined)
-      return this.#builder
+  /** @returns {Promise<string>} */
+  #ensureBuilder() {
+    // the promise is what is memoized, not its value: concurrent builds ask before any answers
+    this.#builder ??= this.#createBuilder()
+
+    return this.#builder
+  }
 
+  /** @returns {Promise<string>} */
+  async #createBuilder() {
     try {
-      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], { silently: true })
+      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], {
+        silently: true
+      })
     } catch {
-      await this.#process.execute('docker', ['buildx', 'create', '--name', BUILDER, '--bootstrap'])
+      await this.#process.execute('docker', [
+        'buildx',
+        'create',
+        '--name',
+        BUILDER,
+        '--bootstrap'
+      ])
     }
 
-    this.#builder = BUILDER
-
     return BUILDER
   }
+}
 
-  /**
-   * @param {string[]} args
-   */
-  #appendCache (args) {
-    const ref = posix.join(this.#registry.base, this.#scope, 'buildcache')
+/**
+ * @template T
+ * @param {T[]} items
+ * @param {(item: T) => Promise<void>} work
+ * @returns {Promise<void>}
+ */
+async function pool(items, work) {
+  const queue = items.slice()
 
-    args.push('--cache-from', `type=registry,ref=${ref}`)
-    args.push('--cache-to', `type=registry,ref=${ref},mode=max,image-manifest=true`)
-  }
+  const workers = Array.from(
+    { length: Math.min(CONCURRENCY, queue.length) },
+    async () => {
+      while (queue.length > 0) await work(queue.shift())
+    }
+  )
+
+  await Promise.all(workers)
 }
 
 const BUILDER = 'toa'
+
+/** How many builds run at once. Past this the uploads contend for the same uplink. */
+const CONCURRENCY = 3

package/src/deployment/registry.test.js

@@ -17,8 +17,7 @@ beforeEach(() => {
   images = []
   process = /** @type {toa.operations.Process} */ {
     execute: mock.fn(async (cmd, args) => {
-      if (args[0] === 'manifest')
-        throw new Error('manifest unknown')
+      if (args[0] === 'manifest') throw new Error('manifest unknown')
 
       if (args[0] === 'buildx' && args[1] === 'inspect')
         throw new Error('builder not found')
@@ -29,30 +28,45 @@ beforeEach(() => {
 
   factory = /** @type {toa.deployment.images.Factory} */ {
     composition: () => createImage('composition-mono'),
-    service: () => createImage('extension-realtime')
+    service: () => createImage('extension-realtime'),
+    mono: () => createImage('mono')
   }
 })
 
 it('should reuse named builder across images', async () => {
-  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
 
   registry.composition(/** @type {any} */ ({}))
   registry.service('.', /** @type {any} */ ({}))
 
   await registry.build()
 
-  const creates = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === 'buildx' && args[1] === 'create')
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
 
   assert.strictEqual(creates.length, 1)
-  assert.deepStrictEqual(creates[0].arguments[1], ['buildx', 'create', '--name', 'toa', '--bootstrap'])
-
-  const builds = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === '--context=default' && args[1] === 'buildx' && args[2] === 'build')
+  assert.deepStrictEqual(creates[0].arguments[1], [
+    'buildx',
+    'create',
+    '--name',
+    'toa',
+    '--bootstrap'
+  ])
+
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) =>
+      args[0] === '--context=default' && args[1] === 'buildx' && args[2] === 'build'
+  )
 
   assert.strictEqual(builds.length, 2)
 
-  for (const { arguments: [, args] } of builds) {
+  for (const {
+    arguments: [, args]
+  } of builds) {
     assert.ok(args.includes('--builder'))
     assert.strictEqual(args[args.indexOf('--builder') + 1], 'toa')
   }
@@ -60,25 +74,28 @@ it('should reuse named builder across images', async () => {
 
 it('should not create builder when it already exists', async () => {
   process.execute = mock.fn(async (cmd, args) => {
-    if (args[0] === 'manifest')
-      throw new Error('manifest unknown')
+    if (args[0] === 'manifest') throw new Error('manifest unknown')
 
     return ''
   })
 
-  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
 
   registry.composition(/** @type {any} */ ({}))
 
   await registry.build()
 
-  const creates = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === 'buildx' && args[1] === 'create')
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
 
   assert.strictEqual(creates.length, 0)
 })
 
-it('should add shared registry cache flags when base is set', async () => {
+it('should not export a build cache', async () => {
   const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
 
   registry.composition(/** @type {any} */ ({}))
@@ -86,35 +103,77 @@ it('should add shared registry cache flags when base is set', async () => {
 
   await registry.build()
 
-  const builds = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === '--context=default' && args[2] === 'build')
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
 
   assert.strictEqual(builds.length, 2)
 
-  for (const { arguments: [, args] } of builds) {
-    assert.ok(args.includes('--cache-from'))
-    assert.ok(args.includes('type=registry,ref=example.com/reg/acme/buildcache'))
-    assert.ok(args.includes('--cache-to'))
-    assert.ok(args.includes('type=registry,ref=example.com/reg/acme/buildcache,mode=max,image-manifest=true'))
+  for (const {
+    arguments: [, args]
+  } of builds) {
+    assert.ok(!args.includes('--cache-from'))
+    assert.ok(!args.includes('--cache-to'))
   }
 })
 
-it('should omit cache flags when base is not set', async () => {
-  const registry = createRegistry({ platforms: ['linux/amd64'] })
+it('should use the default builder for a single platform', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
 
   registry.composition(/** @type {any} */ ({}))
 
   await registry.build()
 
-  const builds = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === '--context=default' && args[2] === 'build')
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
 
-  assert.strictEqual(builds.length, 1)
+  assert.strictEqual(creates.length, 0)
+
+  const {
+    arguments: [, args]
+  } = process.execute.mock.calls.find(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
+
+  assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
+  assert.ok(args.includes('--platform'))
+  assert.strictEqual(args[args.indexOf('--platform') + 1], 'linux/amd64')
+})
+
+it('should create the builder once when images build concurrently', async () => {
+  const registry = createRegistry({
+    base: 'example.com/reg',
+    platforms: ['linux/amd64', 'linux/arm64']
+  })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+  registry.mono(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === 'buildx' && args[1] === 'create'
+  )
+
+  assert.strictEqual(creates.length, 1)
+})
+
+it('should probe every image before building any', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
 
-  const { arguments: [, args] } = builds[0]
+  const calls = process.execute.mock.calls.map(({ arguments: [, args] }) => args[0])
+  const build = calls.indexOf('--context=default')
+  const probed = calls.slice(0, build).filter((argument) => argument === 'manifest')
 
-  assert.ok(!(args.includes('--cache-from')))
-  assert.ok(!(args.includes('--cache-to')))
+  // both probes are spent before the first build starts, rather than one before each
+  assert.strictEqual(probed.length, 2)
 })
 
 it('should use default builder when platforms is null', async () => {
@@ -124,16 +183,18 @@ it('should use default builder when platforms is null', async () => {
 
   await registry.build()
 
-  const builds = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === '--context=default' && args[2] === 'build')
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
 
   assert.strictEqual(builds.length, 1)
 
-  const { arguments: [, args] } = builds[0]
+  const {
+    arguments: [, args]
+  } = builds[0]
 
   assert.strictEqual(args[args.indexOf('--builder') + 1], 'default')
-  assert.ok(!(args.includes('--cache-from')))
-  assert.ok(!(args.includes('--platform')))
+  assert.ok(!args.includes('--platform'))
 })
 
 it('should skip build when image already exists', async () => {
@@ -145,26 +206,39 @@ it('should skip build when image already exists', async () => {
 
   await registry.build()
 
-  const builds = process.execute.mock.calls.filter(({ arguments: [, args] }) =>
-    args[0] === '--context=default' && args[2] === 'build')
+  const builds = process.execute.mock.calls.filter(
+    ({ arguments: [, args] }) => args[0] === '--context=default' && args[2] === 'build'
+  )
 
   assert.strictEqual(builds.length, 0)
-  assert.ok(process.execute.mock.calls.some((call) => call.arguments.length === 3 && isDeepStrictEqual(call.arguments[0], 'docker') && isDeepStrictEqual(call.arguments[1], ['manifest', 'inspect', images[0].reference]) && isDeepStrictEqual(call.arguments[2], { silently: true })))
+  assert.ok(
+    process.execute.mock.calls.some(
+      (call) =>
+        call.arguments.length === 3 &&
+        isDeepStrictEqual(call.arguments[0], 'docker') &&
+        isDeepStrictEqual(call.arguments[1], [
+          'manifest',
+          'inspect',
+          images[0].reference
+        ]) &&
+        isDeepStrictEqual(call.arguments[2], { silently: true })
+    )
+  )
 })
 
 /**
  * @param {object} [registry]
  * @returns {Registry}
  */
-function createRegistry (registry = {}) {
-  return new Registry('acme', registry, factory, process)
+function createRegistry(registry = {}) {
+  return new Registry(registry, factory, process)
 }
 
 /**
  * @param {string} name
  * @returns {toa.deployment.images.Image}
  */
-function createImage (name) {
+function createImage(name) {
   const image = /** @type {toa.deployment.images.Image} */ {
     reference: `example.com/reg/acme/${name}:abcdef12`,
     context: `/tmp/${name}`,

package/src/deployment/service.js

@@ -1,5 +1,7 @@
+// a service is what a deployment carries, and it is constructed
+// oxlint-disable-next-line typescript/no-extraneous-class
 export class Service {
-  constructor (service, image) {
+  constructor(service, image) {
     Object.assign(this, service)
 
     this.name = service.group + '-' + service.name

package/src/deployment/workspace.js

@@ -7,7 +7,7 @@ import { tmpdir } from 'node:os'
  * @param {string} [path]
  * @return {Promise<string>}
  */
-export async function create (type, path) {
+export async function create(type, path) {
   if (path === undefined) return await mkdtemp(join(tmpdir(), 'toa-' + type))
 
   path = resolve(path)

package/src/process.js

@@ -4,7 +4,7 @@ import { execa } from 'execa'
  * @implements {toa.operations.Process}
  */
 export class Process {
-  async execute (cmd, args, options = {}) {
+  async execute(cmd, args, options = {}) {
     console.log('toa>', cmd, args.join(' '))
 
     const command = execa(cmd, args)

package/src/process.test.js

@@ -7,14 +7,18 @@ describe('execute', () => {
   it('should answer what the command wrote', async () => {
     // `execa` is a named export, and calling the module namespace instead answers
     // `execa is not a function` — where only a deployment would have found out
-    const output = await new Process().execute('node', ['-e', 'process.stdout.write("ok")'],
-      { silently: true })
+    const output = await new Process().execute(
+      'node',
+      ['-e', 'process.stdout.write("ok")'],
+      { silently: true }
+    )
 
     assert.strictEqual(output, 'ok')
   })
 
   it('should reject what the command failed at', async () => {
-    await assert.rejects(new Process().execute('node', ['-e', 'process.exit(1)'],
-      { silently: true }))
+    await assert.rejects(
+      new Process().execute('node', ['-e', 'process.exit(1)'], { silently: true })
+    )
   })
 })

package/types/_deployment/composition.d.ts

@@ -1,11 +1,9 @@
 import type * as _deployment from './deployment.js'
 
 declare namespace toa.deployment {
-
   interface Composition extends _deployment.Deployable {
     components: Array<string>
   }
-
 }
 
 export type Composition = toa.deployment.Composition

package/types/_deployment/dependency.d.ts

@@ -2,10 +2,11 @@ import * as _service from './service.js'
 import { dependencies } from '@toa.io/norm/types/context'
 
 declare namespace toa.deployment {
-
   namespace dependency {
-
-    type Constructor = (instances: dependencies.Instance[], annotation: any) => Declaration
+    type Constructor = (
+      instances: dependencies.Instance[],
+      annotation: any
+    ) => Declaration
 
     type Reference = {
       name: string
@@ -19,6 +20,11 @@ declare namespace toa.deployment {
       group: string
       name: string
       version: string
+
+      /** The repository the extension publishes this service's image to, without a tag.
+       *  Stating it lets an application take the published image instead of building one. */
+      image?: string
+
       port: number
       ingress?: {
         host: string
@@ -36,7 +42,7 @@ declare namespace toa.deployment {
       name: string
       value?: string | number
       secret?: {
-        name: string,
+        name: string
         key: string
       }
     }
@@ -52,7 +58,6 @@ declare namespace toa.deployment {
       variables?: Variables
       events?: string[]
     }
-
   }
 
   type Dependency = {
@@ -62,7 +67,6 @@ declare namespace toa.deployment {
     variables?: dependency.Variables
     events?: string[]
   }
-
 }
 
 export type Declaration = toa.deployment.dependency.Declaration

package/types/_deployment/deployment.d.ts

@@ -3,7 +3,6 @@ import type * as _service from './service.js'
 import type * as _dependency from './dependency.js'
 
 declare namespace toa.deployment {
-
   interface Declaration {
     apiVersion: string
     type: string
@@ -25,13 +24,11 @@ declare namespace toa.deployment {
   }
 
   namespace installation {
-
     interface Options {
       wait?: boolean
       target?: string
       namespace?: string
     }
-
   }
 
   namespace template {
@@ -54,7 +51,6 @@ declare namespace toa.deployment {
 
     variables(): _dependency.Variables
   }
-
 }
 
 export namespace installation {

package/types/_deployment/factory.d.ts

@@ -2,13 +2,11 @@ import type * as _operator from './operator.js'
 import * as _registry from './registry.js'
 
 declare namespace toa.deployment {
-
   interface Factory {
     operator(): _operator.Operator
 
     registry(): _registry.Registry
   }
-
 }
 
 export type Factory = toa.deployment.Factory