@toa.io/operations 1.0.0-alpha.284 → 1.0.0-alpha.286

package/src/deployment/images/publish.js

@@ -0,0 +1,98 @@
+import { mkdtemp, writeFile } from 'node:fs/promises'
+import { readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+import { execa } from 'execa'
+
+import { Service } from './service.js'
+
+/**
+ * Builds and pushes the image an extension ships for its service, so that an application
+ * with `registry.services: published` takes it instead of building one.
+ *
+ * The image is built from the package as npm publishes it, not from the workspace: what
+ * `.npmignore` leaves out is not in the tarball, and an application builds from the tarball.
+ *
+ * @param {string} workspace path to the extension's directory in this repository
+ * @param {string} runtime the runtime version, which is the tag and the base image
+ * @param {string[]} platforms
+ * @returns {Promise<string>} the reference pushed
+ */
+export async function publish(workspace, runtime, platforms) {
+  const directory = resolve(workspace)
+  const manifest = read(join(directory, 'package.json'))
+
+  // read from the workspace, where the extension's own dependencies resolve; what is
+  // installed below carries the same constant but not everything it takes to load it
+  const { image } = await import(pathToFileURL(join(directory, manifest.main)).href)
+
+  if (image === undefined)
+    throw new Error(`'${manifest.name}' publishes no service image: it exports no 'image'`)
+
+  const root = await mkdtemp(join(tmpdir(), 'toa-publish'))
+
+  await writeFile(join(root, 'package.json'), '{"private":true}')
+  await run('npm', [
+    'i',
+    '--prefix',
+    root,
+    '--omit=dev',
+    `${manifest.name}@${manifest.version}`
+  ])
+
+  const path = join(root, 'node_modules', manifest.name)
+
+  const service = new Service(SCOPE, { version: runtime }, {}, path, {
+    ...name(image),
+    version: manifest.version
+  })
+
+  service.reference = `${image}:${runtime}`
+
+  await service.prepare(await mkdtemp(join(tmpdir(), 'toa-images')))
+
+  await run('docker', [
+    'buildx',
+    'build',
+    '--platform',
+    platforms.join(','),
+    '--tag',
+    service.reference,
+    '--provenance=false',
+    '--push',
+    service.context
+  ])
+
+  return service.reference
+}
+
+/** The repository's last segment is what `Service` names an image, so it reads back. */
+function name(image) {
+  const match = /\/extension-([^-/]+)-([^/]+)$/.exec(image)
+
+  if (match === null)
+    throw new Error(`'${image}' is not named 'extension-<group>-<name>'`)
+
+  return { group: match[1], name: match[2] }
+}
+
+const read = (path) => JSON.parse(readFileSync(path, 'utf8'))
+
+const run = async (cmd, args) => {
+  console.log('toa>', cmd, args.join(' '))
+
+  await execa(cmd, args, { stdio: 'inherit' })
+}
+
+// the reference is assigned rather than derived from a scope
+const SCOPE = ''
+
+const PLATFORMS = 'linux/amd64,linux/arm64'
+
+if (import.meta.main) {
+  const [workspace, runtime, platforms = PLATFORMS] = process.argv.slice(2)
+
+  console.log(await publish(workspace, runtime, platforms.split(',')))
+}

package/src/deployment/images/runtime-base.test.js

@@ -16,7 +16,7 @@ class TestImage extends Image {
   #name
   #base
 
-  constructor (runtime, registry, dockerfile, name = 'test', base) {
+  constructor(runtime, registry, dockerfile, name = 'test', base) {
     super('acme', runtime, registry)
 
     this.dockerfile = dockerfile
@@ -24,15 +24,15 @@ class TestImage extends Image {
     this.#base = base
   }
 
-  get name () {
+  get name() {
     return this.#name
   }
 
-  get version () {
+  get version() {
     return 'abcdef12'
   }
 
-  get base () {
+  get base() {
     return this.#base
   }
 }
@@ -76,12 +76,18 @@ describe('runtime base image', () => {
     const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
 
     assert.ok(dockerfile.includes('FROM node:24.14.0-alpine3.22'))
-    assert.ok(!(dockerfile.includes(RUNTIME_IMAGE)))
+    assert.ok(!dockerfile.includes(RUNTIME_IMAGE))
   })
 
   it('should allow composition.image override via base', async () => {
     const runtime = { version: '1.0.0-alpha.232' }
-    const image = new TestImage(runtime, {}, compositionDockerfile, 'mono', 'custom.example/base:1')
+    const image = new TestImage(
+      runtime,
+      {},
+      compositionDockerfile,
+      'mono',
+      'custom.example/base:1'
+    )
 
     const path = await image.prepare(root)
     const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')

package/src/deployment/images/service.Dockerfile

@@ -7,7 +7,7 @@ RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
 WORKDIR /service
 COPY --chown=node:node . /service
 
-RUN --mount=type=cache,target=/root/.npm \
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
   npm i --omit=dev
 
 USER node

package/src/deployment/images/service.js

@@ -37,7 +37,7 @@ export class Service extends Image {
    * @param {string} reference
    * @param {toa.deployment.dependency.Service} service
    */
-  constructor (scope, runtime, registry, reference, service) {
+  constructor(scope, runtime, registry, reference, service) {
     super(scope, runtime, registry)
 
     this.service = service.name
@@ -48,15 +48,15 @@ export class Service extends Image {
     this.#version = service.version
   }
 
-  get name () {
+  get name() {
     return 'extension-' + this.#group + '-' + this.#name
   }
 
-  get version () {
+  get version() {
     return this.#version
   }
 
-  async prepare (root) {
+  async prepare(root) {
     const context = await super.prepare(root)
 
     await fs.copy(this.#path, context)

package/src/deployment/operator.js

@@ -11,12 +11,12 @@ export class Operator {
    * @param deployment {toa.deployment.Deployment}
    * @param registry {toa.deployment.Registry}
    */
-  constructor (deployment, registry) {
+  constructor(deployment, registry) {
     this.#deployment = deployment
     this.#registry = registry
   }
 
-  async export (path) {
+  async export(path) {
     const target = await workspace.create('deployment', path)
 
     await this.#deployment.export(target)
@@ -24,32 +24,32 @@ export class Operator {
     return target
   }
 
-  async prepare (path) {
+  async prepare(path) {
     return await this.#registry.prepare(path)
   }
 
-  async push () {
+  async push() {
     await this.#registry.push()
   }
 
-  async install (options = {}) {
+  async install(options = {}) {
     options = Object.assign({}, OPTIONS, options)
 
     await Promise.all([this.export(), this.push()])
     await this.#deployment.install(options)
   }
 
-  async template (options = {}) {
+  async template(options = {}) {
     await this.export()
 
     return await this.#deployment.template(options)
   }
 
-  variables () {
+  variables() {
     return this.#deployment.variables()
   }
 
-  tags () {
+  tags() {
     return this.#registry.tags()
   }
 }

package/src/deployment/registry.js

@@ -1,12 +1,9 @@
-import { posix } from 'node:path'
 import * as workspace from './workspace.js'
 
 /**
  * @implements {toa.deployment.Registry}
  */
 export class Registry {
-  #scope
-
   #registry
 
   #factory
@@ -15,29 +12,28 @@ export class Registry {
 
   #images = []
 
-  /** @type {string | undefined} */
+  /** @type {Promise<string> | undefined} */
   #builder
 
-  constructor (scope, registry, factory, process) {
-    this.#scope = scope
+  constructor(registry, factory, process) {
     this.#registry = registry
     this.#factory = factory
     this.#process = process
   }
 
-  composition (composition) {
+  composition(composition) {
     return this.#create('composition', composition)
   }
 
-  service (path, service) {
+  service(path, service) {
     return this.#create('service', path, service)
   }
 
-  mono (composition) {
+  mono(composition) {
     return this.#create('mono', composition)
   }
 
-  async prepare (root) {
+  async prepare(root) {
     const path = await workspace.create('images', root)
 
     await Promise.all(this.#images.map((image) => image.prepare(path)))
@@ -45,20 +41,17 @@ export class Registry {
     return path
   }
 
-  async build () {
+  async build() {
     await this.prepare()
-
-    for (const image of this.#images)
-      await this.#build(image)
+    await this.#run(false)
   }
 
-  async push () {
+  async push() {
     await this.prepare()
-
-    for (const image of this.#images) await this.#push(image)
+    await this.#run(true)
   }
 
-  tags () {
+  tags() {
     return this.#images.map((image) => image.reference)
   }
 
@@ -67,7 +60,7 @@ export class Registry {
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
-  #create (type, ...args) {
+  #create(type, ...args) {
     const image = this.#factory[type](...args)
 
     this.#images.push(image)
@@ -75,54 +68,65 @@ export class Registry {
     return image
   }
 
+  /**
+   * Every image is probed at once, and what is missing is built concurrently: a build is
+   * bound by the registry it uploads to, not by the runner, so one at a time leaves both idle.
+   *
+   * @param {boolean} push
+   * @returns {Promise<void>}
+   */
+  async #run(push) {
+    const existing = await Promise.all(
+      this.#images.map((image) => this.exists(image.reference))
+    )
+
+    const missing = this.#images.filter((image, index) => {
+      if (existing[index]) console.log('Image already exists, skipping:', image.reference)
+
+      return !existing[index]
+    })
+
+    await pool(missing, (image) => this.#build(image, push))
+  }
+
   /**
    * @param {toa.deployment.images.Image} image
    * @param {boolean} [push]
    * @returns {Promise<void>}
    */
-  async #build (image, push = false) {
-    if (await this.exists(image.reference)) {
-      console.log('Image already exists, skipping:', image.reference)
-      return
-    }
-
+  async #build(image, push = false) {
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push)
-      args.push('--push')
-    else
-      args.push('--load')
+    if (push) args.push('--push')
+    else args.push('--load')
 
     args.push('--tag', image.reference, image.context)
 
-    const multiarch = this.#registry.platforms !== null
-
     if (this.#registry.build?.arguments !== undefined) {
-      for (const arg of this.#registry.build.arguments) args.push('--build-arg', `${arg}=${process.env[arg]}`)
+      for (const arg of this.#registry.build.arguments)
+        args.push('--build-arg', `${arg}=${process.env[arg]}`)
     }
 
-    if (multiarch) {
-      const platform = this.#registry.platforms.join(',')
-      const builder = await this.#ensureBuilder()
+    const platforms = this.#registry.platforms
+
+    if (platforms !== null) args.push('--platform', platforms.join(','))
 
-      args.push('--platform', platform)
-      args.push('--builder', builder)
+    // the container driver is what builds for a platform the runner is not; for a single
+    // platform it costs a buildkit image to pull and buys nothing
+    if (platforms !== null && platforms.length > 1) {
+      args.push('--builder', await this.#ensureBuilder())
 
-      if (this.#registry.base !== undefined)
-        this.#appendCache(args)
-    } else
-      args.push('--builder', 'default')
+      // the driver that produces attestations is this one, and nothing reads them;
+      // each is a manifest list to export and push per image
+      args.push('--provenance=false')
+    } else args.push('--builder', 'default')
 
     args.push('--progress', 'plain')
 
     await this.#process.execute('docker', args)
   }
 
-  async #push (image) {
-    await this.#build(image, true)
-  }
-
-  async exists (tag) {
+  async exists(tag) {
     const args = ['manifest', 'inspect', tag]
 
     try {
@@ -136,30 +140,54 @@ export class Registry {
     return true
   }
 
-  async #ensureBuilder () {
-    if (this.#builder !== undefined)
-      return this.#builder
+  /** @returns {Promise<string>} */
+  #ensureBuilder() {
+    // the promise is what is memoized, not its value: concurrent builds ask before any answers
+    this.#builder ??= this.#createBuilder()
+
+    return this.#builder
+  }
 
+  /** @returns {Promise<string>} */
+  async #createBuilder() {
     try {
-      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], { silently: true })
+      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], {
+        silently: true
+      })
     } catch {
-      await this.#process.execute('docker', ['buildx', 'create', '--name', BUILDER, '--bootstrap'])
+      await this.#process.execute('docker', [
+        'buildx',
+        'create',
+        '--name',
+        BUILDER,
+        '--bootstrap'
+      ])
     }
 
-    this.#builder = BUILDER
-
     return BUILDER
   }
+}
 
-  /**
-   * @param {string[]} args
-   */
-  #appendCache (args) {
-    const ref = posix.join(this.#registry.base, this.#scope, 'buildcache')
+/**
+ * @template T
+ * @param {T[]} items
+ * @param {(item: T) => Promise<void>} work
+ * @returns {Promise<void>}
+ */
+async function pool(items, work) {
+  const queue = items.slice()
 
-    args.push('--cache-from', `type=registry,ref=${ref}`)
-    args.push('--cache-to', `type=registry,ref=${ref},mode=max,image-manifest=true`)
-  }
+  const workers = Array.from(
+    { length: Math.min(CONCURRENCY, queue.length) },
+    async () => {
+      while (queue.length > 0) await work(queue.shift())
+    }
+  )
+
+  await Promise.all(workers)
 }
 
 const BUILDER = 'toa'
+
+/** How many builds run at once. Past this the uploads contend for the same uplink. */
+const CONCURRENCY = 3