@toa.io/operations 1.0.0-alpha.27 → 1.0.0-alpha.272

package/src/deployment/images/composition.js

@@ -5,6 +5,7 @@ const fs = require('fs-extra')
 const { createHash } = require('node:crypto')
 
 const { Image } = require('./image')
+const { undef } = require('@toa.io/concise/source/expressions/undefined')
 
 class Composition extends Image {
   dockerfile = join(__dirname, 'composition.Dockerfile')
@@ -37,23 +38,27 @@ class Composition extends Image {
   }
 
   get base () {
-    if (this.#image !== undefined) {
-      return this.#image
-    }
+    if (this.#image !== undefined) return this.#image
 
-    let image = null
+    const images = new Set(this.#components.map((component) => component.build?.image))
 
-    for (const component of this.#components) {
-      const value = component.build?.image
+    if (images.size > 1)
+      throw new Error(`Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`)
+
+    return images.values().next().value
+  }
+
+  get run () {
+    const commands = []
 
-      if (image !== null && image !== value) {
-        throw new Error(`Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`)
-      }
+    for (const component of this.#components) {
+      const run = component.build?.run
 
-      image = value
+      if (run !== undefined)
+        commands.push(run)
     }
 
-    return image ?? undefined
+    return commands.join('\n')
   }
 
   async prepare (root) {

package/src/deployment/images/factory.js

@@ -2,10 +2,8 @@
 
 const { Composition } = require('./composition')
 const { Service } = require('./service')
+const { Mono } = require('./mono')
 
-/**
- * @implements {toa.deployment.images.Factory}
- */
 class Factory {
   /** @type {string} */
   #scope
@@ -48,6 +46,17 @@ class Factory {
 
     return instance
   }
+
+  /**
+   * @returns {Mono}
+   */
+  mono (composition) {
+    const instance = new Mono(this.#scope, this.#runtime, this.#registry, composition)
+
+    instance.tag()
+
+    return instance
+  }
 }
 
 exports.Factory = Factory

package/src/deployment/images/image.js

@@ -26,9 +26,7 @@ class Image {
   #registry
   #runtime
   #values = {
-    build: {
-      image: 'node:20.9.0-alpine3.18'
-    }
+    build: {}
   }
 
   constructor (scope, runtime, registry) {
@@ -54,6 +52,8 @@ class Image {
 
   get base () {}
 
+  get run () {}
+
   async prepare (root) {
     if (this.dockerfile === undefined) throw new Error('Dockerfile isn\'t specified')
 
@@ -65,7 +65,7 @@ class Image {
 
     const template = await read(this.dockerfile, 'utf-8')
     const contents = template.replace(/{{(\S{1,32})}}/g, (_, key) => this.#value(key))
-    const ignore = 'Dockerfile'
+    const ignore = ['Dockerfile', '**/node_modules'].join('\r\n')
 
     await write(join(path, 'Dockerfile'), contents)
     await write(join(path, '.dockerignore'), ignore)
@@ -77,13 +77,19 @@ class Image {
 
   #setValues () {
     this.#values.runtime = this.#runtime
-    this.#values.build = overwrite(this.#values.build, this.#registry.build)
+    this.#values.build = overwrite({
+      image: `${RUNTIME_IMAGE}:${this.#runtime.version}`
+    }, this.#registry.build)
 
     const image = this.base
 
-    if (image !== undefined) {
+    if (image !== undefined)
       this.#values.build.image = image
-    }
+
+    const run = this.run
+
+    if (run !== undefined)
+      this.#values.build.run = (this.#values.build.run === undefined ? '' : this.#values.build.run + '\n') + run
 
     if (this.#values.build.arguments !== undefined) this.#values.build.arguments = createArguments(this.#values.build.arguments)
     if (this.#values.build.run !== undefined) this.#values.build.run = createRunCommands(this.#values.build.run)
@@ -121,4 +127,7 @@ function createArguments (variables) {
   return args.join('\n')
 }
 
+const RUNTIME_IMAGE = 'ghcr.io/toa-io/runtime'
+
 exports.Image = Image
+exports.RUNTIME_IMAGE = RUNTIME_IMAGE

package/src/deployment/images/mono.Dockerfile

@@ -0,0 +1,18 @@
+FROM {{build.image}}
+
+{{build.arguments}}
+
+ENV NODE_ENV=production
+RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
+RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
+
+WORKDIR /composition
+COPY --chown=node:node . /composition
+
+{{build.run}}
+
+RUN --mount=type=cache,target=/root/.npm \
+  for entry in *; do if [ -f "$entry/package.json" ]; then (cd $entry && npm i --omit=dev); fi; done
+
+USER node
+CMD toa mono *

package/src/deployment/images/mono.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const { join } = require('node:path')
+const fs = require('fs-extra')
+const { createHash } = require('node:crypto')
+
+const { Image } = require('./image')
+
+class Mono extends Image {
+  dockerfile = join(__dirname, 'mono.Dockerfile')
+
+  #image
+  #components
+
+  constructor (scope, runtime, registry, composition) {
+    super(scope, runtime, registry)
+
+    this.#image = composition.image
+    this.#components = composition.components
+  }
+
+  get name () {
+    return 'mono'
+  }
+
+  get version () {
+    const hash = createHash('sha256')
+
+    for (const component of this.#components) {
+      hash.update(component.locator.id)
+      hash.update(component.version)
+    }
+
+    return hash.digest('hex').slice(0, 8)
+  }
+
+  get base () {
+    if (this.#image !== undefined) return this.#image
+
+    const images = new Set(this.#components.map((component) => component.build?.image))
+
+    if (images.size > 1)
+      throw new Error('Mono deployment requires different base images for its components. Specify base image for the composition in the context.')
+
+    return images.values().next().value
+  }
+
+  get run () {
+    const commands = []
+
+    for (const component of this.#components) {
+      const run = component.build?.run
+
+      if (run !== undefined)
+        commands.push(run)
+    }
+
+    return commands.join('\n')
+  }
+
+  async prepare (root) {
+    const context = await super.prepare(root)
+
+    for (const component of this.#components)
+      await fs.copy(component.path, join(context, component.locator.label))
+
+    return context
+  }
+}
+
+exports.Mono = Mono

package/src/deployment/images/runtime-base.test.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const { join } = require('node:path')
+const { readFile } = require('node:fs/promises')
+const { directory } = require('@toa.io/filesystem')
+
+const { Image, RUNTIME_IMAGE } = require('./image')
+
+const compositionDockerfile = join(__dirname, 'composition.Dockerfile')
+const serviceDockerfile = join(__dirname, 'service.Dockerfile')
+
+class TestImage extends Image {
+  dockerfile
+
+  #name
+  #base
+
+  constructor (runtime, registry, dockerfile, name = 'test', base) {
+    super('acme', runtime, registry)
+
+    this.dockerfile = dockerfile
+    this.#name = name
+    this.#base = base
+  }
+
+  get name () {
+    return this.#name
+  }
+
+  get version () {
+    return 'abcdef12'
+  }
+
+  get base () {
+    return this.#base
+  }
+}
+
+describe('runtime base image', () => {
+  /** @type {string} */
+  let root
+
+  beforeEach(async () => {
+    root = await directory.temp('toa-runtime-base-test')
+  })
+
+  it('should default build.image to version-pinned GHCR runtime image', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(runtime, {}, compositionDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    expect(dockerfile).toContain(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.232`)
+    expect(dockerfile).not.toMatch(/npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should not install runtime in service Dockerfile template', async () => {
+    const runtime = { version: '1.0.0-alpha.99' }
+    const image = new TestImage(runtime, {}, serviceDockerfile, 'service-test')
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    expect(dockerfile).toContain(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.99`)
+    expect(dockerfile).not.toMatch(/npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should allow registry.build.image override', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const registry = { build: { image: 'node:24.14.0-alpine3.22' } }
+    const image = new TestImage(runtime, registry, compositionDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    expect(dockerfile).toContain('FROM node:24.14.0-alpine3.22')
+    expect(dockerfile).not.toContain(RUNTIME_IMAGE)
+  })
+
+  it('should allow composition.image override via base', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(runtime, {}, compositionDockerfile, 'mono', 'custom.example/base:1')
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    expect(dockerfile).toContain('FROM custom.example/base:1')
+  })
+})

package/src/deployment/images/service.Dockerfile

@@ -1,14 +1,14 @@
-FROM node:20.9.0-alpine3.18
+FROM {{build.image}}
 
 ENV NODE_ENV=production
 RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
 RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-RUN npm i -g @toa.io/runtime@{{runtime.version}} --omit=dev
 
 WORKDIR /service
 COPY --chown=node:node . /service
 
-RUN npm i --omit=dev
+RUN --mount=type=cache,target=/root/.npm \
+  npm i --omit=dev
 
 USER node
 CMD toa serve .

package/src/deployment/operator.js

@@ -50,6 +50,10 @@ class Operator {
   variables () {
     return this.#deployment.variables()
   }
+
+  tags () {
+    return this.#registry.tags()
+  }
 }
 
 /** @type {toa.deployment.installation.Options} */

package/src/deployment/registry.js

@@ -1,12 +1,14 @@
 'use strict'
 
+const { posix } = require('node:path')
 const workspace = require('./workspace')
-const { newid } = require('@toa.io/generic')
 
 /**
  * @implements {toa.deployment.Registry}
  */
 class Registry {
+  #scope
+
   #registry
 
   #factory
@@ -15,7 +17,11 @@ class Registry {
 
   #images = []
 
-  constructor (registry, factory, process) {
+  /** @type {string | undefined} */
+  #builder
+
+  constructor (scope, registry, factory, process) {
+    this.#scope = scope
     this.#registry = registry
     this.#factory = factory
     this.#process = process
@@ -29,6 +35,10 @@ class Registry {
     return this.#create('service', path, service)
   }
 
+  mono (composition) {
+    return this.#create('mono', composition)
+  }
+
   async prepare (root) {
     const path = await workspace.create('images', root)
 
@@ -40,9 +50,8 @@ class Registry {
   async build () {
     await this.prepare()
 
-    for (const image of this.#images) {
+    for (const image of this.#images)
       await this.#build(image)
-    }
   }
 
   async push () {
@@ -51,8 +60,12 @@ class Registry {
     for (const image of this.#images) await this.#push(image)
   }
 
+  tags () {
+    return this.#images.map((image) => image.reference)
+  }
+
   /**
-   * @param {'composition' | 'service'} type
+   * @param {'composition' | 'service' | 'mono'} type
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
@@ -70,13 +83,17 @@ class Registry {
    * @returns {Promise<void>}
    */
   async #build (image, push = false) {
+    if (await this.exists(image.reference)) {
+      console.log('Image already exists, skipping:', image.reference)
+      return
+    }
+
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push) {
+    if (push)
       args.push('--push')
-    } else {
+    else
       args.push('--load')
-    }
 
     args.push('--tag', image.reference, image.context)
 
@@ -88,14 +105,15 @@ class Registry {
 
     if (multiarch) {
       const platform = this.#registry.platforms.join(',')
-      const builder = await this.#createBuilder()
+      const builder = await this.#ensureBuilder()
 
       args.push('--platform', platform)
       args.push('--builder', builder)
 
-    } else {
+      if (this.#registry.base !== undefined)
+        this.#appendCache(args)
+    } else
       args.push('--builder', 'default')
-    }
 
     args.push('--progress', 'plain')
 
@@ -106,13 +124,43 @@ class Registry {
     await this.#build(image, true)
   }
 
-  async #createBuilder () {
-    const name = `toa-${newid()}`
-    const create = `buildx create --name ${name} --bootstrap --use`.split(' ')
+  async exists (tag) {
+    const args = ['manifest', 'inspect', tag]
+
+    try {
+      await this.#process.execute('docker', args, { silently: true })
+    } catch (error) {
+      console.log(error.message)
+
+      return false
+    }
+
+    return true
+  }
+
+  async #ensureBuilder () {
+    if (this.#builder !== undefined)
+      return this.#builder
+
+    try {
+      await this.#process.execute('docker', ['buildx', 'inspect', BUILDER], { silently: true })
+    } catch {
+      await this.#process.execute('docker', ['buildx', 'create', '--name', BUILDER, '--bootstrap'])
+    }
+
+    this.#builder = BUILDER
+
+    return BUILDER
+  }
 
-    await this.#process.execute('docker', create)
+  /**
+   * @param {string[]} args
+   */
+  #appendCache (args) {
+    const ref = posix.join(this.#registry.base, this.#scope, 'buildcache')
 
-    return name
+    args.push('--cache-from', `type=registry,ref=${ref}`)
+    args.push('--cache-to', `type=registry,ref=${ref},mode=max,image-manifest=true`)
   }
 }
 

package/src/deployment/registry.test.js

@@ -0,0 +1,175 @@
+'use strict'
+
+const { Registry } = require('./registry')
+
+/** @type {toa.operations.Process} */
+let process
+
+/** @type {toa.deployment.images.Factory} */
+let factory
+
+/** @type {toa.deployment.images.Image[]} */
+let images
+
+beforeEach(() => {
+  images = []
+  process = /** @type {toa.operations.Process} */ {
+    execute: jest.fn(async (cmd, args) => {
+      if (args[0] === 'manifest')
+        throw new Error('manifest unknown')
+
+      if (args[0] === 'buildx' && args[1] === 'inspect')
+        throw new Error('builder not found')
+
+      return ''
+    })
+  }
+
+  factory = /** @type {toa.deployment.images.Factory} */ {
+    composition: () => createImage('composition-mono'),
+    service: () => createImage('extension-realtime')
+  }
+})
+
+it('should reuse named builder across images', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === 'buildx' && args[1] === 'create')
+
+  expect(creates).toHaveLength(1)
+  expect(creates[0][1]).toEqual(['buildx', 'create', '--name', 'toa', '--bootstrap'])
+
+  const builds = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === '--context=default' && args[1] === 'buildx' && args[2] === 'build')
+
+  expect(builds).toHaveLength(2)
+
+  for (const [, args] of builds) {
+    expect(args).toContain('--builder')
+    expect(args[args.indexOf('--builder') + 1]).toBe('toa')
+  }
+})
+
+it('should not create builder when it already exists', async () => {
+  process.execute = jest.fn(async (cmd, args) => {
+    if (args[0] === 'manifest')
+      throw new Error('manifest unknown')
+
+    return ''
+  })
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const creates = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === 'buildx' && args[1] === 'create')
+
+  expect(creates).toHaveLength(0)
+})
+
+it('should add shared registry cache flags when base is set', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+  registry.service('.', /** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === '--context=default' && args[2] === 'build')
+
+  expect(builds).toHaveLength(2)
+
+  for (const [, args] of builds) {
+    expect(args).toContain('--cache-from')
+    expect(args).toContain('type=registry,ref=example.com/reg/acme/buildcache')
+    expect(args).toContain('--cache-to')
+    expect(args).toContain('type=registry,ref=example.com/reg/acme/buildcache,mode=max,image-manifest=true')
+  }
+})
+
+it('should omit cache flags when base is not set', async () => {
+  const registry = createRegistry({ platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === '--context=default' && args[2] === 'build')
+
+  expect(builds).toHaveLength(1)
+
+  const [, args] = builds[0]
+
+  expect(args).not.toContain('--cache-from')
+  expect(args).not.toContain('--cache-to')
+})
+
+it('should use default builder when platforms is null', async () => {
+  const registry = createRegistry({ base: 'example.com/reg', platforms: null })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === '--context=default' && args[2] === 'build')
+
+  expect(builds).toHaveLength(1)
+
+  const [, args] = builds[0]
+
+  expect(args[args.indexOf('--builder') + 1]).toBe('default')
+  expect(args).not.toContain('--cache-from')
+  expect(args).not.toContain('--platform')
+})
+
+it('should skip build when image already exists', async () => {
+  process.execute = jest.fn(async () => '')
+
+  const registry = createRegistry({ base: 'example.com/reg', platforms: ['linux/amd64'] })
+
+  registry.composition(/** @type {any} */ ({}))
+
+  await registry.build()
+
+  const builds = process.execute.mock.calls.filter(([, args]) =>
+    args[0] === '--context=default' && args[2] === 'build')
+
+  expect(builds).toHaveLength(0)
+  expect(process.execute).toHaveBeenCalledWith('docker', ['manifest', 'inspect', images[0].reference], { silently: true })
+})
+
+/**
+ * @param {object} [registry]
+ * @returns {Registry}
+ */
+function createRegistry (registry = {}) {
+  return new Registry('acme', registry, factory, process)
+}
+
+/**
+ * @param {string} name
+ * @returns {toa.deployment.images.Image}
+ */
+function createImage (name) {
+  const image = /** @type {toa.deployment.images.Image} */ {
+    reference: `example.com/reg/acme/${name}:abcdef12`,
+    context: `/tmp/${name}`,
+    prepare: jest.fn(async () => `/tmp/${name}`)
+  }
+
+  images.push(image)
+
+  return image
+}

package/types/_deployment/dependency.d.ts

@@ -50,6 +50,7 @@ declare namespace toa.deployment {
       services?: Service[] // dependency.Service
       proxies?: Proxy[]
       variables?: Variables
+      events?: string[]
     }
 
   }
@@ -59,6 +60,7 @@ declare namespace toa.deployment {
     services?: _service.Service[] // deployment.Service
     proxies?: dependency.Proxy[]
     variables?: dependency.Variables
+    events?: string[]
   }
 
 }