@toa.io/operations 1.0.0-alpha.34 → 1.0.0-alpha.341

package/src/deployment/images/image.js

@@ -1,61 +1,90 @@
-'use strict'
+import { join, posix } from 'node:path'
+import { readFile as read, writeFile as write } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
 
-const {
-  join,
-  posix
-} = require('node:path')
-const {
-  readFile: read,
-  writeFile: write
-} = require('node:fs/promises')
-const { createHash } = require('node:crypto')
-
-const { overwrite } = require('@toa.io/generic')
-const { mkdir } = require('node:fs/promises')
+import { overwrite } from '@toa.io/generic'
+import { MAP } from '@toa.io/definitions'
+import { mkdir } from 'node:fs/promises'
 
 /**
  * @implements {toa.deployment.images.Image}
  * @abstract
  */
-class Image {
+export class Image {
   context
   reference
   dockerfile
 
+  /**
+   * The image this one is laid over, built and pushed as an image of its own.
+   * Undefined stands on the base alone, which is what a service does.
+   *
+   * @type {toa.deployment.images.Image | undefined}
+   */
+  dependencies
+
+  /** Whether the build reads `registry.build.arguments`. */
+  arguments = false
+
   #scope
   #registry
   #runtime
   #values = {
-    build: {
-      image: 'node:20.9.0-alpine3.18'
-    }
+    build: {}
   }
 
-  constructor (scope, runtime, registry) {
+  constructor(scope, runtime, registry) {
     this.#scope = scope
     this.#registry = registry
     this.#runtime = runtime
   }
 
-  tag () {
+  tag() {
+    this.reference = posix.join(
+      this.#registry.base ?? '',
+      this.#scope,
+      `${this.name}:${this.digest()}`
+    )
+  }
+
+  /**
+   * What follows the colon: the runtime and the image's own version, digested.
+   *
+   * @returns {string}
+   */
+  digest() {
     const hash = createHash('sha256')
 
     hash.update(this.#runtime.version)
     hash.update(this.version)
 
-    const tag = hash.digest('hex').slice(0, 8)
+    return hash.digest('hex').slice(0, 8)
+  }
 
-    this.reference = posix.join(this.#registry.base ?? '', this.#scope, `${this.name}:${tag}`)
+  /** @returns {string | undefined} */
+  get name() {
+    return undefined
   }
 
-  get name () {}
+  /** @returns {string | undefined} */
+  get version() {
+    return undefined
+  }
 
-  get version () {}
+  /** The image to build `FROM`. Undefined takes the runtime's, which is what a service does.
+   *  @returns {string | undefined} */
+  get base() {
+    return undefined
+  }
 
-  get base () {}
+  /** Build commands to add. Undefined adds none, which is what a service does.
+   *  @returns {string | undefined} */
+  get run() {
+    return undefined
+  }
 
-  async prepare (root) {
-    if (this.dockerfile === undefined) throw new Error('Dockerfile isn\'t specified')
+  async prepare(root) {
+    if (this.dockerfile === undefined) throw new Error("Dockerfile isn't specified")
 
     this.#setValues()
 
@@ -65,7 +94,7 @@ class Image {
 
     const template = await read(this.dockerfile, 'utf-8')
     const contents = template.replace(/{{(\S{1,32})}}/g, (_, key) => this.#value(key))
-    const ignore = 'Dockerfile'
+    const ignore = ['Dockerfile', '**/node_modules'].join('\r\n')
 
     await write(join(path, 'Dockerfile'), contents)
     await write(join(path, '.dockerignore'), ignore)
@@ -75,32 +104,44 @@ class Image {
     return path
   }
 
-  #setValues () {
+  #setValues() {
+    this.#values.map = { file: MAP }
     this.#values.runtime = this.#runtime
-    this.#values.build = overwrite(this.#values.build, this.#registry.build)
+    this.#values.build = overwrite(
+      {
+        image: `${RUNTIME_IMAGE}:${this.#runtime.version}`
+      },
+      this.#registry.build
+    )
 
     const image = this.base
 
-    if (image !== undefined) {
-      this.#values.build.image = image
-    }
+    if (image !== undefined) this.#values.build.image = image
+
+    const run = this.run
+
+    if (run !== undefined)
+      this.#values.build.run =
+        (this.#values.build.run === undefined ? '' : this.#values.build.run + '\n') + run
 
-    if (this.#values.build.arguments !== undefined) this.#values.build.arguments = createArguments(this.#values.build.arguments)
-    if (this.#values.build.run !== undefined) this.#values.build.run = createRunCommands(this.#values.build.run)
+    if (this.#values.build.arguments !== undefined)
+      this.#values.build.arguments = createArguments(this.#values.build.arguments)
+    if (this.#values.build.run !== undefined)
+      this.#values.build.run = createRunCommands(this.#values.build.run)
   }
 
   /**
    * @param key {string}
    * @returns {string}
    */
-  #value (key) {
+  #value(key) {
     const [source, property] = key.split('.')
 
     return this.#values[source]?.[property] ?? ''
   }
 }
 
-function createRunCommands (input) {
+function createRunCommands(input) {
   const lines = input.split('\n')
 
   return lines.reduce((commands, command) => {
@@ -110,7 +151,7 @@ function createRunCommands (input) {
   }, '')
 }
 
-function createArguments (variables) {
+function createArguments(variables) {
   const args = []
 
   for (const variable of variables) {
@@ -121,4 +162,4 @@ function createArguments (variables) {
   return args.join('\n')
 }
 
-exports.Image = Image
+export const RUNTIME_IMAGE = 'ghcr.io/toa-io/runtime'

package/src/deployment/images/image.test.js

@@ -1,7 +1,8 @@
-'use strict'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
 
-const fixtures = require('./image.fixtures')
-const { generate } = require('randomstring')
+import * as fixtures from './image.fixtures.js'
+import { generate } from 'randomstring'
 
 /** @type {toa.deployment.images.Image} */
 let instance
@@ -13,11 +14,16 @@ beforeEach(() => {
 it('should assign url', () => {
   instance.tag()
 
-  expect(instance.reference).toEqual(`${fixtures.registry.base}/${fixtures.scope}/${fixtures.name}:${fixtures.version}`)
+  assert.deepStrictEqual(
+    instance.reference,
+    `${fixtures.registry.base}/${fixtures.scope}/${fixtures.name}:${fixtures.version}`
+  )
 })
 
 describe('prepare', () => {
   it('should throw error if no dockerfile specified', async () => {
-    await expect(instance.prepare(generate())).rejects.toThrow(/Dockerfile isn't specified/)
+    await assert.rejects(instance.prepare(generate()), (error) =>
+      /Dockerfile isn't specified/.test(error.message)
+    )
   })
 })

package/src/deployment/images/index.js

@@ -1,5 +1 @@
-'use strict'
-
-const { Factory } = require('./factory')
-
-exports.Factory = Factory
+export { Factory } from './factory.js'

package/src/deployment/images/mono.Dockerfile

@@ -0,0 +1,11 @@
+FROM {{build.image}}
+
+# the one instruction that touches the filesystem, and linked: the base is not pulled to lay
+# the sources over it, and a push carries this layer and a manifest, not the base's layers.
+# Owned by root like the dependencies beside them: a linked layer knows no user by name,
+# and the runtime reads its sources, it does not write them
+COPY --link . /composition
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa mono * --map {{map.file}}

package/src/deployment/images/mono.js

@@ -0,0 +1,15 @@
+import { join } from 'node:path'
+
+import { Bundle } from './bundle.js'
+
+export class Mono extends Bundle {
+  dockerfile = join(import.meta.dirname, 'mono.Dockerfile')
+
+  get name() {
+    return 'mono'
+  }
+
+  conflict() {
+    return 'Mono deployment requires different base images for its components. Specify base image for the composition in the context.'
+  }
+}

package/src/deployment/images/publish.js

@@ -0,0 +1,91 @@
+import { mkdtemp, writeFile } from 'node:fs/promises'
+import { readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { definition } from '@toa.io/norm'
+
+import { run } from '../../process.js'
+import { Service } from './service.js'
+
+/**
+ * Builds and pushes the image an extension ships for its service, so that an application
+ * with `registry.services: published` takes it instead of building one.
+ *
+ * The image is built from the package as npm publishes it, not from the workspace: what
+ * `files` leaves out is not in the tarball, and an application builds from the tarball.
+ *
+ * @param {string} workspace path to the extension's directory in this repository
+ * @param {string} runtime the runtime version, which is the tag and the base image
+ * @param {string[]} platforms
+ * @returns {Promise<string>} the reference pushed
+ */
+export async function publish(workspace, runtime, platforms) {
+  const directory = resolve(workspace)
+  const manifest = read(join(directory, 'package.json'))
+
+  const { image } = (await definition(manifest.name)).module
+
+  if (image === undefined)
+    throw new Error(
+      `'${manifest.name}' publishes no service image: it exports no 'image'`
+    )
+
+  const root = await mkdtemp(join(tmpdir(), 'toa-publish'))
+
+  await writeFile(join(root, 'package.json'), '{"private":true}')
+  await run('npm', [
+    'i',
+    '--prefix',
+    root,
+    '--omit=dev',
+    `${manifest.name}@${manifest.version}`
+  ])
+
+  const path = join(root, 'node_modules', manifest.name)
+
+  const service = new Service(SCOPE, { version: runtime }, {}, path, {
+    ...name(image),
+    version: manifest.version
+  })
+
+  service.reference = `${image}:${runtime}`
+
+  await service.prepare(await mkdtemp(join(tmpdir(), 'toa-images')))
+
+  await run('docker', [
+    'buildx',
+    'build',
+    '--platform',
+    platforms.join(','),
+    '--tag',
+    service.reference,
+    '--provenance=false',
+    '--push',
+    service.context
+  ])
+
+  return service.reference
+}
+
+/** The repository's last segment is what `Service` names an image, so it reads back. */
+function name(image) {
+  const match = /\/extension-([^-/]+)-([^/]+)$/.exec(image)
+
+  if (match === null)
+    throw new Error(`'${image}' is not named 'extension-<group>-<name>'`)
+
+  return { group: match[1], name: match[2] }
+}
+
+const read = (path) => JSON.parse(readFileSync(path, 'utf8'))
+
+// the reference is assigned rather than derived from a scope
+const SCOPE = ''
+
+const PLATFORMS = 'linux/amd64,linux/arm64'
+
+if (import.meta.main) {
+  const [workspace, runtime, platforms = PLATFORMS] = process.argv.slice(2)
+
+  console.log(await publish(workspace, runtime, platforms.split(',')))
+}

package/src/deployment/images/runtime-base.test.js

@@ -0,0 +1,97 @@
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { join } from 'node:path'
+import { mkdtemp, readFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+
+import { Image, RUNTIME_IMAGE } from './image.js'
+
+const dependenciesDockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
+const serviceDockerfile = join(import.meta.dirname, 'service.Dockerfile')
+
+class TestImage extends Image {
+  dockerfile
+
+  #name
+  #base
+
+  constructor(runtime, registry, dockerfile, name = 'test', base) {
+    super('acme', runtime, registry)
+
+    this.dockerfile = dockerfile
+    this.#name = name
+    this.#base = base
+  }
+
+  get name() {
+    return this.#name
+  }
+
+  get version() {
+    return 'abcdef12'
+  }
+
+  get base() {
+    return this.#base
+  }
+}
+
+describe('runtime base image', () => {
+  /** @type {string} */
+  let root
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), 'toa-runtime-base-test'))
+  })
+
+  it('should default build.image to version-pinned GHCR runtime image', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(runtime, {}, dependenciesDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.232`))
+    assert.doesNotMatch(dockerfile, /npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should not install runtime in service Dockerfile template', async () => {
+    const runtime = { version: '1.0.0-alpha.99' }
+    const image = new TestImage(runtime, {}, serviceDockerfile, 'service-test')
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.99`))
+    assert.doesNotMatch(dockerfile, /npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should allow registry.build.image override', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const registry = { build: { image: 'node:24.22.1-alpine3.24' } }
+    const image = new TestImage(runtime, registry, dependenciesDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM node:24.22.1-alpine3.24'))
+    assert.ok(!dockerfile.includes(RUNTIME_IMAGE))
+  })
+
+  it('should allow composition.image override via base', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(
+      runtime,
+      {},
+      dependenciesDockerfile,
+      'mono',
+      'custom.example/base:1'
+    )
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM custom.example/base:1'))
+  })
+})

package/src/deployment/images/service.Dockerfile

@@ -1,14 +1,26 @@
-FROM node:20.9.0-alpine3.18
+FROM {{build.image}}
 
 ENV NODE_ENV=production
 RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
 RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-RUN npm i -g @toa.io/runtime@{{runtime.version}} --omit=dev
 
 WORKDIR /service
 COPY --chown=node:node . /service
 
-RUN npm i --omit=dev
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  npm i --omit=dev
 
-USER node
-CMD toa serve .
+# a component of the extension's own declares what it imports, and it is installed beside the
+# component rather than beside the extension: an application that runs none of them, and the
+# runtime image every application is built on, carry nothing for them
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  for entry in components/*; do if grep -qs '"dependencies"' "$entry/package.json"; then (cd $entry && npm i --omit=dev); fi; done
+
+# and beside Toa as well, because what an extension imports on a component's behalf — a storage
+# provider's SDK — is imported from where the extension is, which is `/toa` and not here
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  if [ -s .packages ]; then npm i --prefix /toa --omit=dev $(cat .packages); fi
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa serve . --map {{map.file}}

package/src/deployment/images/service.js

@@ -1,12 +1,15 @@
-'use strict'
+import { createRequire } from 'node:module'
+import { join, dirname } from 'node:path'
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
 
-const { join, dirname } = require('node:path')
+import { Image } from './image.js'
+import { cp, writeFile } from 'node:fs/promises'
 
-const { Image } = require('./image')
-const fs = require('fs-extra')
+// a service is named the way a package is, and its directory is where it lives
+const require = createRequire(import.meta.url)
 
-class Service extends Image {
-  dockerfile = join(__dirname, 'service.Dockerfile')
+export class Service extends Image {
+  dockerfile = join(import.meta.dirname, 'service.Dockerfile')
 
   /**
    * Used by Dockerfile
@@ -35,7 +38,7 @@ class Service extends Image {
    * @param {string} reference
    * @param {toa.deployment.dependency.Service} service
    */
-  constructor (scope, runtime, registry, reference, service) {
+  constructor(scope, runtime, registry, reference, service) {
     super(scope, runtime, registry)
 
     this.service = service.name
@@ -46,29 +49,75 @@ class Service extends Image {
     this.#version = service.version
   }
 
-  get name () {
+  get name() {
     return 'extension-' + this.#group + '-' + this.#name
   }
 
-  get version () {
+  get version() {
     return this.#version
   }
 
-  async prepare (root) {
+  async prepare(root) {
     const context = await super.prepare(root)
 
-    await fs.copy(this.#path, context)
+    await cp(this.#path, context, { recursive: true })
+    await writeFile(join(context, PACKAGES), this.#packages().join('\n'))
 
     return context
   }
+
+  /**
+   * What the components this service runs declare, as `name@version`.
+   *
+   * Each is installed beside its component, where the component's own modules import it, and
+   * beside Toa, where an extension that imports on a component's behalf is — a storage
+   * provider's SDK is imported by `@toa.io/extensions.storages`, not by the component that
+   * declares the storage.
+   *
+   * @returns {string[]}
+   */
+  #packages() {
+    const directory = join(this.#path, 'components')
+
+    if (!existsSync(directory)) return []
+
+    /** @type {Record<string, string>} */
+    const packages = {}
+
+    for (const label of readdirSync(directory)) {
+      const manifest = join(directory, label, 'package.json')
+
+      if (!existsSync(manifest)) continue
+
+      Object.assign(packages, JSON.parse(readFileSync(manifest, 'utf8')).dependencies)
+    }
+
+    return Object.entries(packages)
+      .map(([name, version]) => `${name}@${version}`)
+      .sort()
+  }
 }
 
+/** Where the install reads what the components need, one `name@version` per line. */
+const PACKAGES = '.packages'
+
 /**
+ * Where the extension is installed, which is what its image is built from. A deploy install
+ * carries what an extension declares and not the extension, so a service is built only where
+ * the runtime is installed beside the deployment library; elsewhere its image is published.
+ *
  * @param {string} reference
  * @returns {string}
  */
 const find = (reference) => {
-  return dirname(require.resolve(join(reference, 'package.json')))
+  try {
+    return dirname(require.resolve(join(reference, 'package.json')))
+  } catch (error) {
+    throw new Error(
+      `'${reference}' is not installed, and \`registry.services: build\` builds its image ` +
+        'from where it is: install the runtime beside @toa.io/operations, or take the ' +
+        'image the release publishes with `registry.services: published`',
+      { cause: error }
+    )
+  }
 }
-
-exports.Service = Service

package/src/deployment/index.js

@@ -1,5 +1 @@
-'use strict'
-
-const { Factory } = require('./factory')
-
-exports.Factory = Factory
+export { Factory } from './factory.js'

package/src/deployment/operator.js

@@ -1,24 +1,27 @@
-'use strict'
+import * as workspace from './workspace.js'
 
-const workspace = require('./workspace')
-
-class Operator {
+export class Operator {
   /** @type {toa.deployment.Deployment} */
   #deployment
 
   /** @type {toa.deployment.Registry} */
   #registry
 
+  /** @type {string | undefined} */
+  #environment
+
   /**
-   * @param deployment {toa.deployment.Deployment}
-   * @param registry {toa.deployment.Registry}
+   * @param {toa.deployment.Deployment} deployment
+   * @param {toa.deployment.Registry} registry
+   * @param {string} [environment]
    */
-  constructor (deployment, registry) {
+  constructor(deployment, registry, environment) {
     this.#deployment = deployment
     this.#registry = registry
+    this.#environment = environment
   }
 
-  async export (path) {
+  async export(path) {
     const target = await workspace.create('deployment', path)
 
     await this.#deployment.export(target)
@@ -26,33 +29,39 @@ class Operator {
     return target
   }
 
-  async prepare (path) {
+  async prepare(path) {
     return await this.#registry.prepare(path)
   }
 
-  async push () {
+  async push() {
     await this.#registry.push()
   }
 
-  async install (options = {}) {
+  async install(options = {}) {
     options = Object.assign({}, OPTIONS, options)
 
+    // nothing is built, pushed or applied where the cluster cannot run what this deploys
+    await this.#deployment.verify(options)
+
     await Promise.all([this.export(), this.push()])
+    await this.#registry.alias(this.#environment)
     await this.#deployment.install(options)
   }
 
-  async template (options = {}) {
+  async template(options = {}) {
     await this.export()
 
     return await this.#deployment.template(options)
   }
 
-  variables () {
-    return this.#deployment.variables()
+  variables(options) {
+    return this.#deployment.variables(options)
+  }
+
+  tags() {
+    return this.#registry.tags()
   }
 }
 
 /** @type {toa.deployment.installation.Options} */
 const OPTIONS = { wait: false }
-
-exports.Operator = Operator