@toa.io/operations 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/deployment/images/format.test.js

@@ -0,0 +1,149 @@
+import { it, describe, before, after } from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { Locator } from '@toa.io/core'
+import { revive } from '@toa.io/norm'
+
+import { declare, format, normalized } from './format.js'
+
+let root
+
+before(async () => {
+  root = await mkdtemp(join(tmpdir(), 'format-'))
+})
+
+after(async () => {
+  await rm(root, { recursive: true, force: true })
+})
+
+describe('format', () => {
+  it('should read the nearest manifest above', async () => {
+    const workspace = join(root, 'modules')
+    const component = join(workspace, 'components', 'one')
+
+    await mkdir(component, { recursive: true })
+    await writeFile(join(workspace, 'package.json'), JSON.stringify({ type: 'module' }))
+
+    assert.strictEqual(format(component), 'module')
+  })
+
+  it('should default to commonjs', async () => {
+    const workspace = join(root, 'scripts')
+    const component = join(workspace, 'components', 'one')
+
+    await mkdir(component, { recursive: true })
+    await writeFile(join(workspace, 'package.json'), JSON.stringify({ name: 'scripts' }))
+
+    assert.strictEqual(format(component), 'commonjs')
+  })
+})
+
+describe('declare', () => {
+  it('should state the format the component was written under', async () => {
+    const workspace = join(root, 'stated')
+    const source = join(workspace, 'components', 'one')
+    const target = join(root, 'image', 'one')
+
+    await mkdir(source, { recursive: true })
+    await mkdir(target, { recursive: true })
+    await writeFile(join(workspace, 'package.json'), JSON.stringify({ type: 'module' }))
+
+    await declare(source, target, 'default-one')
+
+    const manifest = JSON.parse(await readFile(join(target, 'package.json'), 'utf8'))
+
+    assert.strictEqual(manifest.type, 'module')
+    assert.strictEqual(manifest.name, 'default-one')
+    assert.strictEqual(manifest.private, true)
+  })
+
+  it('should leave a manifest the component ships alone', async () => {
+    const source = join(root, 'own', 'components', 'one')
+    const target = join(root, 'image', 'own')
+
+    await mkdir(source, { recursive: true })
+    await mkdir(target, { recursive: true })
+
+    const own = JSON.stringify({ name: 'own', dependencies: { matchacho: '0.6.0' } })
+
+    await writeFile(join(target, 'package.json'), own)
+    await declare(source, target, 'default-own')
+
+    assert.strictEqual(await readFile(join(target, 'package.json'), 'utf8'), own)
+  })
+})
+
+describe('normalized', () => {
+  it('should write what a process reads instead of normalizing', async () => {
+    const target = join(root, 'image', 'normalized')
+
+    await mkdir(target, { recursive: true })
+
+    const component = {
+      name: 'tasks',
+      namespace: 'todos',
+      version: 'abcdef12',
+      operations: { compute: { type: 'computation' } },
+      path: '/workspace/components/todos.tasks',
+      locator: new Locator('tasks', 'todos'),
+      packages: { cloudinary: '2.11.0' }
+    }
+
+    await normalized(component, target)
+
+    const carried = JSON.parse(
+      await readFile(join(target, 'manifest.toa.json'), 'utf8')
+    )
+
+    // where the manifest is, and what only a deploy reads, are not what it declares
+    assert.ok(!('path' in carried))
+    assert.ok(!('locator' in carried))
+    assert.ok(!('packages' in carried))
+
+    const read = revive(carried, '/composition/todos-tasks')
+
+    assert.deepStrictEqual(read.operations, component.operations)
+    assert.strictEqual(read.version, component.version)
+    assert.strictEqual(read.path, '/composition/todos-tasks')
+    assert.strictEqual(read.locator.id, 'todos.tasks')
+  })
+})
+
+describe('a service image', () => {
+  it('installs what its components declare beside Toa as well', async () => {
+    const { Service } = await import('./service.js')
+
+    const path = join(root, 'extension')
+    const component = join(path, 'components', 'octets')
+
+    await mkdir(component, { recursive: true })
+    await writeFile(join(path, 'package.json'), JSON.stringify({ name: 'ext', version: '1' }))
+    await writeFile(
+      join(component, 'package.json'),
+      JSON.stringify({ dependencies: { cloudinary: '2.11.0', jose: '6.2.10' } })
+    )
+
+    const service = new Service('', { version: '1' }, {}, path, {
+      group: 'g',
+      name: 'n',
+      version: '1'
+    })
+
+    service.reference = 'probe'
+
+    const context = await service.prepare(join(root, 'service'))
+
+    // what an extension imports on a component's behalf is imported from where Toa is
+    assert.strictEqual(
+      await readFile(join(context, '.packages'), 'utf8'),
+      'cloudinary@2.11.0\njose@6.2.10'
+    )
+
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('npm i --prefix /toa --omit=dev $(cat .packages)'))
+  })
+})

package/src/deployment/images/image.fixtures.js

@@ -1,38 +1,36 @@
-'use strict'
+import { Image } from './image.js'
+import { generate } from 'randomstring'
 
-const { Image } = require('./image')
-const { generate } = require('randomstring')
-
-const version = '168b04ff'
-const name = generate()
+const reported = '168b04ff'
+export const name = generate()
 
 /**
  * @implements {toa.deployment.images.Image}
  */
-class Class extends Image {
-  get name () {
+export class Class extends Image {
+  get name() {
     return name
   }
 
-  get version () {
-    return version
+  get version() {
+    return reported
   }
 }
 
 /** @type {toa.norm.context.Runtime} */
-const runtime = {
+export const runtime = {
   version: '0.0.0'
 }
 
 /** @type {toa.norm.context.Registry} */
-const registry = {
+export const registry = {
   base: 'node:alpine'
 }
 
-exports.scope = generate()
-exports.name = name
-exports.version = 'ba2409fc'
-exports.Class = Class
-exports.runtime = runtime
-exports.registry = registry
-exports.process = process
+export const scope = generate()
+
+// distinct from the version `Class` reports
+export const version = 'ba2409fc'
+
+// the fixture stands in for the global process
+export const process = globalThis.process

package/src/deployment/images/image.js

@@ -1,61 +1,90 @@
-'use strict'
+import { join, posix } from 'node:path'
+import { readFile as read, writeFile as write } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
 
-const {
-  join,
-  posix
-} = require('node:path')
-const {
-  readFile: read,
-  writeFile: write
-} = require('node:fs/promises')
-const { createHash } = require('node:crypto')
-
-const { overwrite } = require('@toa.io/generic')
-const { mkdir } = require('node:fs/promises')
+import { overwrite } from '@toa.io/generic'
+import { MAP } from '@toa.io/definitions'
+import { mkdir } from 'node:fs/promises'
 
 /**
  * @implements {toa.deployment.images.Image}
  * @abstract
  */
-class Image {
+export class Image {
   context
   reference
   dockerfile
 
+  /**
+   * The image this one is laid over, built and pushed as an image of its own.
+   * Undefined stands on the base alone, which is what a service does.
+   *
+   * @type {toa.deployment.images.Image | undefined}
+   */
+  dependencies
+
+  /** Whether the build reads `registry.build.arguments`. */
+  arguments = false
+
   #scope
   #registry
   #runtime
   #values = {
-    build: {
-      image: 'node:20.9.0-alpine3.18'
-    }
+    build: {}
   }
 
-  constructor (scope, runtime, registry) {
+  constructor(scope, runtime, registry) {
     this.#scope = scope
     this.#registry = registry
     this.#runtime = runtime
   }
 
-  tag () {
+  tag() {
+    this.reference = posix.join(
+      this.#registry.base ?? '',
+      this.#scope,
+      `${this.name}:${this.digest()}`
+    )
+  }
+
+  /**
+   * What follows the colon: the runtime and the image's own version, digested.
+   *
+   * @returns {string}
+   */
+  digest() {
     const hash = createHash('sha256')
 
     hash.update(this.#runtime.version)
     hash.update(this.version)
 
-    const tag = hash.digest('hex').slice(0, 8)
+    return hash.digest('hex').slice(0, 8)
+  }
 
-    this.reference = posix.join(this.#registry.base ?? '', this.#scope, `${this.name}:${tag}`)
+  /** @returns {string | undefined} */
+  get name() {
+    return undefined
   }
 
-  get name () {}
+  /** @returns {string | undefined} */
+  get version() {
+    return undefined
+  }
 
-  get version () {}
+  /** The image to build `FROM`. Undefined takes the runtime's, which is what a service does.
+   *  @returns {string | undefined} */
+  get base() {
+    return undefined
+  }
 
-  get base () {}
+  /** Build commands to add. Undefined adds none, which is what a service does.
+   *  @returns {string | undefined} */
+  get run() {
+    return undefined
+  }
 
-  async prepare (root) {
-    if (this.dockerfile === undefined) throw new Error('Dockerfile isn\'t specified')
+  async prepare(root) {
+    if (this.dockerfile === undefined) throw new Error("Dockerfile isn't specified")
 
     this.#setValues()
 
@@ -65,7 +94,7 @@ class Image {
 
     const template = await read(this.dockerfile, 'utf-8')
     const contents = template.replace(/{{(\S{1,32})}}/g, (_, key) => this.#value(key))
-    const ignore = 'Dockerfile'
+    const ignore = ['Dockerfile', '**/node_modules'].join('\r\n')
 
     await write(join(path, 'Dockerfile'), contents)
     await write(join(path, '.dockerignore'), ignore)
@@ -75,32 +104,44 @@ class Image {
     return path
   }
 
-  #setValues () {
+  #setValues() {
+    this.#values.map = { file: MAP }
     this.#values.runtime = this.#runtime
-    this.#values.build = overwrite(this.#values.build, this.#registry.build)
+    this.#values.build = overwrite(
+      {
+        image: `${RUNTIME_IMAGE}:${this.#runtime.version}`
+      },
+      this.#registry.build
+    )
 
     const image = this.base
 
-    if (image !== undefined) {
-      this.#values.build.image = image
-    }
+    if (image !== undefined) this.#values.build.image = image
+
+    const run = this.run
+
+    if (run !== undefined)
+      this.#values.build.run =
+        (this.#values.build.run === undefined ? '' : this.#values.build.run + '\n') + run
 
-    if (this.#values.build.arguments !== undefined) this.#values.build.arguments = createArguments(this.#values.build.arguments)
-    if (this.#values.build.run !== undefined) this.#values.build.run = createRunCommands(this.#values.build.run)
+    if (this.#values.build.arguments !== undefined)
+      this.#values.build.arguments = createArguments(this.#values.build.arguments)
+    if (this.#values.build.run !== undefined)
+      this.#values.build.run = createRunCommands(this.#values.build.run)
   }
 
   /**
    * @param key {string}
    * @returns {string}
    */
-  #value (key) {
+  #value(key) {
     const [source, property] = key.split('.')
 
     return this.#values[source]?.[property] ?? ''
   }
 }
 
-function createRunCommands (input) {
+function createRunCommands(input) {
   const lines = input.split('\n')
 
   return lines.reduce((commands, command) => {
@@ -110,7 +151,7 @@ function createRunCommands (input) {
   }, '')
 }
 
-function createArguments (variables) {
+function createArguments(variables) {
   const args = []
 
   for (const variable of variables) {
@@ -121,4 +162,4 @@ function createArguments (variables) {
   return args.join('\n')
 }
 
-exports.Image = Image
+export const RUNTIME_IMAGE = 'ghcr.io/toa-io/runtime'

package/src/deployment/images/image.test.js

@@ -1,7 +1,8 @@
-'use strict'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
 
-const fixtures = require('./image.fixtures')
-const { generate } = require('randomstring')
+import * as fixtures from './image.fixtures.js'
+import { generate } from 'randomstring'
 
 /** @type {toa.deployment.images.Image} */
 let instance
@@ -13,11 +14,16 @@ beforeEach(() => {
 it('should assign url', () => {
   instance.tag()
 
-  expect(instance.reference).toEqual(`${fixtures.registry.base}/${fixtures.scope}/${fixtures.name}:${fixtures.version}`)
+  assert.deepStrictEqual(
+    instance.reference,
+    `${fixtures.registry.base}/${fixtures.scope}/${fixtures.name}:${fixtures.version}`
+  )
 })
 
 describe('prepare', () => {
   it('should throw error if no dockerfile specified', async () => {
-    await expect(instance.prepare(generate())).rejects.toThrow(/Dockerfile isn't specified/)
+    await assert.rejects(instance.prepare(generate()), (error) =>
+      /Dockerfile isn't specified/.test(error.message)
+    )
   })
 })

package/src/deployment/images/index.js

@@ -1,5 +1 @@
-'use strict'
-
-const { Factory } = require('./factory')
-
-exports.Factory = Factory
+export { Factory } from './factory.js'

package/src/deployment/images/mono.Dockerfile

@@ -0,0 +1,11 @@
+FROM {{build.image}}
+
+# the one instruction that touches the filesystem, and linked: the base is not pulled to lay
+# the sources over it, and a push carries this layer and a manifest, not the base's layers.
+# Owned by root like the dependencies beside them: a linked layer knows no user by name,
+# and the runtime reads its sources, it does not write them
+COPY --link . /composition
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa mono * --map {{map.file}}

package/src/deployment/images/mono.js

@@ -0,0 +1,15 @@
+import { join } from 'node:path'
+
+import { Bundle } from './bundle.js'
+
+export class Mono extends Bundle {
+  dockerfile = join(import.meta.dirname, 'mono.Dockerfile')
+
+  get name() {
+    return 'mono'
+  }
+
+  conflict() {
+    return 'Mono deployment requires different base images for its components. Specify base image for the composition in the context.'
+  }
+}

package/src/deployment/images/publish.js

@@ -0,0 +1,91 @@
+import { mkdtemp, writeFile } from 'node:fs/promises'
+import { readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { definition } from '@toa.io/norm'
+
+import { run } from '../../process.js'
+import { Service } from './service.js'
+
+/**
+ * Builds and pushes the image an extension ships for its service, so that an application
+ * with `registry.services: published` takes it instead of building one.
+ *
+ * The image is built from the package as npm publishes it, not from the workspace: what
+ * `files` leaves out is not in the tarball, and an application builds from the tarball.
+ *
+ * @param {string} workspace path to the extension's directory in this repository
+ * @param {string} runtime the runtime version, which is the tag and the base image
+ * @param {string[]} platforms
+ * @returns {Promise<string>} the reference pushed
+ */
+export async function publish(workspace, runtime, platforms) {
+  const directory = resolve(workspace)
+  const manifest = read(join(directory, 'package.json'))
+
+  const { image } = (await definition(manifest.name)).module
+
+  if (image === undefined)
+    throw new Error(
+      `'${manifest.name}' publishes no service image: it exports no 'image'`
+    )
+
+  const root = await mkdtemp(join(tmpdir(), 'toa-publish'))
+
+  await writeFile(join(root, 'package.json'), '{"private":true}')
+  await run('npm', [
+    'i',
+    '--prefix',
+    root,
+    '--omit=dev',
+    `${manifest.name}@${manifest.version}`
+  ])
+
+  const path = join(root, 'node_modules', manifest.name)
+
+  const service = new Service(SCOPE, { version: runtime }, {}, path, {
+    ...name(image),
+    version: manifest.version
+  })
+
+  service.reference = `${image}:${runtime}`
+
+  await service.prepare(await mkdtemp(join(tmpdir(), 'toa-images')))
+
+  await run('docker', [
+    'buildx',
+    'build',
+    '--platform',
+    platforms.join(','),
+    '--tag',
+    service.reference,
+    '--provenance=false',
+    '--push',
+    service.context
+  ])
+
+  return service.reference
+}
+
+/** The repository's last segment is what `Service` names an image, so it reads back. */
+function name(image) {
+  const match = /\/extension-([^-/]+)-([^/]+)$/.exec(image)
+
+  if (match === null)
+    throw new Error(`'${image}' is not named 'extension-<group>-<name>'`)
+
+  return { group: match[1], name: match[2] }
+}
+
+const read = (path) => JSON.parse(readFileSync(path, 'utf8'))
+
+// the reference is assigned rather than derived from a scope
+const SCOPE = ''
+
+const PLATFORMS = 'linux/amd64,linux/arm64'
+
+if (import.meta.main) {
+  const [workspace, runtime, platforms = PLATFORMS] = process.argv.slice(2)
+
+  console.log(await publish(workspace, runtime, platforms.split(',')))
+}

package/src/deployment/images/runtime-base.test.js

@@ -0,0 +1,97 @@
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { join } from 'node:path'
+import { mkdtemp, readFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+
+import { Image, RUNTIME_IMAGE } from './image.js'
+
+const dependenciesDockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
+const serviceDockerfile = join(import.meta.dirname, 'service.Dockerfile')
+
+class TestImage extends Image {
+  dockerfile
+
+  #name
+  #base
+
+  constructor(runtime, registry, dockerfile, name = 'test', base) {
+    super('acme', runtime, registry)
+
+    this.dockerfile = dockerfile
+    this.#name = name
+    this.#base = base
+  }
+
+  get name() {
+    return this.#name
+  }
+
+  get version() {
+    return 'abcdef12'
+  }
+
+  get base() {
+    return this.#base
+  }
+}
+
+describe('runtime base image', () => {
+  /** @type {string} */
+  let root
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), 'toa-runtime-base-test'))
+  })
+
+  it('should default build.image to version-pinned GHCR runtime image', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(runtime, {}, dependenciesDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.232`))
+    assert.doesNotMatch(dockerfile, /npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should not install runtime in service Dockerfile template', async () => {
+    const runtime = { version: '1.0.0-alpha.99' }
+    const image = new TestImage(runtime, {}, serviceDockerfile, 'service-test')
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes(`FROM ${RUNTIME_IMAGE}:1.0.0-alpha.99`))
+    assert.doesNotMatch(dockerfile, /npm i -g @toa\.io\/runtime/)
+  })
+
+  it('should allow registry.build.image override', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const registry = { build: { image: 'node:24.14.0-alpine3.22' } }
+    const image = new TestImage(runtime, registry, dependenciesDockerfile)
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM node:24.14.0-alpine3.22'))
+    assert.ok(!dockerfile.includes(RUNTIME_IMAGE))
+  })
+
+  it('should allow composition.image override via base', async () => {
+    const runtime = { version: '1.0.0-alpha.232' }
+    const image = new TestImage(
+      runtime,
+      {},
+      dependenciesDockerfile,
+      'mono',
+      'custom.example/base:1'
+    )
+
+    const path = await image.prepare(root)
+    const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM custom.example/base:1'))
+  })
+})

package/src/deployment/images/service.Dockerfile

@@ -1,14 +1,26 @@
-FROM node:20.9.0-alpine3.18
+FROM {{build.image}}
 
 ENV NODE_ENV=production
 RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
 RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-RUN npm i -g @toa.io/runtime@{{runtime.version}} --omit=dev
 
 WORKDIR /service
 COPY --chown=node:node . /service
 
-RUN npm i --omit=dev
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  npm i --omit=dev
 
-USER node
-CMD toa serve .
+# a component of the extension's own declares what it imports, and it is installed beside the
+# component rather than beside the extension: an application that runs none of them, and the
+# runtime image every application is built on, carry nothing for them
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  for entry in components/*; do if grep -qs '"dependencies"' "$entry/package.json"; then (cd $entry && npm i --omit=dev); fi; done
+
+# and beside Toa as well, because what an extension imports on a component's behalf — a storage
+# provider's SDK — is imported from where the extension is, which is `/toa` and not here
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  if [ -s .packages ]; then npm i --prefix /toa --omit=dev $(cat .packages); fi
+
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
+CMD toa serve . --map {{map.file}}