@toa.io/operations 1.0.0-alpha.288 → 1.0.0-alpha.289

package/src/deployment/images/dependencies.js

@@ -0,0 +1,141 @@
+import { basename, join } from 'node:path'
+import { existsSync, readFileSync } from 'node:fs'
+import { copyFile, mkdir } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+
+import { Image } from './image.js'
+
+/**
+ * What a bundle's components depend on, installed on the base: the layers that weigh the
+ * most and change the least, so they are an image of their own. Its tag digests everything
+ * the install reads, and nothing else: a change to the sources leaves it where it is, and a
+ * bundle is laid over it without building or moving it again.
+ *
+ * @implements {toa.deployment.images.Image}
+ */
+export class Dependencies extends Image {
+  dockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
+  arguments = true
+
+  /** @type {toa.deployment.images.Bundle} */
+  #owner
+
+  #runtime
+  #registry
+
+  /** @type {string | undefined} */
+  #version
+
+  /**
+   * @param {string} scope
+   * @param {toa.norm.context.Runtime} runtime
+   * @param {toa.norm.context.Registry} registry
+   * @param {toa.deployment.images.Bundle} owner the bundle laid over this image
+   */
+  constructor(scope, runtime, registry, owner) {
+    super(scope, runtime, registry)
+
+    this.#owner = owner
+    this.#runtime = runtime
+    this.#registry = registry
+  }
+
+  /** Same repository as the bundle: a tag apart, not a name apart. */
+  get name() {
+    return this.#owner.name
+  }
+
+  get version() {
+    if (this.#version !== undefined) return this.#version
+
+    const hash = createHash('sha256')
+    const build = this.#registry.build ?? {}
+
+    const inputs = [
+      this.#runtime.version,
+      this.#runtime.registry,
+      this.#runtime.proxy,
+      this.base,
+      build.image,
+      build.run,
+      this.run,
+      ...(build.arguments ?? [])
+    ]
+
+    for (const input of inputs) hash.update(input ?? '').update('\0')
+
+    for (const [label, files] of this.#manifests())
+      for (const file of files)
+        hash.update(label).update(basename(file)).update(readFileSync(file))
+
+    this.#version = hash.digest('hex').slice(0, 8)
+
+    return this.#version
+  }
+
+  digest() {
+    return PREFIX + this.version
+  }
+
+  get base() {
+    if (this.#owner.image !== undefined) return this.#owner.image
+
+    const images = new Set(
+      this.#owner.components.map((component) => component.build?.image)
+    )
+
+    if (images.size > 1) throw new Error(this.#owner.conflict())
+
+    return images.values().next().value
+  }
+
+  get run() {
+    const commands = []
+
+    for (const component of this.#owner.components) {
+      const run = component.build?.run
+
+      if (run !== undefined) commands.push(run)
+    }
+
+    return commands.join('\n')
+  }
+
+  async prepare(root) {
+    const context = await super.prepare(join(root, DIRECTORY))
+
+    for (const [label, files] of this.#manifests()) {
+      const target = join(context, label)
+
+      await mkdir(target, { recursive: true })
+
+      for (const file of files) await copyFile(file, join(target, basename(file)))
+    }
+
+    return context
+  }
+
+  /**
+   * The files the install reads, by component label, in a fixed order.
+   *
+   * @returns {Array<[string, string[]]>}
+   */
+  #manifests() {
+    const manifests = []
+
+    for (const component of this.#owner.components) {
+      const files = MANIFESTS.map((name) => join(component.path, name)).filter(existsSync)
+
+      if (files.length > 0) manifests.push([component.locator.label, files])
+    }
+
+    return manifests.sort(([a], [b]) => a.localeCompare(b))
+  }
+}
+
+const PREFIX = 'deps-'
+
+/** Where the contexts land, apart from the bundles', whose names they share. */
+const DIRECTORY = 'dependencies'
+
+const MANIFESTS = ['package.json', 'package-lock.json']

package/src/deployment/images/dependencies.test.js

@@ -0,0 +1,212 @@
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { join } from 'node:path'
+import { mkdtemp, mkdir, writeFile, readdir, readFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+
+import { Composition } from './composition.js'
+import { Mono } from './mono.js'
+
+/** @type {string} */
+let root
+
+/** @type {toa.norm.context.Runtime} */
+let runtime
+
+/** @type {toa.norm.context.Registry} */
+let registry
+
+/** @type {object} */
+let composition
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'toa-dependencies-test'))
+  runtime = { version: '1.0.0-alpha.288' }
+  registry = { base: 'example.com/reg', platforms: ['linux/amd64'] }
+
+  composition = {
+    name: 'mono',
+    components: [
+      await component('one', { dependencies: { left: '1.0.0' } }),
+      await component('two'),
+      await component('three', { devDependencies: { right: '1.0.0' } }, 'lock')
+    ]
+  }
+})
+
+describe('reference', () => {
+  it('should share the repository and stand a prefix apart', () => {
+    const image = create()
+
+    assert.match(
+      image.dependencies.reference,
+      /^example\.com\/reg\/acme\/composition-mono:deps-[0-9a-f]{8}$/
+    )
+    assert.match(
+      image.reference,
+      /^example\.com\/reg\/acme\/composition-mono:[0-9a-f]{8}$/
+    )
+    assert.strictEqual(image.base, image.dependencies.reference)
+  })
+
+  it('should be the same for the same inputs', () => {
+    assert.strictEqual(create().dependencies.reference, create().dependencies.reference)
+  })
+
+  it('should change with a manifest', async () => {
+    const before = create()
+
+    await writeFile(
+      join(composition.components[0].path, 'package.json'),
+      JSON.stringify({ dependencies: { left: '1.0.1' } })
+    )
+
+    const after = create()
+
+    assert.notStrictEqual(after.dependencies.reference, before.dependencies.reference)
+    assert.notStrictEqual(after.reference, before.reference)
+  })
+
+  it('should change with a lockfile', async () => {
+    const before = create()
+
+    await writeFile(join(composition.components[2].path, 'package-lock.json'), 'other')
+
+    assert.notStrictEqual(create().dependencies.reference, before.dependencies.reference)
+  })
+
+  it('should change with what is run and with the runtime', () => {
+    const before = create()
+
+    composition.components[1].build = { run: 'apk add git' }
+
+    const run = create()
+
+    assert.notStrictEqual(run.dependencies.reference, before.dependencies.reference)
+
+    runtime.version = '1.0.0-alpha.289'
+
+    assert.notStrictEqual(create().dependencies.reference, run.dependencies.reference)
+  })
+
+  it('should change with the registry build settings', () => {
+    const before = create()
+
+    registry.build = { arguments: ['TOKEN'] }
+
+    assert.notStrictEqual(create().dependencies.reference, before.dependencies.reference)
+  })
+
+  it('should not change with the sources', async () => {
+    const before = create()
+
+    await writeFile(
+      join(composition.components[0].path, 'index.js'),
+      'export const changed = true'
+    )
+    composition.components[0].version = 'ffffffff'
+
+    const after = create()
+
+    assert.strictEqual(after.dependencies.reference, before.dependencies.reference)
+    assert.notStrictEqual(after.reference, before.reference)
+  })
+})
+
+describe('prepare', () => {
+  it('should hold the manifests and nothing else', async () => {
+    const image = create()
+    const context = await image.dependencies.prepare(root)
+
+    assert.strictEqual(
+      context,
+      join(root, 'dependencies', `composition-mono.${image.dependencies.version}`)
+    )
+
+    const entries = (await readdir(context, { recursive: true })).sort()
+
+    assert.deepStrictEqual(entries, [
+      '.dockerignore',
+      'Dockerfile',
+      'one',
+      'one/package.json',
+      'three',
+      'three/package-lock.json',
+      'three/package.json'
+    ])
+
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM ghcr.io/toa-io/runtime:1.0.0-alpha.288'))
+    assert.ok(dockerfile.includes('npm i --omit=dev'))
+    assert.ok(dockerfile.includes('WORKDIR /composition'))
+    assert.doesNotMatch(dockerfile, /USER node|CMD /)
+  })
+
+  it('should lay the sources over the dependencies', async () => {
+    const image = create()
+
+    image.dependencies.tag()
+
+    const context = await image.prepare(root)
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+    const lines = dockerfile
+      .split('\n')
+      .filter((line) => line !== '' && !line.startsWith('#'))
+
+    assert.deepStrictEqual(lines, [
+      `FROM ${image.dependencies.reference}`,
+      'COPY --link . /composition',
+      'USER node',
+      'CMD toa compose *'
+    ])
+
+    const entries = (await readdir(context, { recursive: true })).sort()
+
+    assert.ok(entries.includes('one/index.js'))
+    assert.ok(entries.includes('two/package.json')) // declared for the component that had none
+    assert.ok(!entries.some((entry) => entry.includes('node_modules')))
+  })
+
+  it('should run mono the same way', async () => {
+    const image = new Mono('acme', runtime, registry, composition)
+
+    image.tag()
+
+    assert.match(image.dependencies.reference, /\/mono:deps-[0-9a-f]{8}$/)
+
+    const context = await image.prepare(root)
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('CMD toa mono *'))
+  })
+})
+
+/** @returns {Composition} */
+function create() {
+  const image = new Composition('acme', runtime, registry, composition)
+
+  image.tag()
+
+  return image
+}
+
+/**
+ * @param {string} label
+ * @param {object} [manifest]
+ * @param {string} [lock]
+ */
+async function component(label, manifest, lock) {
+  const path = join(root, 'components', label)
+
+  await mkdir(join(path, 'node_modules', 'left'), { recursive: true })
+  await writeFile(join(path, 'index.js'), 'export const changed = false')
+  await writeFile(join(path, 'node_modules', 'left', 'index.js'), '')
+
+  if (manifest !== undefined)
+    await writeFile(join(path, 'package.json'), JSON.stringify(manifest))
+  if (lock !== undefined) await writeFile(join(path, 'package-lock.json'), lock)
+
+  return { path, version: 'abcdef12', locator: { id: `default.${label}`, label } }
+}

package/src/deployment/images/image.js

@@ -14,6 +14,17 @@ export class Image {
   reference
   dockerfile
 
+  /**
+   * The image this one is laid over, built and pushed as an image of its own.
+   * Undefined stands on the base alone, which is what a service does.
+   *
+   * @type {toa.deployment.images.Image | undefined}
+   */
+  dependencies
+
+  /** Whether the build reads `registry.build.arguments`. */
+  arguments = false
+
   #scope
   #registry
   #runtime
@@ -28,18 +39,25 @@ export class Image {
   }
 
   tag() {
+    this.reference = posix.join(
+      this.#registry.base ?? '',
+      this.#scope,
+      `${this.name}:${this.digest()}`
+    )
+  }
+
+  /**
+   * What follows the colon: the runtime and the image's own version, digested.
+   *
+   * @returns {string}
+   */
+  digest() {
     const hash = createHash('sha256')
 
     hash.update(this.#runtime.version)
     hash.update(this.version)
 
-    const tag = hash.digest('hex').slice(0, 8)
-
-    this.reference = posix.join(
-      this.#registry.base ?? '',
-      this.#scope,
-      `${this.name}:${tag}`
-    )
+    return hash.digest('hex').slice(0, 8)
   }
 
   /** @returns {string | undefined} */

package/src/deployment/images/mono.Dockerfile

@@ -1,18 +1,10 @@
 FROM {{build.image}}
 
-{{build.arguments}}
-
-ENV NODE_ENV=production
-RUN if [ "{{runtime.registry}}" != "" ]; then npm set registry {{runtime.registry}}; fi
-RUN if [ "{{runtime.proxy}}" != "" ]; then npm set proxy {{runtime.proxy}}; fi
-
-WORKDIR /composition
-COPY --chown=node:node . /composition
-
-{{build.run}}
-
-RUN --mount=type=cache,target=/root/.npm,sharing=locked \
-  for entry in *; do if grep -qs '"dependencies"' "$entry/package.json"; then (cd $entry && npm i --omit=dev); fi; done
+# the one instruction that touches the filesystem, and linked: the base is not pulled to lay
+# the sources over it, and a push carries this layer and a manifest, not the base's layers.
+# Owned by root like the dependencies beside them: a linked layer knows no user by name,
+# and the runtime reads its sources, it does not write them
+COPY --link . /composition
 
 USER node
 CMD toa mono *

package/src/deployment/images/mono.js

@@ -1,73 +1,15 @@
 import { join } from 'node:path'
-import fs from 'fs-extra'
-import { createHash } from 'node:crypto'
 
-import { Image } from './image.js'
-import { declare } from './format.js'
+import { Bundle } from './bundle.js'
 
-export class Mono extends Image {
+export class Mono extends Bundle {
   dockerfile = join(import.meta.dirname, 'mono.Dockerfile')
 
-  #image
-  #components
-
-  constructor(scope, runtime, registry, composition) {
-    super(scope, runtime, registry)
-
-    this.#image = composition.image
-    this.#components = composition.components
-  }
-
   get name() {
     return 'mono'
   }
 
-  get version() {
-    const hash = createHash('sha256')
-
-    for (const component of this.#components) {
-      hash.update(component.locator.id)
-      hash.update(component.version)
-    }
-
-    return hash.digest('hex').slice(0, 8)
-  }
-
-  get base() {
-    if (this.#image !== undefined) return this.#image
-
-    const images = new Set(this.#components.map((component) => component.build?.image))
-
-    if (images.size > 1)
-      throw new Error(
-        'Mono deployment requires different base images for its components. Specify base image for the composition in the context.'
-      )
-
-    return images.values().next().value
-  }
-
-  get run() {
-    const commands = []
-
-    for (const component of this.#components) {
-      const run = component.build?.run
-
-      if (run !== undefined) commands.push(run)
-    }
-
-    return commands.join('\n')
-  }
-
-  async prepare(root) {
-    const context = await super.prepare(root)
-
-    for (const component of this.#components) {
-      const target = join(context, component.locator.label)
-
-      await fs.copy(component.path, target)
-      await declare(component.path, target, component.locator.label)
-    }
-
-    return context
+  conflict() {
+    return 'Mono deployment requires different base images for its components. Specify base image for the composition in the context.'
   }
 }

package/src/deployment/images/publish.js

@@ -4,8 +4,7 @@ import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
 import { pathToFileURL } from 'node:url'
 
-import { execa } from 'execa'
-
+import { run } from '../../process.js'
 import { Service } from './service.js'
 
 /**
@@ -29,7 +28,9 @@ export async function publish(workspace, runtime, platforms) {
   const { image } = await import(pathToFileURL(join(directory, manifest.main)).href)
 
   if (image === undefined)
-    throw new Error(`'${manifest.name}' publishes no service image: it exports no 'image'`)
+    throw new Error(
+      `'${manifest.name}' publishes no service image: it exports no 'image'`
+    )
 
   const root = await mkdtemp(join(tmpdir(), 'toa-publish'))
 
@@ -80,12 +81,6 @@ function name(image) {
 
 const read = (path) => JSON.parse(readFileSync(path, 'utf8'))
 
-const run = async (cmd, args) => {
-  console.log('toa>', cmd, args.join(' '))
-
-  await execa(cmd, args, { stdio: 'inherit' })
-}
-
 // the reference is assigned rather than derived from a scope
 const SCOPE = ''
 

package/src/deployment/images/runtime-base.test.js

@@ -7,7 +7,7 @@ import { tmpdir } from 'node:os'
 
 import { Image, RUNTIME_IMAGE } from './image.js'
 
-const compositionDockerfile = join(import.meta.dirname, 'composition.Dockerfile')
+const dependenciesDockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
 const serviceDockerfile = join(import.meta.dirname, 'service.Dockerfile')
 
 class TestImage extends Image {
@@ -47,7 +47,7 @@ describe('runtime base image', () => {
 
   it('should default build.image to version-pinned GHCR runtime image', async () => {
     const runtime = { version: '1.0.0-alpha.232' }
-    const image = new TestImage(runtime, {}, compositionDockerfile)
+    const image = new TestImage(runtime, {}, dependenciesDockerfile)
 
     const path = await image.prepare(root)
     const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
@@ -70,7 +70,7 @@ describe('runtime base image', () => {
   it('should allow registry.build.image override', async () => {
     const runtime = { version: '1.0.0-alpha.232' }
     const registry = { build: { image: 'node:24.14.0-alpine3.22' } }
-    const image = new TestImage(runtime, registry, compositionDockerfile)
+    const image = new TestImage(runtime, registry, dependenciesDockerfile)
 
     const path = await image.prepare(root)
     const dockerfile = await readFile(join(path, 'Dockerfile'), 'utf8')
@@ -84,7 +84,7 @@ describe('runtime base image', () => {
     const image = new TestImage(
       runtime,
       {},
-      compositionDockerfile,
+      dependenciesDockerfile,
       'mono',
       'custom.example/base:1'
     )

package/src/deployment/images/service.Dockerfile

@@ -10,5 +10,6 @@ COPY --chown=node:node . /service
 RUN --mount=type=cache,target=/root/.npm,sharing=locked \
   npm i --omit=dev
 
-USER node
+# no USER: the runtime drops to `node` itself, and only a process that started as root can
+# close its environment under /proc — see runtime/runtime/bin/toa
 CMD toa serve .

package/src/deployment/images/service.js

@@ -2,7 +2,7 @@ import { createRequire } from 'node:module'
 import { join, dirname } from 'node:path'
 
 import { Image } from './image.js'
-import fs from 'fs-extra'
+import { cp } from 'node:fs/promises'
 
 // a service is named the way a package is, and its directory is where it lives
 const require = createRequire(import.meta.url)
@@ -59,7 +59,7 @@ export class Service extends Image {
   async prepare(root) {
     const context = await super.prepare(root)
 
-    await fs.copy(this.#path, context)
+    await cp(this.#path, context, { recursive: true })
 
     return context
   }