@toa.io/operations 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/deployment/images/dependencies.js

@@ -0,0 +1,171 @@
+import { basename, join } from 'node:path'
+import { existsSync, readFileSync } from 'node:fs'
+import { copyFile, mkdir, writeFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+
+import { Image } from './image.js'
+
+/**
+ * What a bundle's components depend on, installed on the base: the layers that weigh the
+ * most and change the least, so they are an image of their own. Its tag digests everything
+ * the install reads, and nothing else: a change to the sources leaves it where it is, and a
+ * bundle is laid over it without building or moving it again.
+ *
+ * @implements {toa.deployment.images.Image}
+ */
+export class Dependencies extends Image {
+  dockerfile = join(import.meta.dirname, 'dependencies.Dockerfile')
+  arguments = true
+
+  /** @type {toa.deployment.images.Bundle} */
+  #owner
+
+  #runtime
+  #registry
+
+  /** @type {string | undefined} */
+  #version
+
+  /**
+   * @param {string} scope
+   * @param {toa.norm.context.Runtime} runtime
+   * @param {toa.norm.context.Registry} registry
+   * @param {toa.deployment.images.Bundle} owner the bundle laid over this image
+   */
+  constructor(scope, runtime, registry, owner) {
+    super(scope, runtime, registry)
+
+    this.#owner = owner
+    this.#runtime = runtime
+    this.#registry = registry
+  }
+
+  /** Same repository as the bundle: a tag apart, not a name apart. */
+  get name() {
+    return this.#owner.name
+  }
+
+  get version() {
+    if (this.#version !== undefined) return this.#version
+
+    const hash = createHash('sha256')
+    const build = this.#registry.build ?? {}
+
+    const inputs = [
+      this.#runtime.version,
+      this.#runtime.registry,
+      this.#runtime.proxy,
+      this.base,
+      build.image,
+      build.run,
+      this.run,
+      ...(build.arguments ?? [])
+    ]
+
+    for (const input of inputs) hash.update(input ?? '').update('\0')
+
+    for (const [label, files] of this.#manifests())
+      for (const file of files)
+        hash.update(label).update(basename(file)).update(readFileSync(file))
+
+    hash.update(this.#packages().join('\n'))
+
+    this.#version = hash.digest('hex').slice(0, 8)
+
+    return this.#version
+  }
+
+  digest() {
+    return PREFIX + this.version
+  }
+
+  get base() {
+    if (this.#owner.image !== undefined) return this.#owner.image
+
+    const images = new Set(
+      this.#owner.components.map((component) => component.build?.image)
+    )
+
+    if (images.size > 1) throw new Error(this.#owner.conflict())
+
+    return images.values().next().value
+  }
+
+  get run() {
+    const commands = []
+
+    for (const component of this.#owner.components) {
+      const run = component.build?.run
+
+      if (run !== undefined) commands.push(run)
+    }
+
+    return commands.join('\n')
+  }
+
+  async prepare(root) {
+    const context = await super.prepare(join(root, DIRECTORY))
+
+    for (const [label, files] of this.#manifests()) {
+      const target = join(context, label)
+
+      await mkdir(target, { recursive: true })
+
+      for (const file of files) await copyFile(file, join(target, basename(file)))
+    }
+
+    const packages = this.#packages()
+
+    // written whether or not there are any, so the file is one thing the Dockerfile reads
+    await writeFile(join(context, PACKAGES), packages.join('\n'))
+
+    return context
+  }
+
+  /**
+   * What the extensions install for what this workload runs, as `name@version`: what its
+   * components declare, and what the services it hosts bring. A component states its own
+   * dependencies in its manifest; this is what the packages that read its declaration need,
+   * which its manifest has no way to say.
+   *
+   * @returns {string[]}
+   */
+  #packages() {
+    /** @type {Record<string, string>} */
+    const packages = { ...this.#owner.packages }
+
+    for (const component of this.#owner.components)
+      Object.assign(packages, component.packages)
+
+    return Object.entries(packages)
+      .map(([name, version]) => `${name}@${version}`)
+      .sort()
+  }
+
+  /**
+   * The files the install reads, by component label, in a fixed order.
+   *
+   * @returns {Array<[string, string[]]>}
+   */
+  #manifests() {
+    const manifests = []
+
+    for (const component of this.#owner.components) {
+      const files = MANIFESTS.map((name) => join(component.path, name)).filter(existsSync)
+
+      if (files.length > 0) manifests.push([component.locator.label, files])
+    }
+
+    return manifests.sort(([a], [b]) => a.localeCompare(b))
+  }
+}
+
+const PREFIX = 'deps-'
+
+/** Where the contexts land, apart from the bundles', whose names they share. */
+const DIRECTORY = 'dependencies'
+
+const MANIFESTS = ['package.json', 'package-lock.json']
+
+/** Where the install reads what the extensions need, one `name@version` per line. */
+const PACKAGES = '.packages'

package/src/deployment/images/dependencies.test.js

@@ -0,0 +1,268 @@
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { join } from 'node:path'
+import { mkdtemp, mkdir, writeFile, readdir, readFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+
+import { Composition } from './composition.js'
+import { Mono } from './mono.js'
+
+/** @type {string} */
+let root
+
+/** @type {toa.norm.context.Runtime} */
+let runtime
+
+/** @type {toa.norm.context.Registry} */
+let registry
+
+/** @type {object} */
+let composition
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'toa-dependencies-test'))
+  runtime = { version: '1.0.0-alpha.288' }
+  registry = { base: 'example.com/reg', platforms: ['linux/amd64'] }
+
+  composition = {
+    name: 'mono',
+    components: [
+      await component('one', { dependencies: { left: '1.0.0' } }),
+      await component('two'),
+      await component('three', { devDependencies: { right: '1.0.0' } }, 'lock')
+    ]
+  }
+})
+
+describe('reference', () => {
+  it('should share the repository and stand a prefix apart', () => {
+    const image = create()
+
+    assert.match(
+      image.dependencies.reference,
+      /^example\.com\/reg\/acme\/composition-mono:deps-[0-9a-f]{8}$/
+    )
+    assert.match(
+      image.reference,
+      /^example\.com\/reg\/acme\/composition-mono:[0-9a-f]{8}$/
+    )
+    assert.strictEqual(image.base, image.dependencies.reference)
+  })
+
+  it('should be the same for the same inputs', () => {
+    assert.strictEqual(create().dependencies.reference, create().dependencies.reference)
+  })
+
+  it('should change with a manifest', async () => {
+    const before = create()
+
+    await writeFile(
+      join(composition.components[0].path, 'package.json'),
+      JSON.stringify({ dependencies: { left: '1.0.1' } })
+    )
+
+    const after = create()
+
+    assert.notStrictEqual(after.dependencies.reference, before.dependencies.reference)
+    assert.notStrictEqual(after.reference, before.reference)
+  })
+
+  it('should change with a lockfile', async () => {
+    const before = create()
+
+    await writeFile(join(composition.components[2].path, 'package-lock.json'), 'other')
+
+    assert.notStrictEqual(create().dependencies.reference, before.dependencies.reference)
+  })
+
+  it('should change with what is run and with the runtime', () => {
+    const before = create()
+
+    composition.components[1].build = { run: 'apk add git' }
+
+    const run = create()
+
+    assert.notStrictEqual(run.dependencies.reference, before.dependencies.reference)
+
+    runtime.version = '1.0.0-alpha.289'
+
+    assert.notStrictEqual(create().dependencies.reference, run.dependencies.reference)
+  })
+
+  it('should change with what an extension installs', () => {
+    const before = create()
+
+    composition.components[1].packages = { cloudinary: '2.11.0' }
+
+    const component = create()
+
+    assert.notStrictEqual(component.dependencies.reference, before.dependencies.reference)
+
+    composition.packages = { 'lru-cache': '11.5.2' }
+
+    assert.notStrictEqual(create().dependencies.reference, component.dependencies.reference)
+  })
+
+  it('should change with the registry build settings', () => {
+    const before = create()
+
+    registry.build = { arguments: ['TOKEN'] }
+
+    assert.notStrictEqual(create().dependencies.reference, before.dependencies.reference)
+  })
+
+  it('should not change with the sources', async () => {
+    const before = create()
+
+    await writeFile(
+      join(composition.components[0].path, 'index.js'),
+      'export const changed = true'
+    )
+    composition.components[0].version = 'ffffffff'
+
+    const after = create()
+
+    assert.strictEqual(after.dependencies.reference, before.dependencies.reference)
+    assert.notStrictEqual(after.reference, before.reference)
+  })
+})
+
+describe('prepare', () => {
+  it('should hold the manifests and nothing else', async () => {
+    const image = create()
+    const context = await image.dependencies.prepare(root)
+
+    assert.strictEqual(
+      context,
+      join(root, 'dependencies', `composition-mono.${image.dependencies.version}`)
+    )
+
+    const entries = (await readdir(context, { recursive: true })).sort()
+
+    assert.deepStrictEqual(entries, [
+      '.dockerignore',
+      '.packages',
+      'Dockerfile',
+      'one',
+      'one/package.json',
+      'three',
+      'three/package-lock.json',
+      'three/package.json'
+    ])
+
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('FROM ghcr.io/toa-io/runtime:1.0.0-alpha.288'))
+    assert.ok(dockerfile.includes('npm i --omit=dev'))
+    assert.ok(dockerfile.includes('WORKDIR /composition'))
+    assert.doesNotMatch(dockerfile, /USER node|CMD /)
+  })
+
+  it('should hold what the extensions install, deduplicated and ordered', async () => {
+    composition.components[0].packages = { cloudinary: '2.11.0' }
+    composition.components[1].packages = {
+      cloudinary: '2.11.0',
+      '@aws-sdk/client-s3': '3.1125.0'
+    }
+
+    const image = create()
+    const context = await image.dependencies.prepare(root)
+
+    assert.strictEqual(
+      await readFile(join(context, '.packages'), 'utf8'),
+      '@aws-sdk/client-s3@3.1125.0\ncloudinary@2.11.0'
+    )
+
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('npm i --prefix /toa --omit=dev $(cat .packages)'))
+  })
+
+  it('should hold what a service the composition runs brings', async () => {
+    composition.packages = { 'lru-cache': '11.5.2' }
+    composition.components[0].packages = { cloudinary: '2.11.0' }
+
+    const context = await create().dependencies.prepare(root)
+
+    assert.strictEqual(
+      await readFile(join(context, '.packages'), 'utf8'),
+      'cloudinary@2.11.0\nlru-cache@11.5.2'
+    )
+  })
+
+  it('should hold an empty list where nothing is declared', async () => {
+    const context = await create().dependencies.prepare(root)
+
+    assert.strictEqual(await readFile(join(context, '.packages'), 'utf8'), '')
+  })
+
+  it('should lay the sources over the dependencies', async () => {
+    const image = create()
+
+    image.dependencies.tag()
+
+    const context = await image.prepare(root)
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+    const lines = dockerfile
+      .split('\n')
+      .filter((line) => line !== '' && !line.startsWith('#'))
+
+    // no USER: the runtime starts as root and drops to `node` itself, and nothing else
+    // touches the filesystem, which is what lets the layer be laid over a base never pulled
+    assert.deepStrictEqual(lines, [
+      `FROM ${image.dependencies.reference}`,
+      'COPY --link . /composition',
+      'CMD toa compose * --map /etc/toa/.map.json'
+    ])
+
+    const entries = (await readdir(context, { recursive: true })).sort()
+
+    assert.ok(entries.includes('one/index.js'))
+    assert.ok(entries.includes('two/package.json')) // declared for the component that had none
+    assert.ok(entries.includes('one/manifest.toa.json')) // what the process reads instead of normalizing
+    assert.ok(!entries.some((entry) => entry.includes('node_modules')))
+  })
+
+  it('should run mono the same way', async () => {
+    const image = new Mono('acme', runtime, registry, composition)
+
+    image.tag()
+
+    assert.match(image.dependencies.reference, /\/mono:deps-[0-9a-f]{8}$/)
+
+    const context = await image.prepare(root)
+    const dockerfile = await readFile(join(context, 'Dockerfile'), 'utf8')
+
+    assert.ok(dockerfile.includes('CMD toa mono *'))
+    assert.doesNotMatch(dockerfile, /^USER /m)
+  })
+})
+
+/** @returns {Composition} */
+function create() {
+  const image = new Composition('acme', runtime, registry, composition)
+
+  image.tag()
+
+  return image
+}
+
+/**
+ * @param {string} label
+ * @param {object} [manifest]
+ * @param {string} [lock]
+ */
+async function component(label, manifest, lock) {
+  const path = join(root, 'components', label)
+
+  await mkdir(join(path, 'node_modules', 'left'), { recursive: true })
+  await writeFile(join(path, 'index.js'), 'export const changed = false')
+  await writeFile(join(path, 'node_modules', 'left', 'index.js'), '')
+
+  if (manifest !== undefined)
+    await writeFile(join(path, 'package.json'), JSON.stringify(manifest))
+  if (lock !== undefined) await writeFile(join(path, 'package-lock.json'), lock)
+
+  return { path, version: 'abcdef12', locator: { id: `default.${label}`, label } }
+}

package/src/deployment/images/factory.js

@@ -1,12 +1,8 @@
-'use strict'
+import { Composition } from './composition.js'
+import { Service } from './service.js'
+import { Mono } from './mono.js'
 
-const { Composition } = require('./composition')
-const { Service } = require('./service')
-
-/**
- * @implements {toa.deployment.images.Factory}
- */
-class Factory {
+export class Factory {
   /** @type {string} */
   #scope
 
@@ -21,7 +17,7 @@ class Factory {
    * @param {toa.norm.context.Runtime} runtime
    * @param {toa.norm.context.Registry} registry
    */
-  constructor (scope, runtime, registry) {
+  constructor(scope, runtime, registry) {
     this.#scope = scope
     this.#runtime = runtime
     this.#registry = registry
@@ -30,8 +26,13 @@ class Factory {
   /**
    * @returns {Composition}
    */
-  composition (composition) {
-    const instance = new Composition(this.#scope, this.#runtime, this.#registry, composition)
+  composition(composition) {
+    const instance = new Composition(
+      this.#scope,
+      this.#runtime,
+      this.#registry,
+      composition
+    )
 
     instance.tag()
 
@@ -41,13 +42,28 @@ class Factory {
   /**
    * @returns {Service}
    */
-  service (path, service) {
-    const instance = new Service(this.#scope, this.#runtime, this.#registry, path, service)
+  service(path, service) {
+    const instance = new Service(
+      this.#scope,
+      this.#runtime,
+      this.#registry,
+      path,
+      service
+    )
 
     instance.tag()
 
     return instance
   }
-}
 
-exports.Factory = Factory
+  /**
+   * @returns {Mono}
+   */
+  mono(composition) {
+    const instance = new Mono(this.#scope, this.#runtime, this.#registry, composition)
+
+    instance.tag()
+
+    return instance
+  }
+}

package/src/deployment/images/format.js

@@ -0,0 +1,66 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { writeFile } from 'node:fs/promises'
+import { dirname, join, parse } from 'node:path'
+
+import { NORMALIZED, plain } from '@toa.io/norm'
+
+/**
+ * A component is copied into the image on its own, away from the package that
+ * declared what its files are. Node reads a `.js` as CommonJS unless a manifest
+ * beside it says otherwise, so the one the component was written under is
+ * restated where it lands.
+ *
+ * A component that ships its own manifest already says so, and is left alone.
+ *
+ * @param {string} source the component in the workspace
+ * @param {string} target where it was copied
+ * @param {string} name
+ */
+export async function declare(source, target, name) {
+  if (existsSync(join(target, MANIFEST))) return
+
+  await writeFile(
+    join(target, MANIFEST),
+    JSON.stringify({ name, private: true, type: format(source) }, null, 2) + '\n'
+  )
+}
+
+/**
+ * The manifest as this build read it, written beside the component so that the process which
+ * runs it does not read it again. See `plain` in `@toa.io/norm` for what is left out and why
+ * a build's answer is the one to trust.
+ *
+ * @param {toa.norm.Component} component
+ * @param {string} target where it was copied
+ */
+export async function normalized(component, target) {
+  await writeFile(join(target, NORMALIZED), JSON.stringify(plain(component)))
+}
+
+/**
+ * The module format a directory's files are read as, which the nearest manifest
+ * above it decides.
+ *
+ * @param {string} directory
+ * @returns {'module' | 'commonjs'}
+ */
+export function format(directory) {
+  const { root } = parse(directory)
+
+  let current = directory
+
+  while (current !== root) {
+    const manifest = join(current, MANIFEST)
+
+    if (existsSync(manifest))
+      return JSON.parse(readFileSync(manifest, 'utf8')).type === 'module'
+        ? 'module'
+        : 'commonjs'
+
+    current = dirname(current)
+  }
+
+  return 'commonjs'
+}
+
+const MANIFEST = 'package.json'