@toa.io/operations 0.1.1-dev.3 → 0.2.0-alpha.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/operations",
-  "version": "0.1.1-dev.3",
+  "version": "0.2.0-alpha.4+5884c85",
   "description": "Toa Deployment",
   "homepage": "https://toa.io",
   "author": {
@@ -26,7 +26,8 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
+    "@toa.io/gears": "0.2.0-alpha.4+5884c85",
     "execa": "5.1.1"
   },
-  "gitHead": "8f1e4d0fd8defeb7cb59ae0e8cc291c6e7623abc"
+  "gitHead": "5884c8590499e85e701c41e54b9d20a47e8a4ec5"
 }

package/src/deployment/assets/Dockerfile

@@ -0,0 +1,10 @@
+# TODO: bridge specifics
+
+FROM node:alpine
+
+RUN npm i -g @toa.io/runtime@{{version}}
+
+WORKDIR /composition
+ADD . .
+
+CMD toa compose *

package/src/deployment/assets/chart/Chart.yaml

@@ -0,0 +1,7 @@
+# This file just helps with IDE autocompletion and helm debugging. Actually dynamically replaced by ../chart.js.
+apiVersion: v2
+name: toa-application
+description: Toa Application Chart
+type: application
+version: 0.0.0
+appVersion: "0.0.0"

package/src/deployment/assets/chart/templates/deployment.yaml

@@ -0,0 +1,23 @@
+{{- range .Values.compositions }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: composition-{{ required "deployment name is required" .name }}
+spec:
+  replicas: {{ .replicas | default 2 }}
+  selector:
+    matchLabels:
+      toa.io/composition: {{ .name }}
+  template:
+    metadata:
+      labels:
+        toa.io/composition: {{ .name }}
+        {{- range .components }}
+        {{ . }}: "1"
+        {{- end }}
+    spec:
+      containers:
+        - name: {{ .name }}
+          image: {{ .image }}
+---
+{{- end }}

package/src/deployment/assets/chart/templates/service.yaml

@@ -0,0 +1,16 @@
+{{- range .Values.components }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ . }}
+spec:
+  type: ClusterIP
+  selector:
+    {{ . }}: "1"
+  ports:
+    - name: http-binding
+      protocol: TCP
+      port: 3000
+      targetPort: 3000
+---
+{{- end }}

package/src/deployment/assets/chart/values.yaml

@@ -0,0 +1,18 @@
+# This file just helps with IDE autocompletion and helm debugging. Actually dynamically replaced by ../values.js.
+compositions:
+  - name: todos
+    image: localhost:5000/todos:0.0.0
+    replicas: 2
+    components:
+      - todos-tasks
+      - todos-stats
+  - name: messages
+    image: localhost:5000/messages:0.0.0
+    replicas: 2
+    components:
+      - users-users
+# TODO: create services only if sync binding is being used by a component
+components:
+  - todos-tasks
+  - todos-stats
+  - users-users

package/src/deployment/chart.js

@@ -1,15 +1,79 @@
 'use strict'
 
-const chart = (context, dependencies) => {
-  return {
-    apiVersion: 'v2',
-    type: 'application',
-    name: context.name,
-    description: context.description,
-    version: context.version,
-    appVersion: context.version,
-    dependencies: dependencies.map((dependency) => dependency.chart)
+class Chart {
+  name
+
+  #context
+  #compositions
+  #dependencies
+
+  constructor (context, compositions) {
+    this.name = context.name
+
+    this.#context = context
+    this.#compositions = compositions
+    this.#dependencies = Chart.dependencies(context)
+  }
+
+  get declaration () {
+    const { name, description, version } = this.#context
+    const dependencies = this.#dependencies.map((dependency) => dependency.chart)
+
+    return {
+      apiVersion: 'v2',
+      type: 'application',
+      name,
+      description,
+      version,
+      appVersion: version,
+      dependencies
+    }
+  }
+
+  get values () {
+    const result = {}
+
+    result.compositions = this.#deployments()
+    result.components = this.#context.components.map((component) => component.domain + '-' + component.name)
+
+    for (const { chart, values } of this.#dependencies) result[chart.alias || chart.name] = values
+
+    return result
+  }
+
+  #deployments () {
+    return Array.from(this.#compositions).map((composition) => {
+      const { name, components, replicas, image: { tag: image } } = composition
+
+      return {
+        name,
+        components: components.map((component) => component.locator.label),
+        replicas,
+        image
+      }
+    })
+  }
+
+  static dependencies (context) {
+    const map = (map) => {
+      const list = []
+
+      for (const [key, values] of Object.entries(map)) {
+        const dependency = require(key)
+
+        if (dependency.deployments !== undefined) list.push(...dependency.deployments(values))
+      }
+
+      return list
+    }
+
+    const dependencies = []
+
+    if (context.connectors !== undefined) dependencies.push(...map(context.connectors))
+    if (context.extensions !== undefined) dependencies.push(...map(context.extensions))
+
+    return dependencies
   }
 }
 
-exports.chart = chart
+exports.Chart = Chart

package/src/deployment/composition.js

@@ -0,0 +1,17 @@
+'use strict'
+
+class Composition {
+  name
+  components
+  replicas
+  image
+
+  constructor (composition, image) {
+    this.name = composition.name
+    this.components = composition.components
+    this.replicas = composition.replicas
+    this.image = image
+  }
+}
+
+exports.Composition = Composition

package/src/deployment/compositions.js

@@ -0,0 +1,60 @@
+'use strict'
+
+class Compositions {
+  #compositions
+
+  constructor (context, instantiate) {
+    if (context.compositions === undefined) context.compositions = []
+
+    Compositions.#resolve(context)
+    Compositions.#complete(context)
+
+    this.#compositions = context.compositions.map(instantiate)
+  }
+
+  [Symbol.iterator] = () => this.#compositions.values()
+
+  static #resolve (context) {
+    const map = {}
+
+    for (const component of context.components) {
+      const id = component.locator.id
+
+      map[id] = component
+    }
+
+    for (const composition of context.compositions) {
+      composition.components = composition.components.map((id) => {
+        const component = map[id]
+
+        if (component === undefined) {
+          throw new Error(`Unknown component '${id}' within composition '${composition.name}'`)
+        }
+
+        return component
+      })
+    }
+  }
+
+  static #complete (context) {
+    const composed = new Set(context.compositions.map((composition) =>
+      composition.components.map((component) => component.locator.id)
+    ).flat())
+
+    const names = new Set(context.compositions.map((composition) => composition.name))
+
+    for (const component of context.components) {
+      const { id, label } = component.locator
+
+      if (composed.has(id)) continue
+      if (names.has(label)) throw new Error(`Duplicate composition name '${label}'`)
+
+      context.compositions.push({
+        name: label,
+        components: [component]
+      })
+    }
+  }
+}
+
+exports.Compositions = Compositions

package/src/deployment/copy.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const execa = require('execa')
+
+const copy = async (source, target) => await execa('cp', ['-r', source, target])
+
+exports.copy = copy

package/src/deployment/deployment.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { writeFile: write, rm: remove } = require('node:fs/promises')
+const execa = require('execa')
+const { join } = require('node:path')
+const { yaml } = require('@toa.io/gears')
+
+const { directory } = require('./directory')
+const { copy } = require('./copy')
+
+class Deployment {
+  #chart
+  #images
+
+  constructor (chart, images) {
+    this.#chart = chart
+    this.#images = images
+  }
+
+  async export (path) {
+    if (path === undefined) path = await directory.temp('deployment')
+    else await directory(path)
+
+    await this.#dump(path)
+
+    return path
+  }
+
+  async install (options = {}) {
+    await this.#push()
+    return await this.#upgrade(options)
+  }
+
+  async #dump (path) {
+    const chart = yaml.dump(this.#chart.declaration)
+    const values = yaml.dump(this.#chart.values)
+
+    await copy(join(__dirname, 'assets/chart/templates'), join(path, 'templates'))
+
+    await Promise.all([
+      write(join(path, 'Chart.yaml'), chart),
+      write(join(path, 'values.yaml'), values)
+    ])
+  }
+
+  async #push () {
+    for (const image of this.#images) {
+      await image.build()
+      await image.push()
+    }
+  }
+
+  async #upgrade (options) {
+    const path = await this.export()
+    const args = []
+
+    if (options.wait === true) args.push('--wait')
+    if (options.dry === true) args.push('--dry-run')
+
+    const update = execa('helm', ['dependency', 'update', path])
+
+    update.stdout.pipe(process.stdout)
+    await update
+
+    const upgrade = execa('helm', ['upgrade', this.#chart.name, '-i', ...args, path])
+
+    upgrade.stdout.pipe(process.stdout)
+    const output = await upgrade
+
+    await remove(path, { recursive: true })
+
+    return output
+  }
+}
+
+exports.Deployment = Deployment

package/src/deployment/directory.js

@@ -5,17 +5,15 @@ const { join } = require('node:path')
 const { tmpdir } = require('node:os')
 
 const directory = async (path) => {
-  if (path === undefined) {
-    path = await fs.mkdtemp(join(tmpdir(), 'toa-deployment-'))
-  } else {
-    await fs.mkdir(path, { recursive: true })
+  await fs.mkdir(path, { recursive: true })
 
-    const entries = await fs.readdir(path)
+  const entries = await fs.readdir(path)
 
-    if (entries.length > 0) throw new Error('Target directory must be empty')
-  }
+  if (entries.length > 0) throw new Error('Target directory must be empty')
 
   return path
 }
 
+directory.temp = async (type) => await fs.mkdtemp(join(tmpdir(), `toa-${type}-`))
+
 exports.directory = directory

package/src/deployment/factory.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const { Composition } = require('./composition')
+const { Compositions } = require('./compositions')
+const { Image } = require('./image')
+const { Deployment } = require('./deployment')
+const { Chart } = require('./chart')
+
+class Factory {
+  #context
+
+  constructor (context) {
+    this.#context = context
+  }
+
+  deployment () {
+    const compositions = this.#compositions()
+    const images = Array.from(compositions).map((composition) => composition.image)
+    const chart = new Chart(this.#context, compositions)
+
+    return new Deployment(chart, images)
+  }
+
+  #compositions () {
+    return new Compositions(this.#context, (composition) => this.#composition(composition))
+  }
+
+  #composition (composition) {
+    const image = this.#image(composition)
+
+    return new Composition(composition, image)
+  }
+
+  #image = (composition) => new Image(composition, this.#context)
+}
+
+exports.Factory = Factory

package/src/deployment/image.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const { join } = require('node:path')
+const { readFile: read, writeFile: write, rm: remove } = require('node:fs/promises')
+const execa = require('execa')
+
+const { hash } = require('@toa.io/gears')
+const { directory } = require('./directory')
+const { copy } = require('./copy')
+
+class Image {
+  tag
+
+  #composition
+  #registry
+  #runtime
+
+  constructor (composition, context) {
+    this.#composition = composition
+    this.#registry = context.registry
+    this.#runtime = context.runtime
+
+    this.tag = context.registry + '/' + composition.name + ':' + Image.#tag(composition)
+  }
+
+  async build () {
+    const path = await this.#context()
+    const build = execa('docker', ['build', path, '-t', this.tag])
+
+    build.stdout.pipe(process.stdout)
+
+    await build
+
+    await remove(path, { recursive: true })
+  }
+
+  async push () {
+    const push = execa('docker', ['push', this.tag])
+
+    push.stdout.pipe(process.stdout)
+
+    await push
+  }
+
+  async #context () {
+    const path = await directory.temp('composition')
+    const dockerfile = (await read(DOCKERFILE, 'utf-8')).replace('{{version}}', this.#runtime)
+
+    for (const component of this.#composition.components) {
+      await copy(component.path, join(path, component.locator.id))
+    }
+
+    await write(join(path, 'Dockerfile'), dockerfile)
+
+    return path
+  }
+
+  static #tag (composition) {
+    const components = composition.components.map((component) => component.locator.id).join(';')
+
+    return hash(components)
+  }
+}
+
+const DOCKERFILE = join(__dirname, 'assets/Dockerfile')
+
+exports.Image = Image

package/src/deployment/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { Factory } = require('./factory')
+
+exports.Factory = Factory

package/src/index.js

@@ -1,5 +1,3 @@
 'use strict'
 
-const { Deployment } = require('./deployment')
-
-exports.Deployment = Deployment
+exports.deployment = require('./deployment')

package/test/deployment/compositions.test.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const fixtures = require('./copositions.fixtures')
+const { Compositions } = require('../../src/deployment/compositions')
+
+const clone = require('clone-deep')
+
+let context, compositions
+
+beforeEach(() => {
+  context = clone(fixtures.context)
+  compositions = new Compositions(context, fixtures.instantiate)
+})
+
+it('should instantiate', () => {
+  expect(fixtures.instantiate).toHaveBeenCalledTimes(context.compositions.length)
+})
+
+it('should be iterable over instances', () => {
+  expect(Symbol.iterator in compositions).toEqual(true)
+
+  for (const composition of compositions) expect(fixtures.instantiate).toHaveReturnedWith(composition)
+})
+
+it('should complete compositions', () => {
+  expect(context.compositions.length).toBeGreaterThan(fixtures.context.compositions.length)
+
+  const used = new Set(context.compositions.map((composition) => composition.components).flat())
+
+  expect(used.size).toEqual(context.components.length)
+})

package/test/deployment/copositions.fixtures.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const { repeat, random } = require('@toa.io/gears')
+const { generate } = require('randomstring')
+
+const components = repeat(() => ({
+  domain: generate(),
+  name: generate(),
+  locator: {
+    id: generate(),
+    label: generate()
+  },
+  version: random(10) + '.' + random(20) + '.' + random(30)
+}), random(10) + 5)
+
+const context = {
+  registry: generate(),
+  components,
+  compositions: []
+}
+
+let unused = components.length
+
+while (1) {
+  const use = random(3) + 1
+  const index = components.length - unused
+
+  if (use >= unused) break
+
+  context.compositions.push({
+    name: generate(),
+    components: components.slice(index, index + use)
+      .map((component) => component.locator.id)
+  })
+
+  unused -= use
+}
+
+const instantiate = jest.fn(() => generate())
+
+exports.context = context
+exports.instantiate = instantiate

package/test/deployment/image.fixtures.js

@@ -0,0 +1,26 @@
+'use strict'
+
+const { newid, random, repeat } = require('@toa.io/gears')
+const { join } = require('node:path')
+const { generate } = require('randomstring')
+
+const mock = {
+  execa: jest.fn(() => ({ stdout: { pipe: jest.fn() } }))
+}
+
+const composition = {
+  name: generate(),
+  components: repeat(() => ({ locator: { id: newid() } }), random(5) + 5)
+}
+
+const context = {
+  registry: `registry-${newid()}:${random(999) + 5000}`,
+  runtime: `${random(9)}.${random(9)}.${random(20)}`
+}
+
+const DOCKERFILE = join(__dirname, '../src/images/Dockerfile')
+
+exports.mock = mock
+exports.composition = composition
+exports.context = context
+exports.DOCKERFILE = DOCKERFILE

package/test/deployment/image.test.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const fixtures = require('./image.fixtures')
+const mock = fixtures.mock
+
+jest.mock('execa', () => mock.execa)
+
+const { Image } = require('../../src/deployment/image')
+const { hash } = require('@toa.io/gears')
+
+let image
+
+beforeEach(() => {
+  image = new Image(fixtures.composition, fixtures.context)
+})
+
+it('should provide tag', () => {
+  const tag = fixtures.context.registry + '/' + fixtures.composition.name + ':' +
+    hash(fixtures.composition.components.map((component) => component.locator.id).join(';'))
+
+  expect(image.tag).toEqual(tag)
+})

package/src/deployment/dependencies.js

@@ -1,25 +0,0 @@
-'use strict'
-
-const dependencies = (context) => {
-  const dependencies = map(context.connectors)
-
-  if (context.extensions !== undefined) dependencies.push(...map(context.extensions))
-
-  return dependencies
-}
-
-const map = (map) => {
-  const list = []
-
-  for (const [key, values] of Object.entries(map)) {
-    const dependency = require(key)
-
-    if (dependency.deployments !== undefined) {
-      list.push(...dependency.deployments(values))
-    }
-  }
-
-  return list
-}
-
-exports.dependencies = dependencies

package/src/deployment/values.js

@@ -1,7 +0,0 @@
-'use strict'
-
-const values = (context, dependencies) => {
-  return Object.fromEntries(dependencies.map(({ chart, values }) => [chart.alias || chart.name, values]))
-}
-
-exports.values = values

package/src/deployment.js

@@ -1,53 +0,0 @@
-'use strict'
-
-const fs = require('node:fs/promises')
-const execa = require('execa')
-const { join } = require('node:path')
-const { yaml } = require('@toa.io/gears')
-
-const { directory } = require('./deployment/directory')
-const { chart } = require('./deployment/chart')
-const { values } = require('./deployment/values')
-const { dependencies } = require('./deployment/dependencies')
-
-class Deployment {
-  #context
-  #chart
-  #values
-
-  constructor (context) {
-    const deps = dependencies(context)
-
-    this.#context = context
-    this.#chart = chart(context, deps)
-    this.#values = values(context, deps)
-  }
-
-  async export (path) {
-    path = await directory(path)
-
-    await this.#dump(path)
-
-    return path
-  }
-
-  async install () {
-    const path = await this.export()
-
-    await execa('helm', ['dependency', 'update', path])
-    await execa('helm', ['upgrade', this.#context.name, '-i', path])
-    await fs.rm(path, { recursive: true })
-  }
-
-  async #dump (path) {
-    const chart = yaml.dump(this.#chart)
-    const values = yaml.dump(this.#values)
-
-    await Promise.all([
-      fs.writeFile(join(path, 'Chart.yaml'), chart),
-      fs.writeFile(join(path, 'values.yaml'), values)
-    ])
-  }
-}
-
-exports.Deployment = Deployment