@toa.io/operations 1.0.0-alpha.0 → 1.0.0-alpha.102

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/operations",
-  "version": "1.0.0-alpha.0",
+  "version": "1.0.0-alpha.102",
   "description": "Toa Deployment",
   "homepage": "https://toa.io",
   "author": {
@@ -16,21 +16,23 @@
     "url": "https://github.com/toa-io/toa/issues"
   },
   "engines": {
-    "node": ">= 18.0.0"
+    "node": ">= 20.0.0"
   },
   "publishConfig": {
     "access": "public"
   },
   "main": "src/index.js",
-  "types": "types/index.d.ts",
+  "types": "types/index.ts",
   "scripts": {
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/filesystem": "1.0.0-alpha.0",
-    "@toa.io/generic": "1.0.0-alpha.0",
-    "@toa.io/yaml": "1.0.0-alpha.0",
-    "execa": "5.1.1"
+    "@toa.io/filesystem": "1.0.0-alpha.93",
+    "@toa.io/generic": "1.0.0-alpha.93",
+    "@toa.io/norm": "1.0.0-alpha.102",
+    "@toa.io/yaml": "1.0.0-alpha.93",
+    "execa": "5.1.1",
+    "fs-extra": "11.1.1"
   },
-  "gitHead": "06c64546f6292cc07c52f74b31415101037f7616"
+  "gitHead": "ce5b6e22c1adbec4cc5d4dc5dd8f6bafc41d93d0"
 }

package/src/deployment/.deployment/.describe/compositions.js

@@ -1,10 +1,12 @@
 'use strict'
 
 const { addVariables } = require('./variables')
+const { addMounts } = require('./mounts')
 
-function compositions (compositions, variables) {
+function compositions (compositions, dependency) {
   for (const composition of compositions) {
-    addVariables(composition, variables)
+    addVariables(composition, dependency.variables)
+    addMounts(composition, dependency.mounts)
   }
 }
 

package/src/deployment/.deployment/.describe/mounts.js

@@ -0,0 +1,24 @@
+'use strict'
+
+function addMounts (composition, mounts) {
+  if (mounts === undefined)
+    return
+
+  const used = new Set()
+
+  for (const [key, mount] of Object.entries(mounts)) {
+    if (key !== 'global' && !composition.components?.includes(key))
+      continue
+
+    for (const { name, path, claim } of mount) {
+      if (used.has(name))
+        continue
+
+      composition.mounts ??= []
+      composition.mounts.push({ name, path, claim })
+      used.add(name)
+    }
+  }
+}
+
+exports.addMounts = addMounts

package/src/deployment/.deployment/.describe/variables.js

@@ -1,17 +1,18 @@
 'use strict'
 
-function addVariables (deployment, variables) {
+function addVariables (composition, variables) {
   const used = new Set()
 
-  deployment.variables ??= []
+  composition.variables ??= []
 
   for (const [key, set] of Object.entries(variables)) {
-    if (key !== 'global' && !deployment.components?.includes(key)) continue
+    if (key !== 'global' && !composition.components?.includes(key))
+      continue
 
     for (const variable of set) {
       if (used.has(variable.name)) continue
 
-      deployment.variables.push(variable)
+      composition.variables.push(variable)
       used.add(variable.name)
     }
   }

package/src/deployment/.deployment/describe.js

@@ -1,24 +1,33 @@
 'use strict'
 
-const get = require('./.describe')
+const desc = require('./.describe')
 
 const describe = (context, compositions, dependency) => {
   const { services } = dependency
 
   dependency.variables.global ??= []
-  dependency.variables.global.unshift({ name: 'TOA_ENV', value: context.environment })
 
-  const components = get.components(compositions)
+  dependency.variables.global.unshift(
+    {
+      name: 'TOA_CONTEXT',
+      value: context.name
+    }, {
+      name: 'TOA_ENV',
+      value: context.environment
+    }
+  )
+
+  const components = desc.components(compositions)
   const credentials = context.registry?.credentials
 
-  get.compositions(compositions, dependency.variables, context.environment)
-  get.services(services, dependency.variables)
+  desc.compositions(compositions, dependency)
+  desc.services(services, dependency.variables)
 
   return {
     compositions,
     components,
     services,
-    credentials,
+    credentials
   }
 }
 

package/src/deployment/.deployment/merge.js

@@ -13,14 +13,17 @@ const merge = (dependencies) => {
   /** @type {toa.deployment.dependency.Variables} */
   const variables = {}
 
+  const mounts = {}
+
   for (const dependency of dependencies) {
     if (dependency.references !== undefined) references.push(...dependency.references)
     if (dependency.services !== undefined) services.push(...dependency.services)
     if (dependency.proxies !== undefined) proxies.push(...dependency.proxies)
     if (dependency.variables !== undefined) append(variables, dependency.variables)
+    if (dependency.mounts !== undefined) append(mounts, dependency.mounts)
   }
 
-  return { references, services, proxies, variables }
+  return { references, services, proxies, variables, mounts }
 }
 
 const append = (merged, variables) => {

package/src/deployment/chart/templates/compositions.yaml

@@ -25,9 +25,24 @@ spec:
             {{- include "env.var" . | indent 12 }}
             {{- end }}
           {{- end }}
+          {{- if .mounts }}
+          volumeMounts:
+            {{- range .mounts }}
+            - name: {{ .name }}
+              mountPath: {{ .path }}
+            {{- end }}
+          {{- end }}
       {{- if $.Values.credentials }}
       imagePullSecrets:
         - name: {{ $.Values.credentials }}
       {{- end }}
+      {{- if .mounts }}
+      volumes:
+        {{- range .mounts }}
+        - name: {{ .name }}
+          persistentVolumeClaim:
+            claimName: {{ .claim }}
+        {{- end }}
+      {{- end }}
 ---
 {{- end }}

package/src/deployment/chart/templates/services.yaml

@@ -22,6 +22,16 @@ spec:
             {{- include "env.var" . | indent 12 }}
             {{- end }}
           {{- end }}
+          {{- if .probe }}
+          readinessProbe:
+            httpGet:
+              path: {{ .probe.path }}
+              port: {{ .probe.port }}
+            {{- if .probe.delay }}
+            initialDelaySeconds: {{ .probe.delay }}
+            {{- end }}
+          {{- end }}
+{{- if .port }}
 ---
 apiVersion: v1
 kind: Service
@@ -36,8 +46,10 @@ spec:
       protocol: TCP
       port: {{ .port }}
       targetPort: {{ .port }}
----
+{{- end }}
 {{- if .ingress }}
+{{- $service := .name }}
+---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -51,15 +63,29 @@ spec:
   ingressClassName: {{ .ingress.class }}
   {{- end }}
   rules:
-    - host: {{ required "ingress.host is required" .ingress.host }}
+    {{- range .ingress.hosts }}
+    - host: {{ . }}
       http:
         paths:
           - path: /
             pathType: Prefix
             backend:
               service:
-                name: extension-{{ .name }}
+                name: extension-{{ $service }}
                 port:
                   number: 8000
+    {{- end }}
+    {{- if .ingress.default }}
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: extension-{{ $service }}
+                port:
+                  number: 8000
+    {{- end }}
 {{- end }}
+---
 {{- end }}

package/src/deployment/chart/templates/variables.yaml

@@ -8,5 +8,8 @@
     secretKeyRef:
       name: {{ .secret.name }}
       key: {{ .secret.key }}
+      {{- if .secret.optional }}
+      optional: {{ .secret.optional }}
+      {{- end }}
   {{- end }}
 {{- end }}

package/src/deployment/chart/values.yaml

@@ -3,6 +3,10 @@ compositions:
   - name: todos
     image: localhost:5000/composition-todos:0.0.0
     replicas: 2
+    mounts:
+      - name: mount-name
+        path: /storage
+        claim: storage-pvc
     components:
       - todos-tasks
       - todos-stats
@@ -12,12 +16,12 @@ compositions:
         secret:
           name: secret-name
           key: secret-key
+          optional: false
   - name: users
     image: localhost:5000/composition-users:0.0.0
     replicas: 3
     components:
       - users-users
-# TODO: create component services only if sync binding is being used
 components:
   - todos-tasks
   - todos-stats
@@ -28,7 +32,8 @@ services:
     port: 8000
     replicas: 2
     ingress:
-      host: dummies.toa.io
+      default: true
+      hosts: [dummies.toa.io]
       class: alb
       annotations:
         alb.ingress.kubernetes.io/scheme: internet-facing
@@ -40,6 +45,11 @@ services:
         secret:
           name: secret-name
           key: secret-key
+    probe:
+      port: 8000
+      path: /.ready
+      delay: 1
+
 proxies:
   - name: storage-proxy
     target: host.docker.internal

package/src/deployment/deployment.js

@@ -2,8 +2,8 @@
 
 const { join } = require('node:path')
 const { writeFile: write } = require('node:fs/promises')
-const { directory: { copy } } = require('@toa.io/filesystem')
 const { dump } = require('@toa.io/yaml')
+const fs = require('fs-extra')
 
 const { merge, declare, describe } = require('./.deployment')
 
@@ -28,7 +28,7 @@ class Deployment {
     await Promise.all([
       write(join(target, 'Chart.yaml'), chart),
       write(join(target, 'values.yaml'), values),
-      copy(TEMPLATES, join(target, 'templates'))
+      fs.copy(TEMPLATES, join(target, 'templates'))
     ])
 
     this.#target = target
@@ -42,6 +42,7 @@ class Deployment {
 
     if (options.namespace !== undefined) args.push('-n', options.namespace)
     if (options.wait === true) args.push('--wait')
+    if (options.timeout !== undefined) args.push('--timeout', options.timeout)
 
     await this.#process.execute('helm', ['dependency', 'update', this.#target])
     await this.#process.execute('helm', ['upgrade', this.#chart.name, '-i', ...args, this.#target])

package/src/deployment/factory.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { context: load } = require('@toa.io/norm')
 const { Process } = require('../process')
 const { Operator } = require('./operator')
 const { Factory: ImagesFactory } = require('./images')
@@ -14,7 +15,6 @@ class Factory {
   #dependencies
   #registry
   #process
-  #extensionComponents = []
 
   constructor (context) {
     this.#context = context
@@ -85,6 +85,12 @@ class Factory {
 
     return new Service(service, image)
   }
+
+  static async create (path, environment) {
+    const context = await load(path, environment)
+
+    return new Factory(context)
+  }
 }
 
 exports.Factory = Factory

package/src/deployment/images/composition.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.9.0-alpine3.18
+FROM {{build.image}}
 
 {{build.arguments}}
 

package/src/deployment/images/composition.js

@@ -1,30 +1,23 @@
 'use strict'
 
 const { join } = require('node:path')
-const { hash } = require('@toa.io/generic')
-const { directory: { copy } } = require('@toa.io/filesystem')
+const fs = require('fs-extra')
+const { createHash } = require('node:crypto')
 
 const { Image } = require('./image')
 
 class Composition extends Image {
   dockerfile = join(__dirname, 'composition.Dockerfile')
 
-  /** @type {string} */
   #name
-
-  /** @type {toa.norm.Component[]} */
+  #image
   #components
 
-  /**
-   * @param {string} scope
-   * @param {toa.norm.context.Runtime} runtime
-   * @param {toa.norm.context.Registry} registry
-   * @param {toa.norm.context.Composition} composition
-   */
   constructor (scope, runtime, registry, composition) {
     super(scope, runtime, registry)
 
     this.#name = composition.name
+    this.#image = composition.image
     this.#components = composition.components
   }
 
@@ -33,16 +26,41 @@ class Composition extends Image {
   }
 
   get version () {
-    const tags = this.#components.map((component) => component.locator.id + ':' + component.version)
+    const hash = createHash('sha256')
+
+    for (const component of this.#components) {
+      hash.update(component.locator.id)
+      hash.update(component.version)
+    }
+
+    return hash.digest('hex').slice(0, 8)
+  }
+
+  get base () {
+    if (this.#image !== undefined) {
+      return this.#image
+    }
+
+    let image = null
+
+    for (const component of this.#components) {
+      const value = component.build?.image
+
+      if (image !== null && image !== value) {
+        throw new Error(`Composition '${this.#name}' requires different base images for its components. Specify base image for the composition in the context.`)
+      }
+
+      image = value
+    }
 
-    return hash(tags.join(';'))
+    return image ?? undefined
   }
 
   async prepare (root) {
     const context = await super.prepare(root)
 
     for (const component of this.#components) {
-      await copy(component.path, join(context, component.locator.label))
+      await fs.copy(component.path, join(context, component.locator.label))
     }
 
     return context

package/src/deployment/images/factory.js

@@ -3,9 +3,6 @@
 const { Composition } = require('./composition')
 const { Service } = require('./service')
 
-/**
- * @implements {toa.deployment.images.Factory}
- */
 class Factory {
   /** @type {string} */
   #scope

package/src/deployment/images/image.fixtures.js

@@ -2,9 +2,8 @@
 
 const { Image } = require('./image')
 const { generate } = require('randomstring')
-const { hash } = require('@toa.io/generic')
 
-const version = generate()
+const version = '168b04ff'
 const name = generate()
 
 /**
@@ -22,17 +21,17 @@ class Class extends Image {
 
 /** @type {toa.norm.context.Runtime} */
 const runtime = {
-  version: generate()
+  version: '0.0.0'
 }
 
 /** @type {toa.norm.context.Registry} */
 const registry = {
-  base: generate()
+  base: 'node:alpine'
 }
 
 exports.scope = generate()
 exports.name = name
-exports.version = hash(runtime.version + ';' + version)
+exports.version = 'ba2409fc'
 exports.Class = Class
 exports.runtime = runtime
 exports.registry = registry

package/src/deployment/images/image.js

@@ -1,10 +1,17 @@
 'use strict'
 
-const { join, posix } = require('node:path')
-const { readFile: read, writeFile: write } = require('node:fs/promises')
-
-const { hash, overwrite } = require('@toa.io/generic')
-const { directory } = require('@toa.io/filesystem')
+const {
+  join,
+  posix
+} = require('node:path')
+const {
+  readFile: read,
+  writeFile: write
+} = require('node:fs/promises')
+const { createHash } = require('node:crypto')
+
+const { overwrite } = require('@toa.io/generic')
+const { mkdir } = require('node:fs/promises')
 
 /**
  * @implements {toa.deployment.images.Image}
@@ -13,68 +20,52 @@ const { directory } = require('@toa.io/filesystem')
 class Image {
   context
   reference
-
-  /**
-   * @protected
-   * @type {string}
-   */
   dockerfile
 
-  /** @type {string} */
   #scope
-
-  /** @type {toa.norm.context.Registry} */
   #registry
-
-  /** @type {toa.norm.context.Runtime} */
   #runtime
+  #values = {
+    build: {
+      image: 'node:20.9.0-alpine3.18'
+    }
+  }
 
-  /** @type {{ runtime?: Partial<toa.norm.context.Runtime>, build: object }} */
-  #values = { build: { command: 'echo hello' } }
-
-  /**
-   * @param {string} scope
-   * @param {toa.norm.context.Runtime} runtime
-   * @param {toa.norm.context.Registry} registry
-   */
   constructor (scope, runtime, registry) {
     this.#scope = scope
     this.#registry = registry
     this.#runtime = runtime
-
-    this.#setValues()
   }
 
   tag () {
-    const tag = hash(this.#runtime?.version + ';' + this.version)
+    const hash = createHash('sha256')
+
+    hash.update(this.#runtime.version)
+    hash.update(this.version)
+
+    const tag = hash.digest('hex').slice(0, 8)
 
     this.reference = posix.join(this.#registry.base ?? '', this.#scope, `${this.name}:${tag}`)
   }
 
-  /**
-   * @abstract
-   * @protected
-   * @type {string}
-   */
   get name () {}
 
-  /**
-   * @abstract
-   * @protected
-   * @type {string}
-   */
   get version () {}
 
+  get base () {}
+
   async prepare (root) {
     if (this.dockerfile === undefined) throw new Error('Dockerfile isn\'t specified')
 
+    this.#setValues()
+
     const path = join(root, `${this.name}.${this.version}`)
 
-    await directory.ensure(path)
+    await mkdir(path, { recursive: true })
 
     const template = await read(this.dockerfile, 'utf-8')
     const contents = template.replace(/{{(\S{1,32})}}/g, (_, key) => this.#value(key))
-    const ignore = 'Dockerfile'
+    const ignore = ['Dockerfile', '**/node_modules'].join('\r\n')
 
     await write(join(path, 'Dockerfile'), contents)
     await write(join(path, '.dockerignore'), ignore)
@@ -88,6 +79,12 @@ class Image {
     this.#values.runtime = this.#runtime
     this.#values.build = overwrite(this.#values.build, this.#registry.build)
 
+    const image = this.base
+
+    if (image !== undefined) {
+      this.#values.build.image = image
+    }
+
     if (this.#values.build.arguments !== undefined) this.#values.build.arguments = createArguments(this.#values.build.arguments)
     if (this.#values.build.run !== undefined) this.#values.build.run = createRunCommands(this.#values.build.run)
   }

package/src/deployment/images/service.js

@@ -3,7 +3,7 @@
 const { join, dirname } = require('node:path')
 
 const { Image } = require('./image')
-const { directory: { copy } } = require('@toa.io/filesystem')
+const fs = require('fs-extra')
 
 class Service extends Image {
   dockerfile = join(__dirname, 'service.Dockerfile')
@@ -57,7 +57,7 @@ class Service extends Image {
   async prepare (root) {
     const context = await super.prepare(root)
 
-    await copy(this.#path, context)
+    await fs.copy(this.#path, context)
 
     return context
   }

package/src/deployment/operator.js

@@ -50,6 +50,10 @@ class Operator {
   variables () {
     return this.#deployment.variables()
   }
+
+  tags () {
+    return this.#registry.tags()
+  }
 }
 
 /** @type {toa.deployment.installation.Options} */

package/src/deployment/registry.js

@@ -1,28 +1,20 @@
 'use strict'
 
 const workspace = require('./workspace')
+const { newid } = require('@toa.io/generic')
 
 /**
  * @implements {toa.deployment.Registry}
  */
 class Registry {
-  /** @type {toa.norm.context.Registry} */
   #registry
 
-  /** @type {toa.deployment.images.Factory} */
   #factory
 
-  /** @type {toa.operations.Process} */
   #process
 
-  /** @type {toa.deployment.images.Image[]} */
   #images = []
 
-  /**
-   * @param {toa.norm.context.Registry} registry
-   * @param {toa.deployment.images.Factory} factory
-   * @param {toa.operations.Process} process
-   */
   constructor (registry, factory, process) {
     this.#registry = registry
     this.#factory = factory
@@ -48,7 +40,9 @@ class Registry {
   async build () {
     await this.prepare()
 
-    for (const image of this.#images) await this.#build(image)
+    for (const image of this.#images) {
+      await this.#build(image)
+    }
   }
 
   async push () {
@@ -57,8 +51,12 @@ class Registry {
     for (const image of this.#images) await this.#push(image)
   }
 
+  tags () {
+    return this.#images.map((image) => image.reference)
+  }
+
   /**
-   * @param {"composition" | "service"} type
+   * @param {'composition' | 'service'} type
    * @param {...any} args
    * @returns {toa.deployment.images.Image}
    */
@@ -76,9 +74,18 @@ class Registry {
    * @returns {Promise<void>}
    */
   async #build (image, push = false) {
+    if (await this.exists(image.reference)) {
+      console.log('Image already exists, skipping:', image.reference)
+      return
+    }
+
     const args = ['--context=default', 'buildx', 'build']
 
-    if (push) args.push('--push')
+    if (push) {
+      args.push('--push')
+    } else {
+      args.push('--load')
+    }
 
     args.push('--tag', image.reference, image.context)
 
@@ -90,12 +97,15 @@ class Registry {
 
     if (multiarch) {
       const platform = this.#registry.platforms.join(',')
+      const builder = await this.#createBuilder()
 
       args.push('--platform', platform)
-      args.push('--builder', BUILDER)
+      args.push('--builder', builder)
+    } else {
+      args.push('--builder', 'default')
+    }
 
-      await this.#ensureBuilder()
-    } else args.push('--builder', 'default')
+    args.push('--progress', 'plain')
 
     await this.#process.execute('docker', args)
   }
@@ -104,20 +114,27 @@ class Registry {
     await this.#build(image, true)
   }
 
-  async #ensureBuilder () {
-    const ls = 'buildx ls'.split(' ')
-    const output = await this.#process.execute('docker', ls, { silently: true })
-    const exists = output.split('\n').findIndex((line) => line.startsWith('toa '))
+  async exists (tag) {
+    const args = ['manifest', 'inspect', tag]
+
+    try {
+      await this.#process.execute('docker', args, { silently: true })
+    } catch (error) {
+      console.log(error.message)
 
-    if (exists === -1) await this.#createBuilder()
+      return false
+    }
+
+    return true
   }
 
   async #createBuilder () {
-    const create = `buildx create --name ${BUILDER} --use`.split(' ')
-    const bootstrap = 'buildx inspect --bootstrap'.split(' ')
+    const name = `toa-${newid()}`
+    const create = `buildx create --name ${name} --bootstrap --use`.split(' ')
 
     await this.#process.execute('docker', create)
-    await this.#process.execute('docker', bootstrap)
+
+    return name
   }
 }
 

package/types/_deployment/images/image.d.ts

@@ -4,9 +4,11 @@ declare namespace toa.deployment.images {
     readonly reference: string
     readonly context: string
 
-    tag(): void
+    name: string
 
-    prepare(root: string): Promise<string>
+    tag (): void
+
+    prepare (root: string): Promise<string>
   }
 
 }

package/types/_deployment/registry.d.ts

@@ -1,19 +1,21 @@
 import type * as _norm from '@toa.io/norm/types'
 import type * as _dependency from './dependency'
-import type * as _image from "./images/image"
+import type * as _image from './images/image'
 
 declare namespace toa.deployment {
-  
+
   interface Registry {
-    composition(composition: _norm.Composition): _image.Image
+    composition (composition: _norm.Composition): _image.Image
+
+    service (path: string, service: _dependency.Service): _image.Image
 
-    service(path: string, service: _dependency.Service): _image.Image
+    prepare (path: string): Promise<string>
 
-    prepare(path: string): Promise<string>
+    build (): Promise<void>
 
-    build(): Promise<void>
+    push (): Promise<void>
 
-    push(): Promise<void>
+    tags (): string[]
   }
 
 }

package/types/dependency.ts

@@ -0,0 +1,59 @@
+import type { Manifest } from '@toa.io/norm'
+import type { Locator } from '@toa.io/core'
+
+export interface Service {
+  group: string
+  name: string
+  version: string
+  port?: number
+  ingress?: Ingress
+  variables?: Variable[]
+  components?: string[]
+  probe?: Probe
+}
+
+export interface Variable {
+  name: string
+  value?: string
+  secret?: {
+    name: string
+    key: string
+    optional?: boolean
+  }
+}
+
+export interface Instance<T> {
+  locator: Locator
+  manifest: T
+  component: Manifest
+}
+
+export type Instances<T> = Array<Instance<T>>
+
+export type Variables = Record<'global' | string, Variable[]>
+export type Mounts = Record<'global' | string, Mount[]>
+
+export interface Dependency {
+  services?: Service[]
+  variables?: Variables
+  mounts?: Mounts
+}
+
+interface Ingress {
+  default?: boolean
+  hosts?: string[]
+  class?: string
+  annotations?: object
+}
+
+interface Probe {
+  port: number
+  path: string
+  delay?: number
+}
+
+interface Mount {
+  name: string
+  path: string
+  claim: string
+}

package/types/dependency.d.ts

@@ -1,29 +0,0 @@
-export type Service = {
-  group: string
-  name: string
-  version: string
-  port: number
-  ingress?: {
-    host: string
-    class?: string
-    annotations?: object
-  }
-  variables: Variable[]
-  components?: string[]
-}
-
-export type Variable = {
-  name: string
-  value?: string
-  secret?: {
-    name: string,
-    key: string
-  }
-}
-
-export type Variables = Record<'global' | string, Variable[]>
-
-export type Dependency = {
-  services?: Service[]
-  variables?: Variables
-}