@toa.io/extensions.configuration 0.20.0-dev.31 → 0.20.0-dev.35

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@toa.io/extensions.configuration",
-  "version": "0.20.0-dev.31",
+  "version": "0.20.0-dev.35",
   "description": "Toa Configuration",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "source/index.js",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -15,12 +16,18 @@
   "publishConfig": {
     "access": "public"
   },
-  "dependencies": {
-    "@toa.io/core": "0.20.0-dev.31",
-    "@toa.io/generic": "0.20.0-dev.31",
-    "@toa.io/schema": "0.20.0-dev.31",
-    "@toa.io/yaml": "0.20.0-dev.31",
-    "clone-deep": "4.0.1"
+  "scripts": {
+    "prepublishOnly": "npm run transpile",
+    "transpile": "tsc"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
   },
-  "gitHead": "f9ad6bf2bab1298b96ca52557b7e36b6041cfc88"
+  "gitHead": "04d3317272256b98003a414ee1eb2555b65163a5",
+  "dependencies": {
+    "@toa.io/core": "0.20.0-dev.35",
+    "@toa.io/generic": "0.20.0-dev.35",
+    "@toa.io/schemas": "0.20.0-dev.35"
+  }
 }

package/readme.md

@@ -5,20 +5,24 @@
 ### Define
 
 ```yaml
-# component.toa.yaml
+# manifest.toa.yaml
 name: dummy
 namespace: dummies
 
 configuration:
-  foo: bar
-  baz: 1
+  schema:
+    foo: string
+    bar: number
+  defaults:
+    foo: bar
+    bar: 1
 ```
 
 ### Use
 
 ```javascript
 function transition (input, entity, context) {
-  const { foo, baz } = context.configuration
+  const { foo, bar } = context.configuration
 
   // ...
 }
@@ -30,9 +34,9 @@ function transition (input, entity, context) {
 # context.toa.yaml
 configuration:
   dummies.dummy:
-    foo: qux          # override default value defined by dummies.dummy
+    foo: qux          # override default value
     foo@staging: quux # deployment environment discriminator
-    baz: $BAZ_VALUE   # secret
+    bar: $BAZ_VALUE   # secret
 ```
 
 ### Deploy secrets
@@ -45,130 +49,96 @@ $ toa conceal configuration BAZ_VALUE=$ecr3t
 
 ## Problem Definition
 
-- Components must be reusable in different contexts and deployment environments that are in
-  different configurations.
-- Some algorithm parameters must be deployed secretly.
-
-## Definitions
-
-### Configuration (Distributed System Configuration)
-
-Set of static[^1] parameters for all algorithms within a given system.
-
-### Configuration Schema
-
-Schema is defining component's algorithm parameters (optionally with default values).
+- Components should be runnable in different deployment environments.
+- Some algorithm's parameters should be deployed secretly.
+- Components should be reusable in different contexts.
 
-### Configuration Object
+## Manifest
 
-Value valid against Configuration Schema.
+Component's configuration is declared using `configuration` manifest,
+containing `schema` and optionnaly `defaults` properties.
 
-### Configuration Value
+### Schema
 
-The merge result of Configuration Schema's defaults and Configuration Object.
+Configuration schema is declared with [COS](/libraries/concise).
 
-### Context Configuration
-
-Map of Configuration Objects for components added to a given context.
-
-## Responsibility Segregation
+```yaml
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
 
-Configuration Schema is a *form* of configuration defined by component. Specific *values* for
-specific contexts and deployment environments are defined by Context Configuration according to the
-Schema.
+configuration:
+  schema:
+    foo: string
+    bar: number
+```
 
-See [Reusable Components](#).
+> Introducing non-backward compatible changes to a configuration schema will result in a loss of
+> compatibility with existing contexts and deployment environments.
+> Therefore, configuration schema changes are subject to component versioning.
 
-## Configuration Schema
+If `configuration` object doesn't contain property `schema`, then it is considered to be schema.
 
-Configuration Schema is declared as a component extension
-using [JSON Schema](https://json-schema.org) `object` type.
+```yaml
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
 
-> ![Warning](https://img.shields.io/badge/Warning-yellow)<br/>
-> By introducing non-backward compatible changes to a Configuration Schema, the compatibility
-> with existent contexts and deployment environments will be broken. That is, Configuration
-> Schema changes are subjects of component versioning.
+configuration:
+  foo: string
+  bar: number
+```
 
-> ![Recommendation](https://img.shields.io/badge/Recommendation-green)<br/>
-> Having default values for all required parameters will allow components to be runnable
-> without configuration (i.e. on local environment).
+### Defaults
 
-### Example
+The default configuration value can be provided using the `defaults` property, which should conform
+to the configuration schema.
 
 ```yaml
-# component.toa.yaml
+# manifest.toa.yaml
 name: dummy
 namespace: dummies
 
-extensions:
-  "@toa.io/extensions.configuration":
-    properties:
-      foo:
-        type: string
-        default: 'baz'
-      bar:
-        type: number
-    required: [foo]
+configuration:
+  schema:
+    foo: string
+    bar: number
+  defaults:
+    foo: hello
+    bar: 0
 ```
 
-### Concise Declaration
-
-As it is known that Configuration Schema is declared with a JSON Schema `object` type, any
-configuration declaration without defined `properties` considered as concise. Properties of concise
-declaration are treated as required Configuration Schema properties with the same type as its value
-type and no additional properties allowed.
-
-Also note that a well-known shortcut `configuration` is available.
+#### Schema defaults hint
 
-The next two declarations are equivalent.
+The configuration schema itself can contain default primitive values using the COS syntax.
 
 ```yaml
-# component.toa.yaml
-configuration:
-  foo: baz
-  bar: 1
-```
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
 
-```yaml
-# component.toa.yaml
-extensions:
-  "@toa.io/extensions.configuration":
-    properties:
-      foo:
-        type: string
-        default: baz
-      bar:
-        type: number
-        default: 1
-    additionalProperties: false
-    required: [foo, bar]
+configuration:
+  schema:
+    foo: hello
+    bar: 0
 ```
 
-## Context Configuration
-
-Context Configuration is declared as a context annotation. Its keys must be
-component identifiers, and its values must be Configuration Objects for those
-components.
-
-Context Configuration keys and Configuration Object keys may be defined
-with [deployment environment discriminators](#).
+## Annotation
 
-### Example
+A component's configuration can be overridden using the configuration context annotation.
 
 ```yaml
 # context.toa.yaml
 configuration:
   dummies.dummy:
-    foo: quu
+    foo: bye
     bar: 1
     bar@staging: 2
 ```
 
-## Configuration Secrets
+## Secrets
 
-Context Configuration values which are uppercase strings prefixed with `$` considered as Secrets.
-
-### Example
+Configuration annotation top-level values which are uppercase strings prefixed with `$` considered as secrets.
 
 ```yaml
 # context.toa.yaml
@@ -177,14 +147,10 @@ configuration:
     api-key: $STRIPE_API_KEY
 ```
 
-Configuration values that are assigned with a reference to the Secret must be of type `string`.
-
-### Secrets Deployment
-
 Secrets are not being deployed with context
-deployment ([`toa deploy`](../../runtime/cli/readme.md#deploy)),
-thus must be deployed separately at least once for each deployment environment
-manually ([`toa conceal`](../../runtime/cli/readme.md#conceal)).
+deployment ([`toa deploy`](/runtime/cli/readme.md#deploy)), thus must be deployed separately at
+least once for each deployment environment
+manually ([`toa conceal`](/runtime/cli/readme.md#conceal)).
 
 Deployed kubernetes secret's name is predefined as `configuration`.
 
@@ -194,65 +160,12 @@ $ toa conceal configuration STRIPE_API_KEY=xxxxxxxx
 
 ## Aspect
 
-Configuration Value is available as a well-known operation Aspect `configuration`.
+Component's configuration values are available as a well-known Aspect `configuration`.
 
 ```javascript
-// Node.js bridge
-
 function transition (input, entity, context) {
   const foo = context.configiuration.foo
 
   // ...
 }
 ```
-
-> ![Warning](https://img.shields.io/badge/Warning-yellow)<br/>
-> It is strongly **not** recommended to store a copy of value type configuration
-> values outside operation scope, thus it prevents operation to benefit
-> from [hot updates](#).
->
-> ```javascript
-> // NOT RECOMMENDED
-> let foo
-> 
-> function transition (input, entity, context) {
->   // NOT RECOMMENDED
->   if (foo === undefined) foo = context.configuration.foo
-> 
->   // ...
-> }
-> ```
-> See [Genuine operations](/documentation/design.md#genuine-operations).
-
-## Development Configuration
-
-Configuration can be exported by [`toa env`](/runtime/cli/readme.md#env).
-
-### Local Environment Placeholders
-
-Context Configuration values may contain placeholders that reference environment variables.
-Placeholders are replaced with values if the corresponding environment variables are set.
-
-> Placeholders can only be used with local environment (exported by `toa env`), as these values are
-> not
-> deployed.
-
-```yaml
-# context.toa.yaml
-configuration:
-  dummies.dummy:
-    url@local: https://stage${STAGE}.intranet/
-```
-
-```dotenv
-# .env
-STAGE=82
-```
-
-## Appendix
-
-- [Discussion](./docs/discussion.md)
-- [Configuration consistency](./docs/consistency.md)
-
-[^1]: Cannot be changed without a deployment as new values are considered to be a subject of
-testing. [#146](https://github.com/toa-io/toa/issues/146)

package/schemas/annotation.cos.yaml

@@ -0,0 +1 @@
+<object>

package/schemas/manifest.cos.yaml

@@ -0,0 +1,2 @@
+schema: object
+defaults?: object

package/source/Aspect.test.ts

@@ -0,0 +1,15 @@
+import { Aspect } from './Aspect'
+
+it('should return value', async () => {
+  const configuration = {
+    foo: 'bar',
+    bar: {
+      baz: 'quux'
+    }
+  }
+
+  const aspect = new Aspect(configuration)
+
+  expect(aspect.invoke(['foo'])).toStrictEqual(configuration.foo)
+  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual(configuration.bar.baz)
+})

package/source/Aspect.ts

@@ -0,0 +1,23 @@
+import { Connector, type extensions } from '@toa.io/core'
+
+export class Aspect extends Connector implements extensions.Aspect {
+  public readonly name = 'configuration'
+
+  private readonly value: object
+
+  public constructor (value: object) {
+    super()
+
+    this.value = value
+  }
+
+  public invoke (path: string[]): any {
+    let cursor: any = this.value
+
+    if (path !== undefined)
+      for (const segment of path)
+        cursor = cursor[segment]
+
+    return cursor
+  }
+}

package/source/Factory.ts

@@ -0,0 +1,12 @@
+import { type Locator, type extensions } from '@toa.io/core'
+import { Aspect } from './Aspect'
+import { type Manifest } from './manifest'
+import { get } from './configuration'
+
+export class Factory implements extensions.Factory {
+  public aspect (locator: Locator, manifest: Manifest): extensions.Aspect {
+    const value = get(locator, manifest)
+
+    return new Aspect(value)
+  }
+}

package/source/configuration.test.ts

@@ -0,0 +1,89 @@
+import { encode } from '@toa.io/generic'
+import { Locator } from '@toa.io/core'
+import { generate } from 'randomstring'
+import { get } from './configuration'
+import { type Manifest } from './manifest'
+
+let locator: Locator
+let manifest: Manifest
+
+beforeEach(() => {
+  locator = new Locator(generate(), generate())
+  manifest = { schema: { foo: 'string' } }
+})
+
+afterEach(() => {
+  for (const name of used)
+    process.env[name] = undefined
+
+  used = []
+})
+
+it('should read value', async () => {
+  manifest.schema = { foo: 'string' }
+  const value: object = { foo: generate() }
+
+  set(value)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual(value)
+})
+
+it('should return empty object if no value set', async () => {
+  expect(get(locator, manifest)).toStrictEqual({})
+})
+
+it('should substitute secrets', async () => {
+  const value: object = { foo: '$BAR' }
+
+  set(value)
+  set('bar', '_BAR')
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({ foo: 'bar' })
+})
+
+it('should use defaults', async () => {
+  manifest.schema = { foo: 'string', bar: ['number'], 'baz?': 'string' }
+  manifest.defaults = { foo: 'bar', bar: [1] }
+
+  const values = { bar: [2], baz: 'foo' }
+
+  set(values)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({
+    foo: 'bar',
+    bar: [2],
+    baz: 'foo'
+  })
+})
+
+it('should validate', async () => {
+  manifest.schema = { foo: 'hello', bar: 'number' }
+
+  const values = { bar: 5 }
+
+  set(values)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({
+    foo: 'hello',
+    bar: 5
+  })
+})
+
+function set (value: object | string, key = locator.uppercase): void {
+  const string = typeof value === 'string' ? value : encode(value)
+  const name = 'TOA_CONFIGURATION_' + key
+
+  process.env[name] = string
+
+  used.push(name)
+}
+
+let used: string[] = []

package/source/configuration.ts

@@ -0,0 +1,52 @@
+import { type Locator } from '@toa.io/core'
+import { decode, add } from '@toa.io/generic'
+import * as schemas from '@toa.io/schemas'
+import { PREFIX, SECRET_RX } from './deployment'
+import { type Manifest } from './manifest'
+
+export function get (locator: Locator, manifest: Manifest): Configuration {
+  const values = getConfiguration(locator.uppercase)
+
+  substituteSecrets(values)
+
+  if (manifest.defaults !== undefined) add(values, manifest.defaults)
+
+  const schema = schemas.schema(manifest.schema)
+
+  schema.validate(values)
+
+  return values
+}
+
+function getConfiguration (suffix: string): Configuration {
+  const variable = PREFIX + suffix
+  const string = process.env[variable]
+
+  if (string === undefined) return {}
+  else return decode(string)
+}
+
+function substituteSecrets (configuration: Configuration): void {
+  for (const [key, value] of Object.entries(configuration)) {
+    if (typeof value !== 'string') continue
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) continue
+
+    const name = match.groups?.variable as string
+
+    configuration[key] = getSecret(name)
+  }
+}
+
+function getSecret (name: string): string {
+  const variable = PREFIX + '_' + name
+  const value = process.env[variable]
+
+  if (value === undefined) throw new Error(`${variable} is not set.`)
+
+  return value
+}
+
+export type Configuration = Record<string, any>

package/source/deployment.test.ts

@@ -0,0 +1,21 @@
+import { type Annotation, deployment, type Instance } from './deployment'
+import { type Manifest } from './manifest'
+
+it('should validate annotation', async () => {
+  const wrongType = 'not ok' as unknown as Annotation
+
+  expect(() => deployment([], wrongType)).toThrow('object')
+})
+
+it('should validate values', async () => {
+  const manifest: Manifest = {
+    schema: { foo: 'string', bar: 'number' },
+    defaults: { foo: 'ok', bar: 0 }
+  }
+
+  const locator = { id: 'component' }
+  const instances = [{ manifest, locator }] as unknown as Instance[]
+  const annotation: Annotation = { [locator.id]: { bar: 'not a number' } }
+
+  expect(() => deployment(instances, annotation)).toThrow('number')
+})

package/source/deployment.ts

@@ -0,0 +1,69 @@
+import { type Dependency, type Variable, type Variables } from '@toa.io/operations'
+import * as schemas from '@toa.io/schemas'
+import { encode, overwrite } from '@toa.io/generic'
+import { type Manifest } from './manifest'
+import * as validators from './schemas'
+import type { context } from '@toa.io/norm'
+
+export function deployment (instances: Instance[], annotation: Annotation): Dependency {
+  validators.annotation.validate(annotation)
+
+  const variables: Variables = {}
+
+  for (const instance of instances) {
+    const values = annotation[instance.locator.id]
+
+    if (values === undefined) continue
+
+    validate(instance, values)
+
+    variables[instance.locator.label] = [{
+      name: PREFIX + instance.locator.uppercase,
+      value: encode(values)
+    }]
+
+    const secrets = createSecrets(values)
+
+    variables[instance.locator.label].push(...secrets)
+  }
+
+  return { variables }
+}
+
+function createSecrets (values: object): Variable[] {
+  const secrets: Variable[] = []
+
+  for (const value of Object.values(values)) {
+    if (typeof value !== 'string') continue
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) continue
+
+    const name = match.groups?.variable as string
+
+    secrets.push({
+      name: PREFIX + '_' + name,
+      secret: {
+        name: 'toa-configuration',
+        key: name
+      }
+    })
+  }
+
+  return secrets
+}
+
+function validate (instace: Instance, values: object): void {
+  const defaults = instace.manifest.defaults ?? {}
+  const configuration = overwrite(defaults, values)
+  const schema = schemas.schema(instace.manifest.schema)
+
+  schema.validate(configuration)
+}
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+export type Annotation = Record<string, object>
+export type Instance = context.Dependency<Manifest>

package/source/index.ts

@@ -0,0 +1,3 @@
+export { manifest } from './manifest'
+export { deployment } from './deployment'
+export { Factory } from './Factory'

package/source/manifest.test.ts

@@ -0,0 +1,15 @@
+import { type Manifest, manifest } from './manifest'
+
+it('should validate', async () => {
+  const additional = { schema: {}, foo: 'bar' } as unknown as Manifest
+
+  expect(() => {
+    manifest(additional)
+  }).toThrow('additional')
+
+  const wrongType = { schema: 'not ok' } as unknown as Manifest
+
+  expect(() => {
+    manifest(wrongType)
+  }).toThrow('object')
+})

package/source/manifest.ts

@@ -0,0 +1,15 @@
+import * as schemas from './schemas'
+import { type Configuration } from './configuration'
+
+export function manifest (manifest: Manifest): Manifest {
+  if (manifest.schema === undefined) manifest = { schema: manifest }
+
+  schemas.manifest.validate(manifest)
+
+  return manifest
+}
+
+export interface Manifest {
+  schema: object
+  defaults?: Configuration
+}