@toa.io/extensions.configuration 0.7.3-dev.2 → 0.20.0-dev.1

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@toa.io/extensions.configuration",
-  "version": "0.7.3-dev.2",
+  "version": "0.20.0-dev.1",
   "description": "Toa Configuration",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "src/index.js",
+  "main": "source/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -16,11 +16,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@toa.io/core": "0.8.1-dev.1",
-    "@toa.io/generic": "0.9.0-dev.1",
-    "@toa.io/schema": "0.7.3-dev.2",
-    "@toa.io/yaml": "0.7.3-dev.1",
+    "@toa.io/core": "0.20.0-dev.1",
+    "@toa.io/generic": "0.11.0-dev.3",
+    "@toa.io/schema": "0.20.0-dev.1",
+    "@toa.io/yaml": "0.20.0-dev.1",
     "clone-deep": "4.0.1"
   },
-  "gitHead": "8b3fb929284365afb0d301756dfbcc549734dc99"
+  "gitHead": "23a9b4399af908101aac4ae03f22f7b948dfe55f"
 }

package/readme.md

@@ -30,23 +30,22 @@ function transition (input, entity, context) {
 # context.toa.yaml
 configuration:
   dummies.dummy:
-    foo: qux
-    foo@staging: quux # use deployment environment discriminator
-    baz: $BAZ_VALUE   # use secrets
+    foo: qux          # override default value defined by dummies.dummy
+    foo@staging: quux # deployment environment discriminator
+    baz: $BAZ_VALUE   # secret
 ```
 
 ### Deploy secrets
 
 ```shell
-$ toa conceal
+$ toa conceal configuration BAZ_VALUE '$ecr3t'
 ```
 
 ---
 
 ## Problem Definition
 
-- Components must be reusable in different contexts and deployment environments,
-  that is in different configurations.
+- Components must be reusable in different contexts and deployment environments that are in different configurations.
 - Some algorithm parameters must be deployed secretly.
 
 ## Definitions
@@ -57,8 +56,7 @@ Set of static[^1] parameters for all algorithms within a given system.
 
 ### Configuration Schema
 
-Schema defining component's algorithms parameters (optionally with default
-values).
+Schema is defining component's algorithm parameters (optionally with default values).
 
 ### Configuration Object
 
@@ -66,7 +64,7 @@ Value valid against Configuration Schema.
 
 ### Configuration Value
 
-Merge result of Configuration Schema's defaults and Configuration Object.
+The merge result of Configuration Schema's defaults and Configuration Object.
 
 ### Context Configuration
 
@@ -86,7 +84,7 @@ Configuration Schema is declared as a component extension
 using [JSON Schema](https://json-schema.org) `object` type.
 
 > ![Warning](https://img.shields.io/badge/Warning-yellow)<br/>
-> By introducing non-backward compatible changes to a Configuration Schema the compatibility
+> By introducing non-backward compatible changes to a Configuration Schema, the compatibility
 > with existent contexts and deployment environments will be broken. That is, Configuration
 > Schema changes are subjects of component versioning.
 
@@ -121,7 +119,7 @@ type and no additional properties allowed.
 
 Also note that a well-known shortcut `configuration` is available.
 
-Next two declarations are equivalent.
+The next two declarations are equivalent.
 
 ```yaml
 # component.toa.yaml
@@ -147,8 +145,8 @@ extensions:
 
 ## Context Configuration
 
-Context Configuration is declared as a context annotaion. Its keys must be
-component identifiers and its values must be Configuration Objects for those
+Context Configuration is declared as a context annotation. Its keys must be
+component identifiers, and its values must be Configuration Objects for those
 components.
 
 Context Configuration keys and Configuration Object keys may be defined
@@ -165,15 +163,9 @@ configuration:
     bar@staging: 2
 ```
 
-### Local environment
-
-Configuration Objects for local environment may be created
-by [`toa configure`](../../runtime/cli/readme.md#configure) command.
-
 ## Configuration Secrets
 
-Context Configuration values which are uppercase strings prefixed with `$`
-considered as Secrets.
+Context Configuration values which are uppercase strings prefixed with `$` considered as Secrets.
 
 ### Example
 
@@ -184,6 +176,8 @@ configuration:
     api-key: $STRIPE_API_KEY
 ```
 
+Configuration values that are assigned with a reference to the Secret must be of type `string`.
+
 ### Secrets Deployment
 
 Secrets are not being deployed with context
@@ -191,13 +185,19 @@ deployment ([`toa deploy`](../../runtime/cli/readme.md#deploy)),
 thus must be deployed separately at least once for each deployment environment
 manually ([`toa conceal`](../../runtime/cli/readme.md#conceal)).
 
-## Operation Context
+Deployed kubernetes secret's name is predefined as `configuration`.
+
+```shell
+$ toa conceal configuration STRIPE_API_KEY xxxxxxxx
+```
 
-Configuration Value is available as a well-known operation context extension `configuration`.
+## Aspect
 
-### Usage: node
+Configuration Value is available as a well-known operation Aspect `configuration`.
 
 ```javascript
+// Node.js bridge
+
 function transition (input, entity, context) {
   const foo = context.configiuration.foo
 
@@ -207,25 +207,50 @@ function transition (input, entity, context) {
 
 > ![Warning](https://img.shields.io/badge/Warning-yellow)<br/>
 > It is strongly **not** recommended to store a copy of value type configuration
-> values outside of operation scope, thus it prevents operation to benefit
+> values outside operation scope, thus it prevents operation to benefit
 > from [hot updates](#).
 >
 > ```javascript
-> // THIS IS WEIRD, BAD AND NOT RECOMMENDED
+> // NOT RECOMMENDED
 > let foo
 > 
 > function transition (input, entity, context) {
+>   // NOT RECOMMENDED
 >   if (foo === undefined) foo = context.configuration.foo
 > 
 >   // ...
 > }
 > ```
-> See [Genuine operations](#).
+> See [Genuine operations](/documentation/design.md#genuine-operations).
+
+## Development Configuration
+
+Configuration can be exported by [`toa env`](/runtime/cli/readme.md#env).
+
+### Local Environment Placeholders
+
+Context Configuration values may contain placeholders that reference environment variables.
+Placeholders are replaced with values if the corresponding environment variables are set.
+
+> Placeholders can only be used with local environment (exported by `toa env`), as these values are not
+> deployed.
+
+```yaml
+# context.toa.yaml
+configuration:
+  dummies.dummy:
+    url@local: https://stage${STAGE}.intranet/
+```
+
+```dotenv
+# .env
+STAGE=82
+```
 
 ## Appendix
 
 - [Discussion](./docs/discussion.md)
 - [Configuration consistency](./docs/consistency.md)
 
-[^1]: Cannot be changed without a deployment. New values are considered to be a subject of
+[^1]: Cannot be changed without a deployment as new values are considered to be a subject of
 testing. [#146](https://github.com/toa-io/toa/issues/146)

package/source/.deployment/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const { variables } = require('./variables')
+const { secrets } = require('./secrets')
+
+exports.variables = variables
+exports.secrets = secrets

package/source/.deployment/secrets.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const find = require('../secrets')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} components
+ * @param {object} annotations
+ * @return {toa.deployment.dependency.Variables}
+ */
+function secrets (components, annotations) {
+  /** @type {toa.deployment.dependency.Variables} */
+  const variables = {}
+
+  for (const [id, annotation] of Object.entries(annotations)) {
+    const component = components.find((component) => component.locator.id === id)
+    const label = component.locator.label
+
+    find.secrets(annotation, (variable, key) => {
+      if (variables[label] === undefined) variables[label] = []
+
+      variables[label].push({
+        name: variable,
+        secret: { name: SECRET_NAME, key }
+      })
+    })
+  }
+
+  return variables
+}
+
+const SECRET_NAME = 'toa-configuration'
+
+exports.secrets = secrets

package/{src/deployment.js → source/.deployment/variables.js}

@@ -3,9 +3,12 @@
 const { encode } = require('@toa.io/generic')
 
 /**
- * @type {toa.deployment.dependency.Constructor}
+ * @param {toa.norm.context.dependencies.Instance[]} components
+ * @param {object} annotations
+ * @return {toa.deployment.dependency.Variables}
  */
-const deployment = (components, annotations) => {
+function variables (components, annotations) {
+  /** @type {toa.deployment.dependency.Variables} */
   const variables = {}
 
   for (const [id, annotation] of Object.entries(annotations)) {
@@ -17,7 +20,7 @@ const deployment = (components, annotations) => {
     }]
   }
 
-  return { variables }
+  return variables
 }
 
-exports.deployment = deployment
+exports.variables = variables

package/{src → source}/.manifest/.normalize/verbose.js

@@ -21,7 +21,7 @@ const convert = (node) => {
 }
 
 function property (node) {
-  if (node === null) throw new Error('Configuration: cannot resolve type of null, use JSONSchema declaration.')
+  if (node === null) return { type: 'null', default: null }
 
   const type = Array.isArray(node) ? 'array' : typeof node
 
@@ -36,11 +36,14 @@ const array = (array) => {
 
   const type = typeof array[0]
 
-  return {
+  const schema = {
     type: 'array',
-    items: { type },
     default: array
   }
+
+  if (array.length === 1) schema.items = array[0]
+
+  return schema
 }
 
 const SYM = Symbol()

package/{src → source}/.manifest/schema.yaml

@@ -2,6 +2,7 @@ $schema: https://json-schema.org/draft/2019-09/schema
 $id: https://schemas.toa.io/0.0.0/extensions/configuration/manifest
 
 $ref: 'https://schemas.toa.io/0.0.0/definitions#/definitions/schema'
+type: object
 properties:
   type:
     const: object

package/source/.provider/env.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const { map } = require('@toa.io/generic')
+
+function env (object) {
+  return map(object,
+    /**
+     * @type {toa.generic.map.transform<string>}
+     */
+    (value) => {
+      if (typeof value !== 'string') return
+
+      return value.replaceAll(RX, (match, variable) => process.env[variable] ?? match)
+    })
+}
+
+const RX = /\${(?<variable>[A-Z0-9_]{1,32})}/g
+
+exports.env = env

package/{src → source}/.provider/form.js

@@ -8,7 +8,7 @@ const { traverse } = require('@toa.io/generic')
  */
 const form = (schema) => {
   const defaults = (node) => {
-    if (node.properties !== undefined) return { ...node.properties }
+    if (node.type === 'object' && node.properties !== undefined) return { ...node.properties }
     if (node.default !== undefined) return node.default
 
     return null

package/source/.provider/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const { form } = require('./form')
+const { env } = require('./env')
+
+exports.form = form
+exports.env = env

package/{test/annotations.fixtures.js → source/annotation.fixtures.js}

@@ -10,6 +10,7 @@ const instance = () => {
   const locator = new Locator(name, namespace)
 
   const manifest = {
+    type: 'object',
     properties: {
       foo: {
         type: 'number',

package/{test/annotations.test.js → source/annotation.test.js}

@@ -4,7 +4,7 @@ const clone = require('clone-deep')
 const { generate } = require('randomstring')
 const { sample } = require('@toa.io/generic')
 
-const fixtures = require('./annotations.fixtures')
+const fixtures = require('./annotation.fixtures')
 const { annotation } = require('../')
 
 let input

package/{test → source}/aspect.fixtures.js

@@ -3,21 +3,20 @@
 const { generate } = require('randomstring')
 
 const schema = {
+  type: 'object',
   properties: {
     foo: {
       type: 'string',
       default: generate()
     },
     bar: {
+      type: 'object',
       properties: {
         baz: {
           type: 'number',
           default: 1
         }
       }
-    },
-    quu: {
-      type: 'number'
     }
   }
 }

package/{test → source}/aspect.test.js

@@ -46,8 +46,7 @@ describe('defaults', () => {
       foo: fixtures.schema.properties.foo.default,
       bar: {
         baz: fixtures.schema.properties.bar.properties.baz.default
-      },
-      quu: 0
+      }
     })
   })
 })

package/source/deployment.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const { merge } = require('@toa.io/generic')
+const get = require('./.deployment')
+
+/**
+ * @param {toa.norm.context.dependencies.Instance[]} components
+ * @param {object} annotations
+ * @return {toa.deployment.dependency.Declaration}
+ */
+const deployment = (components, annotations) => {
+  const variables = get.variables(components, annotations)
+  const secrets = get.secrets(components, annotations)
+
+  merge(variables, secrets)
+
+  return { variables }
+}
+
+exports.deployment = deployment

package/{test → source}/deployment.test.js

@@ -1,9 +1,11 @@
 'use strict'
 
-const { encode } = require('@toa.io/generic')
+const clone = require('clone-deep')
+const { encode, sample } = require('@toa.io/generic')
 
 const fixtures = require('./deployment.fixtures')
 const { deployment } = require('../')
+const { generate } = require('randomstring')
 
 /** @type {toa.deployment.dependency.Declaration} */
 let declaration
@@ -43,3 +45,26 @@ it('should map configurations', () => {
     expect(env.value).toStrictEqual(encoded)
   }
 })
+
+it('should declare secrets', async () => {
+  const annotations = clone(fixtures.annotations)
+  const component = sample(fixtures.components)
+  const id = component.locator.id
+  const key = generate()
+  const name = generate().substring(0, 16).toUpperCase()
+  const value = '$' + name
+
+  if (annotations[id] === undefined) annotations[id] = {}
+
+  annotations[id][key] = value
+
+  declaration = deployment(fixtures.components, annotations)
+
+  const variables = declaration.variables[component.locator.label]
+
+  expect(variables).toBeDefined()
+
+  const secret = variables.find((variable) => variable.name === 'TOA_CONFIGURATION__' + name)
+
+  expect(secret).toBeDefined()
+})

package/{src → source}/factory.js

@@ -11,11 +11,11 @@ const { Provider } = require('./provider')
 class Factory {
   /**
    * @param {toa.core.Locator} locator
-   * @param {toa.schema.JSON | Object} declaration
+   * @param {toa.schema.JSON | Object} annotation
    * @return {toa.extensions.configuration.Aspect}
    */
-  aspect (locator, declaration) {
-    const schema = new Schema(declaration)
+  aspect (locator, annotation) {
+    const schema = new Schema(annotation)
     const provider = new Provider(locator, schema)
     const configuration = new Configuration(provider)
 

package/{test → source}/manifest.test.js

@@ -109,9 +109,6 @@ describe('normalization', () => {
       properties: {
         foo: {
           type: 'array',
-          items: {
-            type: 'number'
-          },
           default: [1, 2, 3]
         }
       }
@@ -123,10 +120,4 @@ describe('normalization', () => {
 
     expect(() => manifest(concise)).toThrow(/array items type because it's empty/)
   })
-
-  it('should throw on null', () => {
-    const concise = { foo: null }
-
-    expect(() => manifest(concise)).toThrow(/type of null/)
-  })
 })

package/{src → source}/provider.js

@@ -4,7 +4,9 @@ const clone = require('clone-deep')
 const { decode, encode, empty, overwrite } = require('@toa.io/generic')
 
 const { Connector } = require('@toa.io/core')
-const { form } = require('./.provider/form')
+
+const { secrets } = require('./secrets')
+const { env, form } = require('./.provider')
 
 /**
  * @implements {toa.extensions.configuration.Provider}
@@ -39,11 +41,7 @@ class Provider extends Connector {
   }
 
   async open () {
-    await this.#retrieve()
-  }
-
-  async #source () {
-    return this.#value
+    this.#retrieve()
   }
 
   set (key, value) {
@@ -77,7 +75,11 @@ class Provider extends Connector {
     return this.object === undefined ? undefined : encode(this.object)
   }
 
-  async #retrieve () {
+  #source () {
+    return this.#value
+  }
+
+  #retrieve () {
     const string = process.env[this.key]
     const object = string === undefined ? {} : decode(string)
 
@@ -85,18 +87,14 @@ class Provider extends Connector {
   }
 
   #set (object) {
-    this.#validate(object)
+    object = this.#reveal(object)
+    object = env(object)
+
     this.#merge(object)
 
     this.object = empty(object) ? undefined : object
   }
 
-  #validate (object) {
-    const error = this.#schema.match(object)
-
-    if (error !== null) throw new TypeError(error.message)
-  }
-
   #merge (object) {
     object = clone(object)
 
@@ -104,9 +102,18 @@ class Provider extends Connector {
     const value = overwrite(form, object)
 
     this.#schema.validate(value)
-
     this.#value = value
   }
+
+  #reveal (object) {
+    return secrets(object, (variable) => {
+      if (!(variable in process.env)) throw new Error(`Configuration secret value ${variable} is not set`)
+
+      const base64 = process.env[variable]
+
+      return decode(base64)
+    })
+  }
 }
 
 const PREFIX = 'TOA_CONFIGURATION_'

package/source/provider.test.js

@@ -0,0 +1,130 @@
+'use strict'
+
+/* eslint-disable no-template-curly-in-string */
+
+const { generate } = require('randomstring')
+const { encode } = require('@toa.io/generic')
+
+const { Provider } = require('./provider')
+
+it('should be', async () => {
+  expect(Provider).toBeInstanceOf(Function)
+})
+
+const locator = /** @type {toa.core.Locator} */ { uppercase: generate().toUpperCase() }
+const schema = /** @type {toa.schema.Schema} */ { validate: () => undefined }
+
+/** @type {Provider} */
+let provider
+
+beforeEach(() => {
+  cleanEnv()
+  provider = new Provider(locator, schema)
+})
+
+it('should replace secret values', async () => {
+  const configuration = { foo: '$FOO_SECRET' }
+  const secrets = { FOO_SECRET: generate() }
+
+  setEnv(configuration, secrets)
+
+  await provider.open()
+  const value = provider.source()
+
+  expect(value).toStrictEqual({ foo: secrets.FOO_SECRET })
+})
+
+it('should throw if secret value is not set', async () => {
+  const configuration = { foo: '$FOO_SECRET' }
+
+  setEnv(configuration)
+
+  await expect(provider.open()).rejects.toThrow('FOO_SECRET is not set')
+})
+
+it('should replace nested secrets', async () => {
+  const configuration = { foo: { bar: '$BAR' } }
+  const secrets = { BAR: generate() }
+
+  setEnv(configuration, secrets)
+
+  await provider.open()
+  const value = provider.source()
+
+  expect(value).toStrictEqual({ foo: { bar: secrets.BAR } })
+})
+
+it('should replace placeholders', async () => {
+  const name = 'FOO_VALUE'
+  const configuration = { foo: { bar: 'foo_${' + name + '}' } }
+  const value = generate()
+
+  setEnv(configuration)
+  setVal(name, value)
+
+  await provider.open()
+  const source = provider.source()
+
+  expect(source).toStrictEqual({ foo: { bar: 'foo_' + value } })
+})
+
+it('should replace multiple placeholders', async () => {
+  const configuration = { foo: '${FOO} ${BAR}' }
+
+  setEnv(configuration)
+  setVal('FOO', 'hello')
+  setVal('BAR', 'world')
+
+  await provider.open()
+  const source = provider.source()
+
+  expect(source).toStrictEqual({ foo: 'hello world' })
+})
+
+it('should not replace if variable not set', async () => {
+  const configuration = { foo: '${FOO}' }
+
+  setEnv(configuration)
+
+  await provider.open()
+  const value = provider.source()
+
+  expect(value).toStrictEqual(configuration)
+})
+
+const usedVariables = []
+
+/**
+ * @param {object} configuration
+ * @param {Record<string, string>} [secrets]
+ */
+function setEnv (configuration, secrets) {
+  const variable = PREFIX + locator.uppercase
+  const encoded = encode(configuration)
+
+  setVal(variable, encoded)
+
+  if (secrets !== undefined) {
+    for (const [key, value] of Object.entries(secrets)) {
+      const variable = PREFIX + '_' + key
+
+      process.env[variable] = encode(value)
+      usedVariables.push(variable)
+    }
+  }
+}
+
+function setVal (variable, value) {
+  process.env[variable] = value
+  usedVariables.push(variable)
+}
+
+function cleanEnv () {
+  for (const variable of usedVariables) {
+    delete process.env[variable]
+  }
+
+  usedVariables.length = 0
+}
+
+const PREFIX = 'TOA_CONFIGURATION_'

package/source/secrets.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { map } = require('@toa.io/generic')
+
+/**
+ * @param {object} configuration
+ * @param {(variable: string, name?: string) => void} callback
+ * @returns {object}
+ */
+function secrets (configuration, callback) {
+  return map(configuration, (value) => {
+    if (typeof value !== 'string') return
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) return
+
+    const name = match.groups.variable
+    const variable = PREFIX + name
+
+    return callback(variable, name)
+  })
+}
+
+const PREFIX = 'TOA_CONFIGURATION__'
+const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+
+exports.secrets = secrets

package/source/secrets.test.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const { secrets } = require('./secrets')
+
+it('should be', async () => {
+  expect(secrets).toBeInstanceOf(Function)
+})
+
+it('should find secrets', async () => {
+  const configuration = {
+    foo: {
+      bar: '$BAR_VALUE'
+    },
+    baz: '$BAZ_VALUE'
+  }
+
+  const variables = new Set()
+  const names = new Set()
+
+  secrets(configuration, (variable, name) => {
+    variables.add(variable)
+    names.add(name)
+  })
+
+  expect(variables.has('TOA_CONFIGURATION__BAR_VALUE')).toStrictEqual(true)
+  expect(variables.has('TOA_CONFIGURATION__BAZ_VALUE')).toStrictEqual(true)
+
+  expect(names.has('BAR_VALUE')).toStrictEqual(true)
+  expect(names.has('BAZ_VALUE')).toStrictEqual(true)
+})
+
+it('should replace values', async () => {
+  const configuration = { foo: '$FOO' }
+
+  const output = secrets(configuration, (variable) => 'hello')
+
+  expect(output).toStrictEqual({ foo: 'hello' })
+})
+
+it('should allow numbers in secret names', async () => {
+  const configuration = { foo: '$HOST_0' }
+  const found = new Set()
+
+  secrets(configuration, (variable) => found.add(variable))
+
+  expect(found.has('TOA_CONFIGURATION__HOST_0')).toStrictEqual(true)
+})