@toa.io/extensions.configuration 0.20.0-dev.34 → 0.20.0-dev.35

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@toa.io/extensions.configuration",
-  "version": "0.20.0-dev.34",
+  "version": "0.20.0-dev.35",
   "description": "Toa Configuration",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
-  "main": "source/index.js",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/toa-io/toa.git"
@@ -15,12 +16,18 @@
   "publishConfig": {
     "access": "public"
   },
-  "dependencies": {
-    "@toa.io/core": "0.20.0-dev.34",
-    "@toa.io/generic": "0.20.0-dev.34",
-    "@toa.io/schema": "0.20.0-dev.34",
-    "@toa.io/yaml": "0.20.0-dev.34",
-    "clone-deep": "4.0.1"
+  "scripts": {
+    "prepublishOnly": "npm run transpile",
+    "transpile": "tsc"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
   },
-  "gitHead": "7035b1985fe9bb844069308a272d061bfbd38bf0"
+  "gitHead": "04d3317272256b98003a414ee1eb2555b65163a5",
+  "dependencies": {
+    "@toa.io/core": "0.20.0-dev.35",
+    "@toa.io/generic": "0.20.0-dev.35",
+    "@toa.io/schemas": "0.20.0-dev.35"
+  }
 }

package/readme.md

@@ -49,9 +49,9 @@ $ toa conceal configuration BAZ_VALUE=$ecr3t
 
 ## Problem Definition
 
-- Components must be reusable in different contexts and deployment environments that are in
-  different configurations.
-- Some algorithm parameters must be deployed secretly.
+- Components should be runnable in different deployment environments.
+- Some algorithm's parameters should be deployed secretly.
+- Components should be reusable in different contexts.
 
 ## Manifest
 
@@ -77,6 +77,18 @@ configuration:
 > compatibility with existing contexts and deployment environments.
 > Therefore, configuration schema changes are subject to component versioning.
 
+If `configuration` object doesn't contain property `schema`, then it is considered to be schema.
+
+```yaml
+# manifest.toa.yaml
+name: dummy
+namespace: dummies
+
+configuration:
+  foo: string
+  bar: number
+```
+
 ### Defaults
 
 The default configuration value can be provided using the `defaults` property, which should conform
@@ -126,7 +138,7 @@ configuration:
 
 ## Secrets
 
-Configuration annotation values which are uppercase strings prefixed with `$` considered as secrets.
+Configuration annotation top-level values which are uppercase strings prefixed with `$` considered as secrets.
 
 ```yaml
 # context.toa.yaml
@@ -136,8 +148,8 @@ configuration:
 ```
 
 Secrets are not being deployed with context
-deployment ([`toa deploy`](/runtime/cli/readme.md#deploy)),
-thus must be deployed separately at least once for each deployment environment
+deployment ([`toa deploy`](/runtime/cli/readme.md#deploy)), thus must be deployed separately at
+least once for each deployment environment
 manually ([`toa conceal`](/runtime/cli/readme.md#conceal)).
 
 Deployed kubernetes secret's name is predefined as `configuration`.
@@ -148,7 +160,7 @@ $ toa conceal configuration STRIPE_API_KEY=xxxxxxxx
 
 ## Aspect
 
-Component's configuration value is available as a well-known Aspect `configuration`.
+Component's configuration values are available as a well-known Aspect `configuration`.
 
 ```javascript
 function transition (input, entity, context) {
@@ -157,23 +169,3 @@ function transition (input, entity, context) {
   // ...
 }
 ```
-
-### Local environment placeholders
-
-Configuration annotation values may contain placeholders that reference environment variables.
-Placeholders are replaced with values if the corresponding environment variables are set.
-
-> Placeholders can only be used with local environment (exported
-> by [`toa env`](/runtime/cli/readme.md#env)), as these values are not deployed.
-
-```yaml
-# context.toa.yaml
-configuration:
-  dummies.dummy:
-    url@local: https://stage${STAGE}.intranet/
-```
-
-```dotenv
-# .env
-STAGE=82
-```

package/schemas/annotation.cos.yaml

@@ -0,0 +1 @@
+<object>

package/schemas/manifest.cos.yaml

@@ -0,0 +1,2 @@
+schema: object
+defaults?: object

package/source/Aspect.test.ts

@@ -0,0 +1,15 @@
+import { Aspect } from './Aspect'
+
+it('should return value', async () => {
+  const configuration = {
+    foo: 'bar',
+    bar: {
+      baz: 'quux'
+    }
+  }
+
+  const aspect = new Aspect(configuration)
+
+  expect(aspect.invoke(['foo'])).toStrictEqual(configuration.foo)
+  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual(configuration.bar.baz)
+})

package/source/Aspect.ts

@@ -0,0 +1,23 @@
+import { Connector, type extensions } from '@toa.io/core'
+
+export class Aspect extends Connector implements extensions.Aspect {
+  public readonly name = 'configuration'
+
+  private readonly value: object
+
+  public constructor (value: object) {
+    super()
+
+    this.value = value
+  }
+
+  public invoke (path: string[]): any {
+    let cursor: any = this.value
+
+    if (path !== undefined)
+      for (const segment of path)
+        cursor = cursor[segment]
+
+    return cursor
+  }
+}

package/source/Factory.ts

@@ -0,0 +1,12 @@
+import { type Locator, type extensions } from '@toa.io/core'
+import { Aspect } from './Aspect'
+import { type Manifest } from './manifest'
+import { get } from './configuration'
+
+export class Factory implements extensions.Factory {
+  public aspect (locator: Locator, manifest: Manifest): extensions.Aspect {
+    const value = get(locator, manifest)
+
+    return new Aspect(value)
+  }
+}

package/source/configuration.test.ts

@@ -0,0 +1,89 @@
+import { encode } from '@toa.io/generic'
+import { Locator } from '@toa.io/core'
+import { generate } from 'randomstring'
+import { get } from './configuration'
+import { type Manifest } from './manifest'
+
+let locator: Locator
+let manifest: Manifest
+
+beforeEach(() => {
+  locator = new Locator(generate(), generate())
+  manifest = { schema: { foo: 'string' } }
+})
+
+afterEach(() => {
+  for (const name of used)
+    process.env[name] = undefined
+
+  used = []
+})
+
+it('should read value', async () => {
+  manifest.schema = { foo: 'string' }
+  const value: object = { foo: generate() }
+
+  set(value)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual(value)
+})
+
+it('should return empty object if no value set', async () => {
+  expect(get(locator, manifest)).toStrictEqual({})
+})
+
+it('should substitute secrets', async () => {
+  const value: object = { foo: '$BAR' }
+
+  set(value)
+  set('bar', '_BAR')
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({ foo: 'bar' })
+})
+
+it('should use defaults', async () => {
+  manifest.schema = { foo: 'string', bar: ['number'], 'baz?': 'string' }
+  manifest.defaults = { foo: 'bar', bar: [1] }
+
+  const values = { bar: [2], baz: 'foo' }
+
+  set(values)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({
+    foo: 'bar',
+    bar: [2],
+    baz: 'foo'
+  })
+})
+
+it('should validate', async () => {
+  manifest.schema = { foo: 'hello', bar: 'number' }
+
+  const values = { bar: 5 }
+
+  set(values)
+
+  const result = get(locator, manifest)
+
+  expect(result).toStrictEqual({
+    foo: 'hello',
+    bar: 5
+  })
+})
+
+function set (value: object | string, key = locator.uppercase): void {
+  const string = typeof value === 'string' ? value : encode(value)
+  const name = 'TOA_CONFIGURATION_' + key
+
+  process.env[name] = string
+
+  used.push(name)
+}
+
+let used: string[] = []

package/source/configuration.ts

@@ -0,0 +1,52 @@
+import { type Locator } from '@toa.io/core'
+import { decode, add } from '@toa.io/generic'
+import * as schemas from '@toa.io/schemas'
+import { PREFIX, SECRET_RX } from './deployment'
+import { type Manifest } from './manifest'
+
+export function get (locator: Locator, manifest: Manifest): Configuration {
+  const values = getConfiguration(locator.uppercase)
+
+  substituteSecrets(values)
+
+  if (manifest.defaults !== undefined) add(values, manifest.defaults)
+
+  const schema = schemas.schema(manifest.schema)
+
+  schema.validate(values)
+
+  return values
+}
+
+function getConfiguration (suffix: string): Configuration {
+  const variable = PREFIX + suffix
+  const string = process.env[variable]
+
+  if (string === undefined) return {}
+  else return decode(string)
+}
+
+function substituteSecrets (configuration: Configuration): void {
+  for (const [key, value] of Object.entries(configuration)) {
+    if (typeof value !== 'string') continue
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) continue
+
+    const name = match.groups?.variable as string
+
+    configuration[key] = getSecret(name)
+  }
+}
+
+function getSecret (name: string): string {
+  const variable = PREFIX + '_' + name
+  const value = process.env[variable]
+
+  if (value === undefined) throw new Error(`${variable} is not set.`)
+
+  return value
+}
+
+export type Configuration = Record<string, any>

package/source/deployment.test.ts

@@ -0,0 +1,21 @@
+import { type Annotation, deployment, type Instance } from './deployment'
+import { type Manifest } from './manifest'
+
+it('should validate annotation', async () => {
+  const wrongType = 'not ok' as unknown as Annotation
+
+  expect(() => deployment([], wrongType)).toThrow('object')
+})
+
+it('should validate values', async () => {
+  const manifest: Manifest = {
+    schema: { foo: 'string', bar: 'number' },
+    defaults: { foo: 'ok', bar: 0 }
+  }
+
+  const locator = { id: 'component' }
+  const instances = [{ manifest, locator }] as unknown as Instance[]
+  const annotation: Annotation = { [locator.id]: { bar: 'not a number' } }
+
+  expect(() => deployment(instances, annotation)).toThrow('number')
+})

package/source/deployment.ts

@@ -0,0 +1,69 @@
+import { type Dependency, type Variable, type Variables } from '@toa.io/operations'
+import * as schemas from '@toa.io/schemas'
+import { encode, overwrite } from '@toa.io/generic'
+import { type Manifest } from './manifest'
+import * as validators from './schemas'
+import type { context } from '@toa.io/norm'
+
+export function deployment (instances: Instance[], annotation: Annotation): Dependency {
+  validators.annotation.validate(annotation)
+
+  const variables: Variables = {}
+
+  for (const instance of instances) {
+    const values = annotation[instance.locator.id]
+
+    if (values === undefined) continue
+
+    validate(instance, values)
+
+    variables[instance.locator.label] = [{
+      name: PREFIX + instance.locator.uppercase,
+      value: encode(values)
+    }]
+
+    const secrets = createSecrets(values)
+
+    variables[instance.locator.label].push(...secrets)
+  }
+
+  return { variables }
+}
+
+function createSecrets (values: object): Variable[] {
+  const secrets: Variable[] = []
+
+  for (const value of Object.values(values)) {
+    if (typeof value !== 'string') continue
+
+    const match = value.match(SECRET_RX)
+
+    if (match === null) continue
+
+    const name = match.groups?.variable as string
+
+    secrets.push({
+      name: PREFIX + '_' + name,
+      secret: {
+        name: 'toa-configuration',
+        key: name
+      }
+    })
+  }
+
+  return secrets
+}
+
+function validate (instace: Instance, values: object): void {
+  const defaults = instace.manifest.defaults ?? {}
+  const configuration = overwrite(defaults, values)
+  const schema = schemas.schema(instace.manifest.schema)
+
+  schema.validate(configuration)
+}
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+export type Annotation = Record<string, object>
+export type Instance = context.Dependency<Manifest>

package/source/index.ts

@@ -0,0 +1,3 @@
+export { manifest } from './manifest'
+export { deployment } from './deployment'
+export { Factory } from './Factory'

package/source/manifest.test.ts

@@ -0,0 +1,15 @@
+import { type Manifest, manifest } from './manifest'
+
+it('should validate', async () => {
+  const additional = { schema: {}, foo: 'bar' } as unknown as Manifest
+
+  expect(() => {
+    manifest(additional)
+  }).toThrow('additional')
+
+  const wrongType = { schema: 'not ok' } as unknown as Manifest
+
+  expect(() => {
+    manifest(wrongType)
+  }).toThrow('object')
+})

package/source/manifest.ts

@@ -0,0 +1,15 @@
+import * as schemas from './schemas'
+import { type Configuration } from './configuration'
+
+export function manifest (manifest: Manifest): Manifest {
+  if (manifest.schema === undefined) manifest = { schema: manifest }
+
+  schemas.manifest.validate(manifest)
+
+  return manifest
+}
+
+export interface Manifest {
+  schema: object
+  defaults?: Configuration
+}

package/source/schemas.ts

@@ -0,0 +1,8 @@
+import { resolve } from 'node:path'
+import * as schemas from '@toa.io/schemas'
+
+const path = resolve(__dirname, '../schemas')
+const namespace = schemas.namespace(path)
+
+export const manifest = namespace.schema('manifest')
+export const annotation = namespace.schema('annotation')

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./transpiled"
+  },
+  "include": [
+    "source"
+  ]
+}

package/docs/discussion.md

@@ -1,109 +0,0 @@
-# Discussion
-
-## Change Requests
-
-- [x] feat(configuration): add configuration extension
-    - manifest (schema) validation
-    - context extension
-- [x] feat(formation): add well-known extension 'configuration'
-    - component
-    - context
-- [x] feat(node): add well-known context extension 'configuration'
-- [x] feat(configuration): add concise declarations
-- [x] feat(configuration): add runtime configuration resolution
-- [x] feat(cli): add `toa configure <key> <value> --reset`
-    - validate type
-- [x] feat(operations): add configuration deployment
-    - annotations (values) validation
-- [ ] feat(configuration): add secrets resolution
-- [ ] feat(operations): add secrets deployment
-- [ ] feat(cli): add `toa conceal`
-    - validate type
-- [ ] feat(cli): add `toa configure`
-    - prompt required values
-    - use JSON Schema title
-
-## Statements
-
-### Common
-
-- Secrets are being deployed separately by `toa conceal` command
-
-### 1: Environment variables
-
-- Configuration values and secrets are mapped as environment variables to composition deployments
-- Extensions may expose *deployment mutators*, which are able to modify deployment declaration
-- Configuration context extension reads environment variables to resolve configuration and secrets
-
-### 2: Dedicated Components
-
-- Hot updates
-- [Configuration consistency](consistency.md)
-
-## Questions
-
-### Where are values comes from?
-
-Environment variables.
-
-### Is there a configuration service or configuration component?
-
-No. It will be implemented later as a part of [consistent configuration](consistency.md).
-
-### How are configuration values being stored?
-
-As a kubernetes secrets mapped as environment variables.
-
-### Where are secrets being stored and how do they resolve to value?
-
-As a kubernetes secrets mapped as environment variables.
-
-### Is configuration a single environment variable or a set (one per component)?
-
-#### Context Configuration
-
-In later versions, context extension will resolve configuration values by component locator. Given
-that it is yet
-unknown when this will happen, a certain context might have appeared which configuration is big
-enough to not fit the
-environment variable limitations.
-
-That is, Context Configuration must be mapped as a set of environment variables (one per component).
-Values are
-serialized Configuration Objects.
-
-> This will also allow to configure local environment per component.
-
-#### Secrets
-
-Secrets are mapped per secret as they are not bound to components.
-
-### Is configuration a single kubernetes secret or a set (one per component)?
-
-#### Configuration
-
-Single secret with a set of values per component.
-
-#### Secrets
-
-Once kubernetes secret per configuration secret.
-
-### Is there an option to configure local environment?
-
-<dl>
-<dt><code>toa configure &lt;component&gt;</code></dt>
-<dd>Create local environment configuration values</dd>
-</dl>
-
-### Whose responsibility is to call annotations?
-
-- norm
-- deployment
-
-`toa export context` should throw errors if context has invalid annotations, and it's not a part of
-the deployment.
-
-## References
-
-- [#125](https://github.com/toa-io/toa/issues/125)
-- [#132](https://github.com/toa-io/toa/issues/132)

package/source/.deployment/index.js

@@ -1,7 +0,0 @@
-'use strict'
-
-const { variables } = require('./variables')
-const { secrets } = require('./secrets')
-
-exports.variables = variables
-exports.secrets = secrets

package/source/.deployment/secrets.js

@@ -1,35 +0,0 @@
-'use strict'
-
-const find = require('../secrets')
-
-/**
- * @param {toa.norm.context.dependencies.Instance[]} components
- * @param {object} annotations
- * @return {toa.deployment.dependency.Variables}
- */
-function secrets (components, annotations) {
-  if (annotations === undefined) return {}
-
-  /** @type {toa.deployment.dependency.Variables} */
-  const variables = {}
-
-  for (const [id, annotation] of Object.entries(annotations)) {
-    const component = components.find((component) => component.locator.id === id)
-    const label = component.locator.label
-
-    find.secrets(annotation, (variable, key) => {
-      if (variables[label] === undefined) variables[label] = []
-
-      variables[label].push({
-        name: variable,
-        secret: { name: SECRET_NAME, key }
-      })
-    })
-  }
-
-  return variables
-}
-
-const SECRET_NAME = 'toa-configuration'
-
-exports.secrets = secrets

package/source/.deployment/variables.js

@@ -1,23 +0,0 @@
-'use strict'
-
-const { encode } = require('@toa.io/generic')
-
-function variables (components, annotations) {
-  if (annotations === undefined) return {}
-
-  /** @type {toa.deployment.dependency.Variables} */
-  const variables = {}
-
-  for (const [id, annotation] of Object.entries(annotations)) {
-    const component = components.find((component) => component.locator.id === id)
-
-    variables[component.locator.label] = [{
-      name: 'TOA_CONFIGURATION_' + component.locator.uppercase,
-      value: encode(annotation)
-    }]
-  }
-
-  return variables
-}
-
-exports.variables = variables