@toa.io/extensions.configuration 1.0.0-alpha.29 → 1.0.0-alpha.291

package/readme.md

@@ -11,8 +11,12 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
   defaults:
     foo: bar
     bar: 1
@@ -21,7 +25,7 @@ configuration:
 ### Use
 
 ```javascript
-function transition (input, entity, context) {
+function transition(input, entity, context) {
   const { foo, bar } = context.configuration
 
   // ...
@@ -34,9 +38,9 @@ function transition (input, entity, context) {
 # context.toa.yaml
 configuration:
   dummies.dummy:
-    foo: qux          # override default value
+    foo: qux # override default value
     foo@staging: quux # deployment environment discriminator
-    bar: $BAZ_VALUE   # secret
+    bar: $BAZ_VALUE # secret
 ```
 
 ### Deploy secrets
@@ -45,22 +49,28 @@ configuration:
 $ toa conceal configuration BAZ_VALUE=$ecr3t
 ```
 
----
+### Change at runtime
 
-## Problem Definition
+```http
+POST /configuration/values/dummies.dummy/ HTTP/1.1
+authorization: Token ...
+content-type: application/yaml
 
-- Components should be runnable in different deployment environments.
-- Some algorithm's parameters should be deployed secretly.
-- Components should be reusable in different contexts.
+configuration:
+  foo: quux
+  bar: 2
+```
+
+---
 
 ## Manifest
 
-Component's configuration is declared using `configuration` manifest,
-containing `schema` and optionnaly `defaults` properties.
+Component's configuration is declared using the `configuration` manifest, containing `schema`
+and optionally `defaults` properties.
 
 ### Schema
 
-Configuration schema is declared with [COS](/libraries/concise).
+Configuration schema is declared with [JSON Schema](https://json-schema.org).
 
 ```yaml
 # manifest.toa.yaml
@@ -69,30 +79,18 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
-```
-
-> Introducing non-backward compatible changes to a configuration schema will result in a loss of
-> compatibility with existing contexts and deployment environments.
-> Therefore, configuration schema changes are subject to component versioning.
-
-If `configuration` object doesn't contain property `schema`, then it is considered to be schema.
-
-```yaml
-# manifest.toa.yaml
-name: dummy
-namespace: dummies
-
-configuration:
-  foo: string
-  bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
 ```
 
 ### Defaults
 
-The default configuration value can be provided using the `defaults` property, which should conform
-to the configuration schema.
+The default configuration value can be provided using the `defaults` property, which should
+conform to the configuration schema.
 
 ```yaml
 # manifest.toa.yaml
@@ -101,27 +99,22 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
   defaults:
     foo: hello
     bar: 0
 ```
 
-#### Schema defaults hint
+### Epoch
 
-The configuration schema itself can contain default primitive values using the COS syntax.
-
-```yaml
-# manifest.toa.yaml
-name: dummy
-namespace: dummies
-
-configuration:
-  schema:
-    foo: hello
-    bar: 0
-```
+The configuration epoch of a component is the SHA-256 of its configuration schema, as
+canonical JSON. A configuration object belongs to the epoch of the schema it was validated
+against. A schema change is a new epoch.
 
 ## Annotation
 
@@ -136,9 +129,26 @@ configuration:
     bar@staging: 2
 ```
 
+The annotated object is deployed as the defaults of the component for its epoch, in place of
+the manifest `defaults`.
+
+Every key names a component except `resources`, which is the values service's own: it deploys
+like any other and states what it may take. A component actually named `resources` is written
+with its namespace, `default.resources` — the bare form is only shorthand for that.
+
+```yaml
+# context.toa.yaml
+configuration:
+  resources:
+    cpu: [200m, 1000m]
+    memory: [200Mi, 500Mi]
+  dummies.dummy:
+    foo: bye
+```
+
 ## Secrets
 
-Configuration annotation top-level values which are uppercase strings prefixed with `$` considered as secrets.
+Configuration values which are uppercase strings prefixed with `$` are considered as secrets.
 
 ```yaml
 # context.toa.yaml
@@ -158,14 +168,118 @@ Deployed kubernetes secret's name is predefined as `configuration`.
 $ toa conceal configuration STRIPE_API_KEY=xxxxxxxx
 ```
 
+A secret is substituted in the component's process, from the variable
+`TOA_CONFIGURATION__<NAME>` deployed to it — a variable the component's code cannot read.
+The values service holds and returns the reference.
+
+In the component, a secret is a `Secret` object: `unwrap()` returns the string, while
+`toString()`, JSON and `util.inspect` give `<REDACTED>`.
+
+```javascript
+function transition(input, entity, context) {
+  const key = context.configuration.apiKey.unwrap()
+
+  // ...
+}
+```
+
+A value a component reads as a secret is given as a reference: a plain string in its place
+has no `unwrap`.
+
+## Values
+
+Configuration values are held by the `configuration.values` component, deployed as the
+`configuration-values` service. The service is deployed with the variable
+`TOA_CONFIGURATION_VALUES`: the epoch, the schema and the defaults of every component
+declaring configuration.
+
+```json
+{
+  "dummies.dummy": {
+    "epoch": "3f2a…",
+    "schema": { "type": "object", "properties": { "foo": { "type": "string" } } },
+    "defaults": { "foo": "bye" }
+  }
+}
+```
+
+Configuration objects are immutable. Creating a configuration is creating a new object for
+the component's current epoch; the latest object for a component and an epoch is the last
+created one. Each object records its `originator`.
+
+The configuration of a component for an epoch is:
+
+1. The latest object created for the component and the epoch;
+2. Otherwise, the deployed defaults, if the epoch is the deployed one;
+3. Otherwise, none.
+
+### Operations
+
+- `get({ component, epoch? })`: the configuration with the schema it is checked against, as
+  `{ configuration, schema, epoch }`, or `null` when there is none. The epoch is the deployed
+  one when omitted; an epoch the deployment does not know has no `schema`.
+- `fetch([{ component, epoch }])`: the same for several pairs at once, as
+  `[{ component, epoch, configuration }]`.
+- `list()`: every component's configuration for its deployed epoch, by component name, as
+  `[{ component, epoch, schema, configuration }]`.
+- `create({ component, configuration, originator })`: a new object for the component's
+  deployed epoch. The configuration must satisfy the schema. Errors: `UNKNOWN_COMPONENT`,
+  `INVALID_CONFIGURATION`.
+
+Creating a configuration publishes the `configuration.values.created` event with the
+object as stored.
+
+### Resources
+
+| Method | Path                                | Role                          |
+| ------ | ----------------------------------- | ----------------------------- |
+| `GET`  | `/configuration/values/`            | `system:configuration:get`    |
+| `GET`  | `/configuration/values/:component/` | `system:configuration:get`    |
+| `POST` | `/configuration/values/:component/` | `system:configuration:create` |
+
+`GET /configuration/values/` lists every component's configuration for its deployed epoch, by
+component name, as `[{ component, epoch, schema, configuration }]`.
+
+`GET /configuration/values/:component/` returns `{ configuration, schema, epoch }` for the
+deployed epoch, `404` when there is none.
+
+`POST` takes `{ configuration }`, records the Identity as the `originator`, and returns
+`{ id, epoch }`. A configuration not satisfying the schema, or an unknown component, is
+`422`.
+
+## UI
+
+The values service serves a page listing the configured components and creating
+configurations, mounted at `/.configuration` on port `8003`. Reading it needs the
+`system:configuration:get` role, creating needs `system:configuration:create`.
+
+The page is always published: unlike the introspection annotation, the configuration
+annotation is the per-component values map and has nowhere to carry a switch.
+
 ## Aspect
 
 Component's configuration values are available as a well-known Aspect `configuration`.
 
 ```javascript
-function transition (input, entity, context) {
-  const foo = context.configiuration.foo
+function transition(input, entity, context) {
+  const foo = context.configuration.foo
 
   // ...
 }
 ```
+
+On start, a component requests its configuration for its epoch from the values service and
+waits until there is one, reporting every fifth attempt. The schema is applied, and secrets
+are substituted. What is served is what was stored, whole: a `default` written into the schema
+fills nothing, so a value every component is to have is declared in `defaults`. After a configuration is created, the running component receives the new
+object and takes it when its `CREATED` is later than that of the value it holds.
+
+### Local override
+
+When the variable `TOA_CONFIGURATION_<NAMESPACE>_<NAME>` is set, the component's
+configuration is the variable's value with the manifest `defaults` filled in and the schema
+applied, and the values service is not used.
+
+```shell
+$ TOA_CONFIGURATION_DUMMIES_DUMMY='{"foo":"local"}' toa run components/dummy
+```

package/schemas/annotation.cos.yaml

@@ -1 +1,29 @@
-<object>
+# context.toa.yaml configuration annotation
+#
+# Every key names a component and holds that component's values, except `resources`, which
+# is the values service's own: it deploys like any other and has to state what it may take.
+# A component actually named `resources` is written with its namespace, `default.resources`
+# — the bare form is only shorthand for that.
+properties:
+  resources:
+    # `null` deploys the service without any, which every deployment has to state one way
+    # or the other — see `migrations/262.md`.
+    type: object
+    nullable: true
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+patternProperties:
+  ^(?!resources$).+$:
+    type: object
+type: object

package/schemas/manifest.cos.yaml

@@ -1,2 +1,9 @@
-schema: object
-defaults?: object
+properties:
+  schema:
+    type: object
+  defaults:
+    type: object
+additionalProperties: false
+type: object
+required:
+  - schema

package/source/Aspect.test.ts

@@ -1,15 +1,119 @@
-import { Aspect } from './Aspect'
+import { it, beforeEach, afterEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+import { isDeepStrictEqual } from 'node:util'
 
-it('should return value', async () => {
-  const configuration = {
-    foo: 'bar',
-    bar: {
-      baz: 'quux'
+import { Connector, Locator } from '@toa.io/core'
+import { generate } from 'randomstring'
+import { Aspect } from './Aspect.js'
+import type { Client, Listener } from './Client.js'
+import type { Manifest } from './manifest.js'
+
+class Fake extends Connector {
+  public readonly fetch = mock.fn(async () => ({
+    configuration: { foo: 'served' },
+    created: 5
+  }))
+  public readonly subscribe = mock.fn()
+  public readonly unsubscribe = mock.fn()
+}
+
+const manifest: Manifest = {
+  schema: {
+    type: 'object',
+    properties: {
+      foo: { type: 'string' },
+      bar: { type: 'object', properties: { baz: { type: 'string' } } }
     }
-  }
+  },
+  defaults: { bar: { baz: 'quux' } }
+}
+
+let locator: Locator
+
+beforeEach(() => {
+  locator = new Locator(generate(), generate())
+})
+
+afterEach(() => {
+  delete process.env['TOA_CONFIGURATION_' + locator.uppercase]
+})
+
+it('should be named', async () => {
+  assert.deepStrictEqual(new Aspect(locator, manifest, null).name, 'configuration')
+})
+
+it('should resolve locally without a client', async () => {
+  process.env['TOA_CONFIGURATION_' + locator.uppercase] = JSON.stringify({ foo: 'local' })
+
+  const aspect = new Aspect(locator, manifest, null)
+
+  await aspect.connect()
+
+  assert.deepStrictEqual(aspect.invoke(), { foo: 'local', bar: { baz: 'quux' } })
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'local')
+  assert.deepStrictEqual(aspect.invoke(['bar', 'baz']), 'quux')
+})
+
+it('should fetch from the client and follow it', async () => {
+  const client = new Fake()
+  const aspect = new Aspect(locator, manifest, client as unknown as Client)
+
+  await aspect.connect()
+
+  assert.deepStrictEqual(client.connected, true)
+  assert.strictEqual(client.fetch.mock.callCount(), 1)
+
+  const [component, epoch] = client.fetch.mock.calls[0].arguments as unknown as [
+    string,
+    string
+  ]
+
+  assert.deepStrictEqual(component, locator.id)
+  assert.match(epoch, /^[a-f0-9]{64}$/)
+  // what the service serves is what was stored, and the manifest defaults are not applied to it
+  assert.deepStrictEqual(aspect.invoke(), { foo: 'served' })
+
+  assert.ok(
+    client.subscribe.mock.calls.some(
+      (call: any) =>
+        call.arguments.length === 3 &&
+        isDeepStrictEqual(call.arguments[0], component) &&
+        isDeepStrictEqual(call.arguments[1], epoch) &&
+        typeof call.arguments[2] === 'function'
+    )
+  )
+
+  const listener = client.subscribe.mock.calls[0].arguments[2] as Listener
+
+  listener({ configuration: { foo: 'updated' }, created: 6 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // what is not newer than the held value is left alone
+  listener({ configuration: { foo: 'stale' }, created: 6 })
+  listener({ configuration: { foo: 'older' }, created: 4 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // a value that does not fit keeps the previous one
+  listener({ configuration: { foo: { nested: true } }, created: 7 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // and the one after it still applies
+  listener({ configuration: { foo: 'latest' }, created: 8 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'latest')
 
-  const aspect = new Aspect(configuration)
+  await aspect.disconnect()
 
-  expect(aspect.invoke(['foo'])).toStrictEqual(configuration.foo)
-  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual(configuration.bar.baz)
+  assert.ok(
+    client.unsubscribe.mock.calls.some(
+      (call: any) =>
+        call.arguments.length === 3 &&
+        isDeepStrictEqual(call.arguments[0], component) &&
+        isDeepStrictEqual(call.arguments[1], epoch) &&
+        isDeepStrictEqual(call.arguments[2], listener)
+    )
+  )
 })

package/source/Aspect.ts

@@ -1,23 +1,91 @@
-import { Connector, type extensions } from '@toa.io/core'
+import { console } from 'openspan'
+import { Connector, type Locator } from '@toa.io/core'
+import type { extensions } from '@toa.io/core/types'
+import { fit, local, type Node } from './configuration.js'
+import { epoch } from './epoch.js'
+import type { Client, Value } from './Client.js'
+import type { Manifest } from './manifest.js'
 
 export class Aspect extends Connector implements extensions.Aspect {
   public readonly name = 'configuration'
 
-  private readonly value: object
+  private readonly locator: Locator
+  private readonly manifest: Manifest
+  private readonly client: Client | null
+  private readonly epoch: string
+  private value: Node = {}
+  private created = 0
 
-  public constructor (value: object) {
+  /**
+   * Without a client the value is local: the variable, the defaults and the schema.
+   * With one, the value is what the service holds, and it follows the service.
+   */
+  public constructor(locator: Locator, manifest: Manifest, client: Client | null) {
     super()
 
-    this.value = value
+    this.locator = locator
+    this.manifest = manifest
+    this.client = client
+    this.epoch = epoch(manifest.schema)
+
+    if (client !== null) this.depends(client)
   }
 
-  public invoke (path: string[]): any {
+  public invoke(path?: string[]): any {
     let cursor: any = this.value
 
-    if (path !== undefined)
-      for (const segment of path)
-        cursor = cursor[segment]
+    if (path !== undefined) for (const segment of path) cursor = cursor[segment]
 
     return cursor
   }
+
+  protected override async open(): Promise<void> {
+    if (this.client === null) {
+      this.value = local(this.locator, this.manifest)
+
+      return
+    }
+
+    const { configuration, created } = await this.client.fetch(
+      this.locator.id,
+      this.epoch
+    )
+
+    this.value = fit(configuration, this.manifest)
+    this.created = created
+    this.client.subscribe(this.locator.id, this.epoch, this.listener)
+
+    console.info('Configuration resolved', {
+      component: this.locator.id,
+      epoch: this.epoch,
+      created
+    })
+  }
+
+  protected override async close(): Promise<void> {
+    this.client?.unsubscribe(this.locator.id, this.epoch, this.listener)
+  }
+
+  private readonly listener = ({ configuration, created }: Value): void => {
+    // deliveries may repeat or cross: only what is newer than the held value replaces it
+    if (created <= this.created) return
+
+    try {
+      this.value = fit(configuration, this.manifest)
+      this.created = created
+
+      console.info('Configuration updated', {
+        component: this.locator.id,
+        epoch: this.epoch,
+        created
+      })
+    } catch (error) {
+      // the service validated it against the schema of its epoch, so the two schemas differ
+      console.error('Configuration value does not match the schema', {
+        component: this.locator.id,
+        epoch: this.epoch,
+        error
+      })
+    }
+  }
 }

package/source/Client.test.ts

@@ -0,0 +1,125 @@
+import { it, beforeEach, afterEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+import { isDeepStrictEqual } from 'node:util'
+
+import { Connector, type Locator, type Receiver } from '@toa.io/core'
+import type { Message } from '@toa.io/core/types'
+import { timeout } from '@toa.io/generic'
+import { Client, type Fetched } from './Client.js'
+import { EVENT } from './const.js'
+import type { Host } from './Factory.js'
+
+class Remote extends Connector {
+  public readonly invoke = mock.fn(
+    async (
+      _endpoint: string,
+      request: { input: Array<{ component: string; epoch: string }> }
+    ) =>
+      request.input.map((pair): Fetched => ({
+        ...pair,
+        configuration: this.values[pair.component] ?? null,
+        created: this.values[pair.component] === undefined ? 0 : 7
+      }))
+  )
+
+  public values: Record<string, object | null> = {}
+}
+
+let remote: Remote
+let receiver: Receiver | null
+let host: Host
+let client: Client
+
+beforeEach(() => {
+  remote = new Remote()
+  receiver = null
+
+  host = {
+    remote: mock.fn(async (locator: Locator) => {
+      assert.deepStrictEqual(locator.id, 'configuration.values')
+
+      return remote
+    }),
+    receive: mock.fn(async (label: string, consumer: Receiver) => {
+      assert.deepStrictEqual(label, EVENT)
+
+      receiver = consumer
+
+      return new Connector()
+    })
+  } as unknown as Host
+
+  client = new Client(host, { base: 10, max: 20, warn: 2 })
+})
+
+afterEach(async () => {
+  await client.disconnect()
+})
+
+it('should send the requests of one tick as one call', async () => {
+  remote.values = { 'a.one': { foo: 1 }, 'a.two': { foo: 2 } }
+
+  await client.connect()
+
+  const [one, two] = await Promise.all([
+    client.fetch('a.one', 'e1'),
+    client.fetch('a.two', 'e2')
+  ])
+
+  assert.deepStrictEqual(one, { configuration: { foo: 1 }, created: 7 })
+  assert.deepStrictEqual(two, { configuration: { foo: 2 }, created: 7 })
+  assert.strictEqual(remote.invoke.mock.callCount(), 1)
+  assert.deepStrictEqual(remote.invoke.mock.calls[0].arguments[1], {
+    input: [
+      { component: 'a.one', epoch: 'e1' },
+      { component: 'a.two', epoch: 'e2' }
+    ]
+  })
+})
+
+it('should keep asking until served', async () => {
+  await client.connect()
+
+  const fetching = client.fetch('a.one', 'e1')
+  const deadline = Date.now() + 1000
+
+  while (remote.invoke.mock.calls.length < 2 && Date.now() < deadline) await timeout(5)
+
+  assert.ok(remote.invoke.mock.calls.length >= 2)
+
+  remote.values = { 'a.one': { foo: 1 } }
+
+  assert.deepStrictEqual((await fetching).configuration, { foo: 1 })
+})
+
+it('should hand a created object to its subscribers', async () => {
+  await client.connect()
+
+  const listener = mock.fn()
+  const other = mock.fn()
+
+  client.subscribe('a.one', 'e1', listener)
+  client.subscribe('a.one', 'e0', other)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 2 }, CREATED: 12 }
+  } satisfies Message)
+
+  assert.ok(
+    listener.mock.calls.some(
+      (call: any) =>
+        call.arguments.length === 1 &&
+        isDeepStrictEqual(call.arguments[0], { configuration: { foo: 2 }, created: 12 })
+    )
+  )
+  assert.strictEqual(other.mock.callCount(), 0)
+  assert.strictEqual(remote.invoke.mock.callCount(), 0)
+
+  client.unsubscribe('a.one', 'e1', listener)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 3 }, CREATED: 13 }
+  } satisfies Message)
+
+  assert.strictEqual(listener.mock.callCount(), 1)
+})