@toa.io/extensions.configuration 1.0.0-alpha.273 → 1.0.0-alpha.274

package/features/steps/UI.ts

@@ -0,0 +1,88 @@
+import assert from 'node:assert'
+import * as http from 'node:http'
+import { resolve } from 'node:path'
+import { after, binding, given, then, when } from 'cucumber-tsflow'
+import { UI } from '../../source/UI'
+import { UI_PORT } from '../../source/const'
+
+/**
+ * The scenarios are about the server, not about the page: it is pointed at a fixture
+ * directory, so that running them never needs a UI build.
+ */
+@binding()
+export class Site {
+  private server: UI | null = null
+  private response: Response | null = null
+
+  @given('the UI is published')
+  public async publish (): Promise<void> {
+    this.server = new UI(UI_PORT, resolve(__dirname, '..', 'site'))
+
+    await this.server.connect()
+  }
+
+  @when('{string} is requested')
+  public async request (path: string): Promise<void> {
+    this.response = await get(path)
+  }
+
+  @then('the status is {int}')
+  public statusIs (status: number): void {
+    assert.equal(this.response?.status, status)
+  }
+
+  @then('the body contains {string}')
+  public bodyContains (text: string): void {
+    assert.ok(this.response?.body.includes(text),
+      `Expected the body to contain '${text}', got '${this.response?.body}'`)
+  }
+
+  @then('the body is empty')
+  public bodyIsEmpty (): void {
+    assert.equal(this.response?.body, '')
+  }
+
+  @then('the {string} header is {string}')
+  public headerIs (name: string, value: string): void {
+    assert.equal(this.response?.headers[name], value)
+  }
+
+  @after()
+  public async shutdown (): Promise<void> {
+    await this.server?.disconnect()
+
+    this.server = null
+    this.response = null
+  }
+}
+
+/**
+ * Raw `http`, not `fetch`: the URL parser normalizes `%2e%2e` away, and one of the
+ * scenarios is about exactly that segment reaching the server.
+ */
+async function get (path: string): Promise<Response> {
+  return await new Promise((resolve, reject) => {
+    // `agent: false` — a keep-alive socket outlives the scenario that opened it
+    const request = http.get({ port: UI_PORT, path, agent: false }, (response) => {
+      let body = ''
+
+      response.setEncoding('utf8')
+      response.on('data', (chunk: string) => (body += chunk))
+      response.on('end', () => {
+        resolve({
+          status: response.statusCode!,
+          headers: response.headers as Record<string, string>,
+          body
+        })
+      })
+    })
+
+    request.on('error', reject)
+  })
+}
+
+interface Response {
+  status: number
+  headers: Record<string, string>
+  body: string
+}

package/features/steps/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "noEmit": true,
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true
+  }
+}

package/features/ui.feature

@@ -0,0 +1,85 @@
+@ui
+Feature: Publishing the UI
+
+  The values service serves the UI on its own port, out of the directory `ui` builds. The
+  mount path is baked into the bundle and forwarded by the ingress, so the server routes
+  relative to it.
+
+  Scenario: The root is the way in
+    Given the UI is published
+    When "/" is requested
+    Then the status is 302
+    And the "location" header is "/.configuration/"
+
+  Scenario: The UI answers at its mount path
+    Given the UI is published
+    When "/.configuration" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A trailing slash is the same resource
+    Given the UI is published
+    When "/.configuration/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A query string does not change the resource
+    Given the UI is published
+    When "/.configuration?epoch=e1" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: An unknown route is the page
+    The client router knows what routes there are; this server does not.
+
+    Given the UI is published
+    When "/.configuration/whatever" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A route that looks like a file is still a route
+    A component is named `namespace.component`, and the address of its configuration
+    carries that name — so an extension is no way to tell an asset from a route.
+
+    Given the UI is published
+    When "/.configuration/dummies.dummy/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: And with no trailing slash to give it away
+    Given the UI is published
+    When "/.configuration/dummies.dummy" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A hashed asset is served forever
+    Given the UI is published
+    When "/.configuration/_app/immutable/asset.js" is requested
+    Then the status is 200
+    And the "content-type" header is "text/javascript; charset=utf-8"
+    And the "cache-control" header is "public, max-age=31536000, immutable"
+
+  Scenario: The page itself is never cached
+    Given the UI is published
+    When "/.configuration/" is requested
+    Then the "cache-control" header is "no-cache"
+
+  Scenario: A missing asset is missing
+    Given the UI is published
+    When "/.configuration/_app/immutable/absent.js" is requested
+    Then the status is 404
+
+  Scenario: The path of the values API is not served by the UI
+    Given the UI is published
+    When "/configuration/values/" is requested
+    Then the status is 404
+
+  Scenario: A path that only looks like the mount path is absent
+    Given the UI is published
+    When "/.configurationable" is requested
+    Then the status is 404
+
+  Scenario: Nothing outside the site is reachable
+    Given the UI is published
+    When "/.configuration/%2e%2e/%2e%2e/package.json" is requested
+    Then the status is 404

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.configuration",
-  "version": "1.0.0-alpha.273",
+  "version": "1.0.0-alpha.274",
   "description": "Toa Configuration",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -17,16 +17,23 @@
     "access": "public"
   },
   "scripts": {
-    "transpile": "tsc"
+    "transpile": "tsc && tsc -p ./components/configuration.values",
+    "build:ui": "npm --prefix ui ci && npm --prefix ui run build",
+    "prepack": "npm run build:ui",
+    "features": "npx cucumber-js"
   },
   "jest": {
     "preset": "ts-jest",
-    "testEnvironment": "node"
+    "testEnvironment": "node",
+    "testPathIgnorePatterns": [
+      "<rootDir>/ui"
+    ]
   },
-  "gitHead": "ad3284850ece95ffd5325fd0995707fc144c9c3d",
+  "gitHead": "7df32eb2872155c45681c299036b1efbeefd6518",
   "dependencies": {
     "@toa.io/core": "1.0.0-alpha.273",
     "@toa.io/generic": "1.0.0-alpha.273",
-    "@toa.io/schemas": "1.0.0-alpha.273"
+    "@toa.io/schemas": "1.0.0-alpha.274",
+    "openspan": "1.0.0-alpha.272"
   }
 }

package/readme.md

@@ -49,22 +49,28 @@ configuration:
 $ toa conceal configuration BAZ_VALUE=$ecr3t
 ```
 
----
+### Change at runtime
 
-## Problem Definition
+```http
+POST /configuration/values/dummies.dummy/ HTTP/1.1
+authorization: Token ...
+content-type: application/yaml
 
-- Components should be runnable in different deployment environments.
-- Some algorithm's parameters should be deployed secretly.
-- Components should be reusable in different contexts.
+configuration:
+  foo: quux
+  bar: 2
+```
+
+---
 
 ## Manifest
 
-Component's configuration is declared using `configuration` manifest,
-containing `schema` and optionnaly `defaults` properties.
+Component's configuration is declared using the `configuration` manifest, containing `schema`
+and optionally `defaults` properties.
 
 ### Schema
 
-Configuration schema is declared with [JSONSchema](https://json-schema.org).
+Configuration schema is declared with [JSON Schema](https://json-schema.org).
 
 ```yaml
 # manifest.toa.yaml
@@ -73,34 +79,18 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
-```
-
-> Introducing non-backward compatible changes to a configuration schema will result in a loss of
-> compatibility with existing contexts and deployment environments.
-> Therefore, configuration schema changes are subject to component versioning.
-
-If `configuration` object doesn't contain property `schema`, then it is considered to be schema.
-
-```yaml
-# manifest.toa.yaml
-name: dummy
-namespace: dummies
-
-configuration:
-  type: object
-  properties:
-    foo:
-      type: string
-    bar:
-      type: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
 ```
 
 ### Defaults
 
-The default configuration value can be provided using the `defaults` property, which should conform
-to the configuration schema.
+The default configuration value can be provided using the `defaults` property, which should
+conform to the configuration schema.
 
 ```yaml
 # manifest.toa.yaml
@@ -120,6 +110,12 @@ configuration:
     bar: 0
 ```
 
+### Epoch
+
+The configuration epoch of a component is the SHA-256 of its configuration schema, as
+canonical JSON. A configuration object belongs to the epoch of the schema it was validated
+against. A schema change is a new epoch.
+
 ## Annotation
 
 A component's configuration can be overridden using the configuration context annotation.
@@ -133,9 +129,26 @@ configuration:
     bar@staging: 2
 ```
 
+The annotated object is deployed as the defaults of the component for its epoch, in place of
+the manifest `defaults`.
+
+Every key names a component except `resources`, which is the values service's own: it deploys
+like any other and states what it may take. A component actually named `resources` is written
+with its namespace, `default.resources` — the bare form is only shorthand for that.
+
+```yaml
+# context.toa.yaml
+configuration:
+  resources:
+    cpu: [200m, 1000m]
+    memory: [200Mi, 500Mi]
+  dummies.dummy:
+    foo: bye
+```
+
 ## Secrets
 
-Configuration annotation top-level values which are uppercase strings prefixed with `$` considered as secrets.
+Configuration values which are uppercase strings prefixed with `$` are considered as secrets.
 
 ```yaml
 # context.toa.yaml
@@ -155,14 +168,117 @@ Deployed kubernetes secret's name is predefined as `configuration`.
 $ toa conceal configuration STRIPE_API_KEY=xxxxxxxx
 ```
 
+A secret is substituted in the component's process, from the variable
+`TOA_CONFIGURATION__<NAME>` deployed to it. The values service holds and returns the
+reference.
+
+In the component, a secret is a `Secret` object: `unwrap()` returns the string, while
+`toString()`, JSON and `util.inspect` give `<REDACTED>`.
+
+```javascript
+function transition (input, entity, context) {
+  const key = context.configuration.apiKey.unwrap()
+
+  // ...
+}
+```
+
+A value a component reads as a secret is given as a reference: a plain string in its place
+has no `unwrap`.
+
+## Values
+
+Configuration values are held by the `configuration.values` component, deployed as the
+`configuration-values` service. The service is deployed with the variable
+`TOA_CONFIGURATION_VALUES`: the epoch, the schema and the defaults of every component
+declaring configuration.
+
+```json
+{
+  "dummies.dummy": {
+    "epoch": "3f2a…",
+    "schema": { "type": "object", "properties": { "foo": { "type": "string" } } },
+    "defaults": { "foo": "bye" }
+  }
+}
+```
+
+Configuration objects are immutable. Creating a configuration is creating a new object for
+the component's current epoch; the latest object for a component and an epoch is the last
+created one. Each object records its `originator`.
+
+The configuration of a component for an epoch is:
+
+1. The latest object created for the component and the epoch;
+2. Otherwise, the deployed defaults, if the epoch is the deployed one;
+3. Otherwise, none.
+
+### Operations
+
+- `get({ component, epoch? })`: the configuration with the schema it is checked against, as
+  `{ configuration, schema, epoch }`, or `null` when there is none. The epoch is the deployed
+  one when omitted; an epoch the deployment does not know has no `schema`.
+- `fetch([{ component, epoch }])`: the same for several pairs at once, as
+  `[{ component, epoch, configuration }]`.
+- `list()`: every component's configuration for its deployed epoch, by component name, as
+  `[{ component, epoch, schema, configuration }]`.
+- `create({ component, configuration, originator })`: a new object for the component's
+  deployed epoch. The configuration must satisfy the schema. Errors: `UNKNOWN_COMPONENT`,
+  `INVALID_CONFIGURATION`.
+
+Creating a configuration publishes the `configuration.values.created` event with the
+object as stored.
+
+### Resources
+
+| Method | Path                                  | Role                          |
+|--------|---------------------------------------|-------------------------------|
+| `GET`  | `/configuration/values/`              | `system:configuration:get`    |
+| `GET`  | `/configuration/values/:component/`   | `system:configuration:get`    |
+| `POST` | `/configuration/values/:component/`   | `system:configuration:create` |
+
+`GET /configuration/values/` lists every component's configuration for its deployed epoch, by
+component name, as `[{ component, epoch, schema, configuration }]`.
+
+`GET /configuration/values/:component/` returns `{ configuration, schema, epoch }` for the
+deployed epoch, `404` when there is none.
+
+`POST` takes `{ configuration }`, records the Identity as the `originator`, and returns
+`{ id, epoch }`. A configuration not satisfying the schema, or an unknown component, is
+`422`.
+
+## UI
+
+The values service serves a page listing the configured components and creating
+configurations, mounted at `/.configuration` on port `8003`. Reading it needs the
+`system:configuration:get` role, creating needs `system:configuration:create`.
+
+The page is always published: unlike the introspection annotation, the configuration
+annotation is the per-component values map and has nowhere to carry a switch.
+
 ## Aspect
 
 Component's configuration values are available as a well-known Aspect `configuration`.
 
 ```javascript
 function transition (input, entity, context) {
-  const foo = context.configiuration.foo
+  const foo = context.configuration.foo
 
   // ...
 }
 ```
+
+On start, a component requests its configuration for its epoch from the values service and
+waits until there is one, reporting every fifth attempt. The schema is applied, and secrets
+are substituted. After a configuration is created, the running component receives the new
+object and takes it when its `_created` is later than that of the value it holds.
+
+### Local override
+
+When the variable `TOA_CONFIGURATION_<NAMESPACE>_<NAME>` is set, the component's
+configuration is the variable's value with the manifest `defaults` and the schema applied,
+and the values service is not used.
+
+```shell
+$ TOA_CONFIGURATION_DUMMIES_DUMMY='{"foo":"local"}' toa run components/dummy
+```

package/schemas/annotation.cos.yaml

@@ -1,4 +1,29 @@
-type: object
+# context.toa.yaml configuration annotation
+#
+# Every key names a component and holds that component's values, except `resources`, which
+# is the values service's own: it deploys like any other and has to state what it may take.
+# A component actually named `resources` is written with its namespace, `default.resources`
+# — the bare form is only shorthand for that.
+properties:
+  resources:
+    # `null` deploys the service without any, which every deployment has to state one way
+    # or the other — see `migrations/263.md`.
+    type: object
+    nullable: true
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
 patternProperties:
-  ^.+$:
+  ^(?!resources$).+$:
     type: object
+type: object

package/source/Aspect.test.ts

@@ -1,15 +1,91 @@
+import { Connector, Locator } from '@toa.io/core'
+import { generate } from 'randomstring'
 import { Aspect } from './Aspect'
+import type { Client, Listener } from './Client'
+import type { Manifest } from './manifest'
 
-it('should return value', async () => {
-  const configuration = {
-    foo: 'bar',
-    bar: {
-      baz: 'quux'
+class Fake extends Connector {
+  public readonly fetch = jest.fn(async () => ({ configuration: { foo: 'served' }, created: 5 }))
+  public readonly subscribe = jest.fn()
+  public readonly unsubscribe = jest.fn()
+}
+
+const manifest: Manifest = {
+  schema: {
+    type: 'object',
+    properties: {
+      foo: { type: 'string' },
+      bar: { type: 'object', properties: { baz: { type: 'string' } }, default: { baz: 'quux' } }
     }
   }
+}
+
+let locator: Locator
+
+beforeEach(() => {
+  locator = new Locator(generate(), generate())
+})
+
+afterEach(() => {
+  delete process.env['TOA_CONFIGURATION_' + locator.uppercase]
+})
+
+it('should be named', async () => {
+  expect(new Aspect(locator, manifest, null).name).toStrictEqual('configuration')
+})
+
+it('should resolve locally without a client', async () => {
+  process.env['TOA_CONFIGURATION_' + locator.uppercase] = JSON.stringify({ foo: 'local' })
+
+  const aspect = new Aspect(locator, manifest, null)
+
+  await aspect.connect()
+
+  expect(aspect.invoke()).toStrictEqual({ foo: 'local', bar: { baz: 'quux' } })
+  expect(aspect.invoke(['foo'])).toStrictEqual('local')
+  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual('quux')
+})
+
+it('should fetch from the client and follow it', async () => {
+  const client = new Fake()
+  const aspect = new Aspect(locator, manifest, client as unknown as Client)
+
+  await aspect.connect()
+
+  expect(client.connected).toStrictEqual(true)
+  expect(client.fetch).toHaveBeenCalledTimes(1)
+
+  const [component, epoch] = client.fetch.mock.calls[0] as unknown as [string, string]
+
+  expect(component).toStrictEqual(locator.id)
+  expect(epoch).toMatch(/^[a-f0-9]{64}$/)
+  expect(aspect.invoke()).toStrictEqual({ foo: 'served', bar: { baz: 'quux' } })
+
+  expect(client.subscribe).toHaveBeenCalledWith(component, epoch, expect.any(Function))
+
+  const listener = client.subscribe.mock.calls[0][2] as Listener
+
+  listener({ configuration: { foo: 'updated' }, created: 6 })
+
+  expect(aspect.invoke(['foo'])).toStrictEqual('updated')
+
+  // what is not newer than the held value is left alone
+  listener({ configuration: { foo: 'stale' }, created: 6 })
+  listener({ configuration: { foo: 'older' }, created: 4 })
+
+  expect(aspect.invoke(['foo'])).toStrictEqual('updated')
+
+  // a value that does not fit keeps the previous one
+  listener({ configuration: { foo: { nested: true } }, created: 7 })
+
+  expect(aspect.invoke(['foo'])).toStrictEqual('updated')
+
+  // and the one after it still applies
+  listener({ configuration: { foo: 'latest' }, created: 8 })
+
+  expect(aspect.invoke(['foo'])).toStrictEqual('latest')
 
-  const aspect = new Aspect(configuration)
+  await aspect.disconnect()
 
-  expect(aspect.invoke(['foo'])).toStrictEqual(configuration.foo)
-  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual(configuration.bar.baz)
+  expect(client.unsubscribe).toHaveBeenCalledWith(component, epoch, listener)
 })

package/source/Aspect.ts

@@ -1,17 +1,37 @@
-import { Connector, type extensions } from '@toa.io/core'
+import { console } from 'openspan'
+import { Connector, type Locator, type extensions } from '@toa.io/core'
+import { fit, local, type Node } from './configuration'
+import { epoch } from './epoch'
+import type { Client, Value } from './Client'
+import type { Manifest } from './manifest'
 
 export class Aspect extends Connector implements extensions.Aspect {
   public readonly name = 'configuration'
 
-  private readonly value: object
+  private readonly locator: Locator
+  private readonly manifest: Manifest
+  private readonly client: Client | null
+  private readonly epoch: string
+  private value: Node = {}
+  private created = 0
 
-  public constructor (value: object) {
+  /**
+   * Without a client the value is local: the variable, the defaults and the schema.
+   * With one, the value is what the service holds, and it follows the service.
+   */
+  public constructor (locator: Locator, manifest: Manifest, client: Client | null) {
     super()
 
-    this.value = value
+    this.locator = locator
+    this.manifest = manifest
+    this.client = client
+    this.epoch = epoch(manifest.schema)
+
+    if (client !== null)
+      this.depends(client)
   }
 
-  public invoke (path: string[]): any {
+  public invoke (path?: string[]): any {
     let cursor: any = this.value
 
     if (path !== undefined)
@@ -20,4 +40,38 @@ export class Aspect extends Connector implements extensions.Aspect {
 
     return cursor
   }
+
+  protected override async open (): Promise<void> {
+    if (this.client === null) {
+      this.value = local(this.locator, this.manifest)
+
+      return
+    }
+
+    const { configuration, created } = await this.client.fetch(this.locator.id, this.epoch)
+
+    this.value = fit(configuration, this.manifest)
+    this.created = created
+    this.client.subscribe(this.locator.id, this.epoch, this.listener)
+  }
+
+  protected override async close (): Promise<void> {
+    this.client?.unsubscribe(this.locator.id, this.epoch, this.listener)
+  }
+
+  private readonly listener = ({ configuration, created }: Value): void => {
+    // deliveries may repeat or cross: only what is newer than the held value replaces it
+    if (created <= this.created)
+      return
+
+    try {
+      this.value = fit(configuration, this.manifest)
+      this.created = created
+
+      console.info('Configuration updated', { component: this.locator.id, created })
+    } catch (error) {
+      // the service validated it against the schema of its epoch, so the two schemas differ
+      console.error('Configuration value does not match the schema', { component: this.locator.id, error })
+    }
+  }
 }