@toa.io/extensions.configuration 1.0.0-alpha.28 → 1.0.0-alpha.281

package/features/steps/UI.ts

@@ -0,0 +1,91 @@
+import assert from 'node:assert'
+import * as http from 'node:http'
+import { resolve } from 'node:path'
+import tsflow from 'cucumber-tsflow'
+
+import { UI } from '../../source/UI.js'
+import { UI_PORT } from '../../source/const.js'
+
+const { after, binding, given, then, when } = tsflow
+
+/**
+ * The scenarios are about the server, not about the page: it is pointed at a fixture
+ * directory, so that running them never needs a UI build.
+ */
+@binding()
+export class Site {
+  private server: UI | null = null
+  private response: Response | null = null
+
+  @given('the UI is published')
+  public async publish (): Promise<void> {
+    this.server = new UI(UI_PORT, resolve(import.meta.dirname, '..', 'site'))
+
+    await this.server.connect()
+  }
+
+  @when('{string} is requested')
+  public async request (path: string): Promise<void> {
+    this.response = await get(path)
+  }
+
+  @then('the status is {int}')
+  public statusIs (status: number): void {
+    assert.equal(this.response?.status, status)
+  }
+
+  @then('the body contains {string}')
+  public bodyContains (text: string): void {
+    assert.ok(this.response?.body.includes(text),
+      `Expected the body to contain '${text}', got '${this.response?.body}'`)
+  }
+
+  @then('the body is empty')
+  public bodyIsEmpty (): void {
+    assert.equal(this.response?.body, '')
+  }
+
+  @then('the {string} header is {string}')
+  public headerIs (name: string, value: string): void {
+    assert.equal(this.response?.headers[name], value)
+  }
+
+  @after()
+  public async shutdown (): Promise<void> {
+    await this.server?.disconnect()
+
+    this.server = null
+    this.response = null
+  }
+}
+
+/**
+ * Raw `http`, not `fetch`: the URL parser normalizes `%2e%2e` away, and one of the
+ * scenarios is about exactly that segment reaching the server.
+ */
+async function get (path: string): Promise<Response> {
+  return await new Promise((resolve, reject) => {
+    // `agent: false` — a keep-alive socket outlives the scenario that opened it
+    const request = http.get({ port: UI_PORT, path, agent: false }, (response) => {
+      let body = ''
+
+      response.setEncoding('utf8')
+      response.on('data', (chunk: string) => (body += chunk))
+      response.on('end', () => {
+        resolve({
+          status: response.statusCode!,
+          headers: response.headers as Record<string, string>,
+          body
+        })
+      })
+    })
+
+    request.on('error', reject)
+  })
+}
+
+interface Response {
+  status: number
+  headers: Record<string, string>
+  body: string
+}

package/features/steps/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "noEmit": true,
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true
+  }
+}

package/features/ui.feature

@@ -0,0 +1,93 @@
+@ui
+Feature: Publishing the UI
+
+  The values service serves the UI on its own port, out of the directory `ui` builds. The
+  mount path is baked into the bundle and forwarded by the ingress, so the server routes
+  relative to it.
+
+  Scenario: The root is the way in
+    Given the UI is published
+    When "/" is requested
+    Then the status is 302
+    And the "location" header is "/.configuration/"
+
+  Scenario: The UI answers at its mount path
+    Given the UI is published
+    When "/.configuration" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A trailing slash is the same resource
+    Given the UI is published
+    When "/.configuration/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A query string does not change the resource
+    Given the UI is published
+    When "/.configuration?epoch=e1" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: An unknown route is the page
+    The client router knows what routes there are; this server does not.
+
+    Given the UI is published
+    When "/.configuration/whatever" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A route that looks like a file is still a route
+    A component is named `namespace.component`, and the address of its configuration
+    carries that name — so an extension is no way to tell an asset from a route.
+
+    Given the UI is published
+    When "/.configuration/dummies.dummy/" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: And with no trailing slash to give it away
+    Given the UI is published
+    When "/.configuration/dummies.dummy" is requested
+    Then the status is 200
+    And the body contains "the page"
+
+  Scenario: A hashed asset is served forever
+    Given the UI is published
+    When "/.configuration/_app/immutable/asset.js" is requested
+    Then the status is 200
+    And the "content-type" header is "text/javascript; charset=utf-8"
+    And the "cache-control" header is "public, max-age=31536000, immutable"
+
+  Scenario: An icon is held, but not forever
+    It is named for what it is rather than for its content, so it is asked about again.
+
+    Given the UI is published
+    When "/.configuration/favicon.ico" is requested
+    Then the status is 200
+    And the "cache-control" header is "public, max-age=86400"
+
+  Scenario: The page itself is never cached
+    Given the UI is published
+    When "/.configuration/" is requested
+    Then the "cache-control" header is "no-cache"
+
+  Scenario: A missing asset is missing
+    Given the UI is published
+    When "/.configuration/_app/immutable/absent.js" is requested
+    Then the status is 404
+
+  Scenario: The path of the values API is not served by the UI
+    Given the UI is published
+    When "/configuration/values/" is requested
+    Then the status is 404
+
+  Scenario: A path that only looks like the mount path is absent
+    Given the UI is published
+    When "/.configurationable" is requested
+    Then the status is 404
+
+  Scenario: Nothing outside the site is reachable
+    Given the UI is published
+    When "/.configuration/%2e%2e/%2e%2e/package.json" is requested
+    Then the status is 404

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/extensions.configuration",
-  "version": "1.0.0-alpha.28",
+  "version": "1.0.0-alpha.281",
+  "type": "module",
   "description": "Toa Configuration",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -17,16 +18,16 @@
     "access": "public"
   },
   "scripts": {
-    "transpile": "tsc"
+    "transpile": "tsc && tsc -p ./components/configuration.values",
+    "build:ui": "npm --prefix ui ci && npm --prefix ui run build",
+    "prepack": "npm run build:ui",
+    "features": "TSX_TSCONFIG_PATH=features/steps/tsconfig.json NODE_OPTIONS=--import=tsx cucumber-js"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node"
-  },
-  "gitHead": "89c3b7d1314e357c9e33df8d95f8d46954cd3bcf",
+  "gitHead": "2cf23f576c46efbe56c21427d877cef3e7ed98d4",
   "dependencies": {
-    "@toa.io/core": "1.0.0-alpha.28",
-    "@toa.io/generic": "1.0.0-alpha.28",
-    "@toa.io/schemas": "1.0.0-alpha.28"
+    "@toa.io/core": "1.0.0-alpha.278",
+    "@toa.io/generic": "1.0.0-alpha.278",
+    "@toa.io/schemas": "1.0.0-alpha.278",
+    "openspan": "1.0.0-alpha.277"
   }
 }

package/readme.md

@@ -11,8 +11,12 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
   defaults:
     foo: bar
     bar: 1
@@ -45,22 +49,28 @@ configuration:
 $ toa conceal configuration BAZ_VALUE=$ecr3t
 ```
 
----
+### Change at runtime
 
-## Problem Definition
+```http
+POST /configuration/values/dummies.dummy/ HTTP/1.1
+authorization: Token ...
+content-type: application/yaml
 
-- Components should be runnable in different deployment environments.
-- Some algorithm's parameters should be deployed secretly.
-- Components should be reusable in different contexts.
+configuration:
+  foo: quux
+  bar: 2
+```
+
+---
 
 ## Manifest
 
-Component's configuration is declared using `configuration` manifest,
-containing `schema` and optionnaly `defaults` properties.
+Component's configuration is declared using the `configuration` manifest, containing `schema`
+and optionally `defaults` properties.
 
 ### Schema
 
-Configuration schema is declared with [COS](/libraries/concise).
+Configuration schema is declared with [JSON Schema](https://json-schema.org).
 
 ```yaml
 # manifest.toa.yaml
@@ -69,30 +79,18 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
-```
-
-> Introducing non-backward compatible changes to a configuration schema will result in a loss of
-> compatibility with existing contexts and deployment environments.
-> Therefore, configuration schema changes are subject to component versioning.
-
-If `configuration` object doesn't contain property `schema`, then it is considered to be schema.
-
-```yaml
-# manifest.toa.yaml
-name: dummy
-namespace: dummies
-
-configuration:
-  foo: string
-  bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
 ```
 
 ### Defaults
 
-The default configuration value can be provided using the `defaults` property, which should conform
-to the configuration schema.
+The default configuration value can be provided using the `defaults` property, which should
+conform to the configuration schema.
 
 ```yaml
 # manifest.toa.yaml
@@ -101,27 +99,22 @@ namespace: dummies
 
 configuration:
   schema:
-    foo: string
-    bar: number
+    type: object
+    properties:
+      foo:
+        type: string
+      bar:
+        type: number
   defaults:
     foo: hello
     bar: 0
 ```
 
-#### Schema defaults hint
+### Epoch
 
-The configuration schema itself can contain default primitive values using the COS syntax.
-
-```yaml
-# manifest.toa.yaml
-name: dummy
-namespace: dummies
-
-configuration:
-  schema:
-    foo: hello
-    bar: 0
-```
+The configuration epoch of a component is the SHA-256 of its configuration schema, as
+canonical JSON. A configuration object belongs to the epoch of the schema it was validated
+against. A schema change is a new epoch.
 
 ## Annotation
 
@@ -136,9 +129,26 @@ configuration:
     bar@staging: 2
 ```
 
+The annotated object is deployed as the defaults of the component for its epoch, in place of
+the manifest `defaults`.
+
+Every key names a component except `resources`, which is the values service's own: it deploys
+like any other and states what it may take. A component actually named `resources` is written
+with its namespace, `default.resources` — the bare form is only shorthand for that.
+
+```yaml
+# context.toa.yaml
+configuration:
+  resources:
+    cpu: [200m, 1000m]
+    memory: [200Mi, 500Mi]
+  dummies.dummy:
+    foo: bye
+```
+
 ## Secrets
 
-Configuration annotation top-level values which are uppercase strings prefixed with `$` considered as secrets.
+Configuration values which are uppercase strings prefixed with `$` are considered as secrets.
 
 ```yaml
 # context.toa.yaml
@@ -158,14 +168,117 @@ Deployed kubernetes secret's name is predefined as `configuration`.
 $ toa conceal configuration STRIPE_API_KEY=xxxxxxxx
 ```
 
+A secret is substituted in the component's process, from the variable
+`TOA_CONFIGURATION__<NAME>` deployed to it. The values service holds and returns the
+reference.
+
+In the component, a secret is a `Secret` object: `unwrap()` returns the string, while
+`toString()`, JSON and `util.inspect` give `<REDACTED>`.
+
+```javascript
+function transition (input, entity, context) {
+  const key = context.configuration.apiKey.unwrap()
+
+  // ...
+}
+```
+
+A value a component reads as a secret is given as a reference: a plain string in its place
+has no `unwrap`.
+
+## Values
+
+Configuration values are held by the `configuration.values` component, deployed as the
+`configuration-values` service. The service is deployed with the variable
+`TOA_CONFIGURATION_VALUES`: the epoch, the schema and the defaults of every component
+declaring configuration.
+
+```json
+{
+  "dummies.dummy": {
+    "epoch": "3f2a…",
+    "schema": { "type": "object", "properties": { "foo": { "type": "string" } } },
+    "defaults": { "foo": "bye" }
+  }
+}
+```
+
+Configuration objects are immutable. Creating a configuration is creating a new object for
+the component's current epoch; the latest object for a component and an epoch is the last
+created one. Each object records its `originator`.
+
+The configuration of a component for an epoch is:
+
+1. The latest object created for the component and the epoch;
+2. Otherwise, the deployed defaults, if the epoch is the deployed one;
+3. Otherwise, none.
+
+### Operations
+
+- `get({ component, epoch? })`: the configuration with the schema it is checked against, as
+  `{ configuration, schema, epoch }`, or `null` when there is none. The epoch is the deployed
+  one when omitted; an epoch the deployment does not know has no `schema`.
+- `fetch([{ component, epoch }])`: the same for several pairs at once, as
+  `[{ component, epoch, configuration }]`.
+- `list()`: every component's configuration for its deployed epoch, by component name, as
+  `[{ component, epoch, schema, configuration }]`.
+- `create({ component, configuration, originator })`: a new object for the component's
+  deployed epoch. The configuration must satisfy the schema. Errors: `UNKNOWN_COMPONENT`,
+  `INVALID_CONFIGURATION`.
+
+Creating a configuration publishes the `configuration.values.created` event with the
+object as stored.
+
+### Resources
+
+| Method | Path                                  | Role                          |
+|--------|---------------------------------------|-------------------------------|
+| `GET`  | `/configuration/values/`              | `system:configuration:get`    |
+| `GET`  | `/configuration/values/:component/`   | `system:configuration:get`    |
+| `POST` | `/configuration/values/:component/`   | `system:configuration:create` |
+
+`GET /configuration/values/` lists every component's configuration for its deployed epoch, by
+component name, as `[{ component, epoch, schema, configuration }]`.
+
+`GET /configuration/values/:component/` returns `{ configuration, schema, epoch }` for the
+deployed epoch, `404` when there is none.
+
+`POST` takes `{ configuration }`, records the Identity as the `originator`, and returns
+`{ id, epoch }`. A configuration not satisfying the schema, or an unknown component, is
+`422`.
+
+## UI
+
+The values service serves a page listing the configured components and creating
+configurations, mounted at `/.configuration` on port `8003`. Reading it needs the
+`system:configuration:get` role, creating needs `system:configuration:create`.
+
+The page is always published: unlike the introspection annotation, the configuration
+annotation is the per-component values map and has nowhere to carry a switch.
+
 ## Aspect
 
 Component's configuration values are available as a well-known Aspect `configuration`.
 
 ```javascript
 function transition (input, entity, context) {
-  const foo = context.configiuration.foo
+  const foo = context.configuration.foo
 
   // ...
 }
 ```
+
+On start, a component requests its configuration for its epoch from the values service and
+waits until there is one, reporting every fifth attempt. The schema is applied, and secrets
+are substituted. After a configuration is created, the running component receives the new
+object and takes it when its `_created` is later than that of the value it holds.
+
+### Local override
+
+When the variable `TOA_CONFIGURATION_<NAMESPACE>_<NAME>` is set, the component's
+configuration is the variable's value with the manifest `defaults` and the schema applied,
+and the values service is not used.
+
+```shell
+$ TOA_CONFIGURATION_DUMMIES_DUMMY='{"foo":"local"}' toa run components/dummy
+```

package/schemas/annotation.cos.yaml

@@ -1 +1,29 @@
-<object>
+# context.toa.yaml configuration annotation
+#
+# Every key names a component and holds that component's values, except `resources`, which
+# is the values service's own: it deploys like any other and has to state what it may take.
+# A component actually named `resources` is written with its namespace, `default.resources`
+# — the bare form is only shorthand for that.
+properties:
+  resources:
+    # `null` deploys the service without any, which every deployment has to state one way
+    # or the other — see `migrations/263.md`.
+    type: object
+    nullable: true
+    properties:
+      cpu:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+      memory:
+        type: array
+        items:
+          type: string
+        minItems: 2
+        maxItems: 2
+patternProperties:
+  ^(?!resources$).+$:
+    type: object
+type: object

package/schemas/manifest.cos.yaml

@@ -1,2 +1,9 @@
-schema: object
-defaults?: object
+properties:
+  schema:
+    type: object
+  defaults:
+    type: object
+additionalProperties: false
+type: object
+required:
+  - schema

package/source/Aspect.test.ts

@@ -1,15 +1,95 @@
-import { Aspect } from './Aspect'
+import { it, beforeEach, afterEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+import { isDeepStrictEqual } from 'node:util'
 
-it('should return value', async () => {
-  const configuration = {
-    foo: 'bar',
-    bar: {
-      baz: 'quux'
+import { Connector, Locator } from '@toa.io/core'
+import { generate } from 'randomstring'
+import { Aspect } from './Aspect.js'
+import type { Client, Listener } from './Client.js'
+import type { Manifest } from './manifest.js'
+
+class Fake extends Connector {
+  public readonly fetch = mock.fn(async () => ({ configuration: { foo: 'served' }, created: 5 }))
+  public readonly subscribe = mock.fn()
+  public readonly unsubscribe = mock.fn()
+}
+
+const manifest: Manifest = {
+  schema: {
+    type: 'object',
+    properties: {
+      foo: { type: 'string' },
+      bar: { type: 'object', properties: { baz: { type: 'string' } }, default: { baz: 'quux' } }
     }
   }
+}
+
+let locator: Locator
+
+beforeEach(() => {
+  locator = new Locator(generate(), generate())
+})
+
+afterEach(() => {
+  delete process.env['TOA_CONFIGURATION_' + locator.uppercase]
+})
+
+it('should be named', async () => {
+  assert.deepStrictEqual(new Aspect(locator, manifest, null).name, 'configuration')
+})
+
+it('should resolve locally without a client', async () => {
+  process.env['TOA_CONFIGURATION_' + locator.uppercase] = JSON.stringify({ foo: 'local' })
+
+  const aspect = new Aspect(locator, manifest, null)
+
+  await aspect.connect()
+
+  assert.deepStrictEqual(aspect.invoke(), { foo: 'local', bar: { baz: 'quux' } })
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'local')
+  assert.deepStrictEqual(aspect.invoke(['bar', 'baz']), 'quux')
+})
+
+it('should fetch from the client and follow it', async () => {
+  const client = new Fake()
+  const aspect = new Aspect(locator, manifest, client as unknown as Client)
+
+  await aspect.connect()
+
+  assert.deepStrictEqual(client.connected, true)
+  assert.strictEqual(client.fetch.mock.callCount(), 1)
+
+  const [component, epoch] = client.fetch.mock.calls[0].arguments as unknown as [string, string]
+
+  assert.deepStrictEqual(component, locator.id)
+  assert.match(epoch, /^[a-f0-9]{64}$/)
+  assert.deepStrictEqual(aspect.invoke(), { foo: 'served', bar: { baz: 'quux' } })
+
+  assert.ok(client.subscribe.mock.calls.some((call: any) => call.arguments.length === 3 && isDeepStrictEqual(call.arguments[0], component) && isDeepStrictEqual(call.arguments[1], epoch) && typeof call.arguments[2] === 'function'))
+
+  const listener = client.subscribe.mock.calls[0].arguments[2] as Listener
+
+  listener({ configuration: { foo: 'updated' }, created: 6 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // what is not newer than the held value is left alone
+  listener({ configuration: { foo: 'stale' }, created: 6 })
+  listener({ configuration: { foo: 'older' }, created: 4 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // a value that does not fit keeps the previous one
+  listener({ configuration: { foo: { nested: true } }, created: 7 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'updated')
+
+  // and the one after it still applies
+  listener({ configuration: { foo: 'latest' }, created: 8 })
+
+  assert.deepStrictEqual(aspect.invoke(['foo']), 'latest')
 
-  const aspect = new Aspect(configuration)
+  await aspect.disconnect()
 
-  expect(aspect.invoke(['foo'])).toStrictEqual(configuration.foo)
-  expect(aspect.invoke(['bar', 'baz'])).toStrictEqual(configuration.bar.baz)
+  assert.ok(client.unsubscribe.mock.calls.some((call: any) => call.arguments.length === 3 && isDeepStrictEqual(call.arguments[0], component) && isDeepStrictEqual(call.arguments[1], epoch) && isDeepStrictEqual(call.arguments[2], listener)))
 })