@toa.io/extensions.configuration 1.0.0-alpha.29 → 1.0.0-alpha.291

package/source/configuration.test.ts

@@ -1,85 +1,147 @@
-import { encode } from '@toa.io/generic'
+import { describe, it, beforeEach, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+
 import { Locator } from '@toa.io/core'
 import { generate } from 'randomstring'
-import { get } from './configuration'
-import { type Manifest } from './manifest'
+import { fit, local, overridden } from './configuration.js'
+import { Secret } from './Secret.js'
+import { type Manifest } from './manifest.js'
 
 let locator: Locator
 let manifest: Manifest
 
 beforeEach(() => {
   locator = new Locator(generate(), generate())
-  manifest = { schema: { foo: 'string' } }
+  manifest = {
+    schema: {
+      type: 'object',
+      properties: { foo: { type: 'string' } }
+    }
+  }
 })
 
 afterEach(() => {
-  for (const name of used)
-    delete process.env[name]
+  for (const name of used) delete process.env[name]
 
   used = []
 })
 
-it('should read value', async () => {
-  manifest.schema = { foo: 'string' }
+describe('overridden', () => {
+  it('should be false without the variable', async () => {
+    assert.deepStrictEqual(overridden(locator), false)
+  })
 
-  const value: object = { foo: generate() }
+  it('should be true with the variable', async () => {
+    set({})
 
-  set(value)
+    assert.deepStrictEqual(overridden(locator), true)
+  })
+})
 
-  const result = get(locator, manifest)
+describe('local', () => {
+  it('should read value', async () => {
+    const value: object = { foo: generate() }
 
-  expect(result).toStrictEqual(value)
-})
+    set(value)
 
-it('should return empty object if no value set', async () => {
-  expect(get(locator, manifest)).toStrictEqual({})
-})
+    assert.deepStrictEqual(local(locator, manifest), value)
+  })
 
-it('should substitute secrets', async () => {
-  const value: object = { foo: '$BAR' }
+  it('should return empty object if no value set', async () => {
+    assert.deepStrictEqual(local(locator, manifest), {})
+  })
 
-  set(value)
-  set('bar', '_BAR')
+  it('should substitute secrets', async () => {
+    set({ foo: '$BAR' })
+    set('bar', '_BAR')
 
-  const result = get(locator, manifest)
+    const { foo } = local(locator, manifest)
 
-  expect(result).toStrictEqual({ foo: 'bar' })
-})
+    assert.ok(foo instanceof Secret)
+    assert.deepStrictEqual((foo as Secret).unwrap(), 'bar')
+    assert.deepStrictEqual(String(foo), '<REDACTED>')
+  })
 
-it('should use defaults', async () => {
-  manifest.schema = { foo: 'string', bar: ['number'], 'baz?': 'string' }
-  manifest.defaults = { foo: 'bar', bar: [1] }
+  it('should substitute secrets in defaults', async () => {
+    manifest.defaults = { foo: '$BAR' }
 
-  const values = { bar: [2], baz: 'foo' }
+    set('bar', '_BAR')
 
-  set(values)
+    assert.deepStrictEqual((local(locator, manifest).foo as Secret).unwrap(), 'bar')
+  })
+
+  it('should use defaults', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'array', items: { type: 'number' } },
+        baz: { type: 'string' }
+      },
+      required: ['foo', 'bar']
+    }
+    manifest.defaults = { foo: 'bar', bar: [1] }
+
+    set({ bar: [2], baz: 'foo' })
+
+    assert.deepStrictEqual(local(locator, manifest), { foo: 'bar', bar: [2], baz: 'foo' })
+  })
+
+  it('should validate', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number' }
+      }
+    }
+
+    set({ bar: 5 })
 
-  const result = get(locator, manifest)
+    assert.deepStrictEqual(local(locator, manifest), { bar: 5 })
 
-  expect(result).toStrictEqual({
-    foo: 'bar',
-    bar: [2],
-    baz: 'foo'
+    set({ bar: 'five' })
+
+    assert.throws(
+      () => local(locator, manifest),
+      (error: Error) => /must be number/.test(error.message)
+    )
   })
 })
 
-it('should validate', async () => {
-  manifest.schema = { foo: 'hello', bar: 'number' }
+describe('fit', () => {
+  it('should substitute secrets and apply the schema', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number' }
+      }
+    }
+
+    set('secret', '_FOO')
 
-  const values = { bar: 5 }
+    const raw = { foo: '$FOO', bar: 1 }
+    const values = fit(raw, manifest)
 
-  set(values)
+    assert.deepStrictEqual((values.foo as Secret).unwrap(), 'secret')
+    assert.deepStrictEqual(values.bar, 1)
+    assert.deepStrictEqual(raw, { foo: '$FOO', bar: 1 }) // untouched
+  })
 
-  const result = get(locator, manifest)
+  it('should not apply the manifest defaults', async () => {
+    manifest.defaults = { foo: 'hello' }
+
+    assert.deepStrictEqual(fit({}, manifest), {})
+  })
 
-  expect(result).toStrictEqual({
-    foo: 'hello',
-    bar: 5
+  it('should throw on a value not fitting the schema', async () => {
+    assert.throws(() => fit({ foo: { nested: true } }, manifest))
   })
 })
 
-function set (value: object | string, key = locator.uppercase): void {
-  const string = typeof value === 'string' ? value : encode(value)
+function set(value: object | string, key = locator.uppercase): void {
+  const string = typeof value === 'string' ? value : JSON.stringify(value)
   const name = 'TOA_CONFIGURATION_' + key
 
   process.env[name] = string

package/source/configuration.ts

@@ -1,39 +1,54 @@
 import { type Locator } from '@toa.io/core'
-import { decode, add } from '@toa.io/generic'
+import { add, environment } from '@toa.io/generic'
 import * as schemas from '@toa.io/schemas'
-import { PREFIX, SECRET_RX } from './deployment'
-import { type Manifest } from './manifest'
+import { PREFIX, SECRET_RX } from './const.js'
+import { Secret } from './Secret.js'
 import type { Schema } from '@toa.io/schemas'
+import type { Manifest } from './manifest.js'
 
-export function get (locator: Locator, manifest: Manifest): Node {
-  const values = getConfiguration(locator.uppercase)
+/** The variable is set, so the values service is not consulted. */
+export function overridden(locator: Locator): boolean {
+  return environment.get(PREFIX + locator.uppercase) !== undefined
+}
+
+/** The variable, the manifest defaults, then the schema. */
+export function local(locator: Locator, manifest: Manifest): Node {
+  const values = read(locator.uppercase)
+
+  if (manifest.defaults !== undefined) add(values, manifest.defaults)
+
+  return fit(values, manifest)
+}
 
+/** A copy of the value with the schema applied and the secrets substituted. */
+export function fit(raw: object, manifest: Manifest): Node {
+  // a copy of the caller's, and one of this realm: what came over the wire is JSON anyway
+  const values = JSON.parse(JSON.stringify(raw)) as Node
+
+  // the schema sees the references, which are the strings it declares
+  validate(values, manifest)
   substituteSecrets(values)
 
-  if (manifest.defaults !== undefined)
-    add(values, manifest.defaults)
+  return values
+}
 
+function validate(values: Node, manifest: Manifest): void {
   const schema: Schema<any> = schemas.schema(manifest.schema)
 
   schema.validate(values)
-
-  return values
 }
 
-function getConfiguration (suffix: string): Node {
+function read(suffix: string): Node {
   const variable = PREFIX + suffix
-  const string = process.env[variable]
+  const string = environment.get(variable)
 
-  if (string === undefined)
-    return {}
-  else
-    return decode(string)
+  if (string === undefined) return {}
+  else return JSON.parse(string)
 }
 
-function substituteSecrets (configuration: Node): void {
+function substituteSecrets(configuration: Node): void {
   for (const [key, value] of Object.entries(configuration)) {
-    if (typeof value === 'object' && value !== null)
-      substituteSecrets(value as Node)
+    if (typeof value === 'object' && value !== null) substituteSecrets(value as Node)
 
     if (typeof value !== 'string') continue
 
@@ -43,13 +58,13 @@ function substituteSecrets (configuration: Node): void {
 
     const name = match.groups?.variable
 
-    configuration[key] = getSecret(name!)
+    configuration[key] = new Secret(getSecret(name!))
   }
 }
 
-function getSecret (name: string): string {
+function getSecret(name: string): string {
   const variable = PREFIX + '_' + name
-  const value = process.env[variable]
+  const value = environment.get(variable)
 
   if (value === undefined) throw new Error(`${variable} is not set.`)
 

package/source/const.ts

@@ -0,0 +1,30 @@
+import { environment } from '@toa.io/generic'
+import type { Source } from '@toa.io/core/types'
+
+/** Per-component variables: the local override and the secrets. */
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+/** The map of every configured component, on the values service. */
+export const VALUES = 'TOA_CONFIGURATION_VALUES'
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+
+/** Where the UI is mounted; `/configuration/*` belongs to the component's own API. */
+export const UI_PATH = '/.configuration'
+export const UI_PORT = 8003
+
+/**
+ * The port the UI is served on. `UI_PORT` is what a deployment publishes and what the chart
+ * renders; the variable is for a machine that runs more than one Toa process — a Toa checkout
+ * beside an application, or two applications — where one of them has to give it up.
+ */
+export const UI_PORT_ENV = 'TOA_CONFIGURATION_UI_PORT'
+export function uiPort(): number {
+  const value = environment.get(UI_PORT_ENV)
+
+  return value === undefined ? UI_PORT : Number(value)
+}
+
+export const EVENT = 'configuration.values.created'
+
+export const SOURCE: Source = { service: 'configuration' }

package/source/context.ts

@@ -0,0 +1,13 @@
+import type { Manifest } from './manifest.js'
+import type { Contribution } from '@toa.io/core/types'
+
+/** What this extension puts on the context of a component that declares it. */
+export function context(declaration: Manifest): Contribution {
+  const contribution: Contribution = { name: 'configuration', schema: declaration.schema }
+
+  // a value the schema declares a secret is one on the context, not a string
+  if (JSON.stringify(declaration.schema).includes('"secret"'))
+    contribution.imports = { '@toa.io/extensions.configuration': ['Secret'] }
+
+  return contribution
+}

package/source/deployment.test.ts

@@ -1,21 +1,112 @@
-import { type Annotation, deployment, type Instance } from './deployment'
-import { type Manifest } from './manifest'
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { Locator } from '@toa.io/core'
+import {
+  type Annotation,
+  type Instance,
+  deployment,
+  describe as map
+} from './deployment.js'
+import { epoch } from './epoch.js'
+import { EVENT, VALUES } from './const.js'
+
+const schema = {
+  type: 'object',
+  properties: { foo: { type: 'string' }, key: { type: 'string' } }
+}
+
+function instance(name: string, defaults?: Record<string, unknown>): Instance {
+  return {
+    locator: new Locator(name, 'configuration'),
+    manifest: { schema, defaults },
+    component: {} as any
+  }
+}
 
 it('should validate annotation', async () => {
   const wrongType = 'not ok' as unknown as Annotation
 
-  expect(() => deployment([], wrongType)).toThrow('object')
+  assert.throws(
+    () => deployment([], wrongType),
+    (error: any) => /object/.test(error.message)
+  )
 })
 
-it('should validate values', async () => {
-  const manifest: Manifest = {
-    schema: { foo: 'string', bar: 'number' },
-    defaults: { foo: 'ok', bar: 0 }
-  }
+it('should reject unknown components', async () => {
+  assert.throws(
+    () => deployment([instance('base')], { 'configuration.nope': {} }),
+    (error: any) =>
+      /Component 'configuration\.nope' does not request configuration/.test(error.message)
+  )
+})
+
+it('should deploy the values service with the map', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const annotation = { 'configuration.other': { foo: 'set' } }
+
+  const dependency = deployment(instances, annotation)
+
+  assert.deepStrictEqual(dependency.events, [EVENT])
+  assert.strictEqual(dependency.services?.length, 1)
+
+  const service = dependency.services![0]
+
+  assert.deepStrictEqual(service.group, 'configuration')
+  assert.deepStrictEqual(service.name, 'values')
+  assert.deepStrictEqual(service.components, ['configuration-values'])
+
+  const variable = service.variables!.find((variable) => variable.name === VALUES)
+
+  assert.notStrictEqual(variable, undefined)
+  assert.deepStrictEqual(JSON.parse(variable!.value!), map(instances, annotation))
+})
+
+it('should describe every component', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const values = map(instances, { 'configuration.other': { foo: 'set' } })
+
+  assert.deepStrictEqual(values, {
+    'configuration.base': { epoch: epoch(schema), schema, defaults: { foo: 'hello' } },
+    'configuration.other': { epoch: epoch(schema), schema, defaults: { foo: 'set' } }
+  })
+})
+
+it('should prefer the context over the manifest defaults', async () => {
+  const values = map([instance('base', { foo: 'hello' })], {
+    'configuration.base': { foo: 'bye' }
+  })
+
+  assert.deepStrictEqual(values['configuration.base'].defaults, { foo: 'bye' })
+})
+
+it('should not map values to the component', async () => {
+  const dependency = deployment([instance('base')], {
+    'configuration.base': { foo: 'set' }
+  })
+
+  assert.deepStrictEqual(dependency.variables, {})
+})
+
+it('should map secrets to the component', async () => {
+  const dependency = deployment([instance('base')], {
+    'configuration.base': { key: '$KEY' }
+  })
 
-  const locator = { id: 'component' }
-  const instances = [{ manifest, locator }] as unknown as Instance[]
-  const annotation: Annotation = { [locator.id]: { bar: 'not a number' } }
+  assert.deepStrictEqual(dependency.variables, {
+    'configuration-base': [
+      {
+        name: 'TOA_CONFIGURATION__KEY',
+        secret: { name: 'toa-configuration', key: 'KEY' }
+      }
+    ]
+  })
 
-  expect(() => deployment(instances, annotation)).toThrow('number')
+  // the service holds the reference, not the secret
+  assert.deepStrictEqual(
+    map([instance('base')], { 'configuration.base': { key: '$KEY' } })[
+      'configuration.base'
+    ].defaults,
+    { key: '$KEY' }
+  )
 })

package/source/deployment.ts

@@ -1,38 +1,91 @@
+import { readFileSync } from 'node:fs'
 import assert from 'node:assert'
-import { type Dependency, type Variable, type Variables } from '@toa.io/operations'
-import * as schemas from '@toa.io/schemas'
-import { encode, overwrite } from '@toa.io/generic'
-import { type Manifest } from './manifest'
-import * as validators from './schemas'
-import type { Schema } from '@toa.io/schemas'
+import {
+  type Dependency,
+  type Resources,
+  type Service,
+  type Variable,
+  type Variables
+} from '@toa.io/operations'
+import { add } from '@toa.io/generic'
+import { components } from './Composition.js'
+import { EVENT, PREFIX, SECRET_RX, UI_PATH, UI_PORT, VALUES } from './const.js'
+import { epoch } from './epoch.js'
+import * as validators from './schemas.js'
+import type { Manifest } from './manifest.js'
 import type { context } from '@toa.io/norm'
 
-export function deployment (instances: Instance[], annotation: Annotation = {}): Dependency {
-  validate(annotation, instances)
+/** Where Toa's release publishes this service's image. An application takes it
+ *  instead of building one when its context says `registry.services: published`. */
+export const image = 'ghcr.io/toa-io/extension-configuration-values'
+
+export function deployment(
+  instances: Instance[],
+  annotation: Annotation = {}
+): Dependency {
+  const { resources, values } = split(annotation)
+
+  annotation = prepare(values, instances)
 
   const variables: Variables = {}
 
   for (const instance of instances) {
     const values = annotation[instance.locator.id]
 
-    validateInstance(instance, values)
-
     if (values === undefined) continue
 
-    variables[instance.locator.label] = [{
-      name: PREFIX + instance.locator.uppercase,
-      value: encode(values)
-    }]
-
     const secrets = createSecrets(values)
 
-    variables[instance.locator.label].push(...secrets)
+    if (secrets.length > 0) variables[instance.locator.label] = secrets
+  }
+
+  const service: Service = {
+    group: 'configuration',
+    name: 'values',
+    image,
+    version: JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
+      .version,
+    components: components().labels,
+    resources,
+    // the service that holds the values also serves the page that reads them
+    port: UI_PORT,
+    ingress: { path: UI_PATH },
+    variables: [
+      {
+        name: VALUES,
+        value: JSON.stringify(describe(instances, annotation))
+      }
+    ]
+  }
+
+  return { services: [service], variables, events: [EVENT] }
+}
+
+/** What the values service is given: the epoch, the schema and the defaults of every component. */
+export function describe(instances: Instance[], annotation: Annotation = {}): Values {
+  annotation = prepare(split(annotation).values, instances)
+
+  const values: Values = {}
+
+  for (const { locator, manifest } of instances) {
+    const deployed = annotation[locator.id]
+
+    values[locator.id] = {
+      epoch: epoch(manifest.schema),
+      schema: manifest.schema,
+      // what the Context says stands over what the manifest declares, value by value: a
+      // deployment that names one of them does not thereby unset the rest
+      defaults:
+        deployed === undefined
+          ? manifest.defaults
+          : add(structuredClone(deployed), manifest.defaults ?? {})
+    }
   }
 
-  return { variables }
+  return values
 }
 
-function createSecrets (values: object): Variable[] {
+function createSecrets(values: object): Variable[] {
   const secrets: Variable[] = []
 
   for (const value of Object.values(values)) {
@@ -61,26 +114,47 @@ function createSecrets (values: object): Variable[] {
   return secrets
 }
 
-function validate (annotation: Annotation, instances: Instance[]): void {
+/**
+ * The service's own resources, and the component values that are the rest of the annotation.
+ *
+ * Every key here names a component, so the one option the service has of its own needs a
+ * name that cannot be mistaken for one. `resources` is reserved: a component actually
+ * called that is written with its namespace, `default.resources`, which is what an id is
+ * anyway — the bare form is the shorthand.
+ */
+function split(annotation: Annotation): { resources?: Resources; values: Annotation } {
   validators.annotation.validate(annotation)
 
-  const requested = instances.map((instance) => instance.locator.id)
+  const { resources, ...values } = annotation as Annotation & { resources?: Resources }
 
-  for (const id of Object.keys(annotation))
-    assert.ok(requested.includes(id),
-      `Component '${id}' does not request configuration or does not exist.`)
+  return { resources, values }
 }
 
-function validateInstance (instace: Instance, values: object = {}): void {
-  const defaults = instace.manifest.defaults ?? {}
-  const configuration = overwrite(defaults, values)
-  const schema: Schema<any> = schemas.schema(instace.manifest.schema)
+/** Validated, keyed by full component ids, and checked against the components that ask. */
+function prepare(annotation: Annotation, instances: Instance[]): Annotation {
+  const normalized: Annotation = {}
+  const requested = instances.map((instance) => instance.locator.id)
 
-  schema.validate(configuration)
-}
+  for (const [key, values] of Object.entries(annotation)) {
+    const id = key.includes('.') ? key : 'default.' + key
+
+    assert.ok(
+      requested.includes(id),
+      `Component '${id}' does not request configuration or does not exist.`
+    )
 
-export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
-export const PREFIX = 'TOA_CONFIGURATION_'
+    normalized[id] = values
+  }
+
+  return normalized
+}
 
 export type Annotation = Record<string, object>
 export type Instance = context.Dependency<Manifest>
+export type Values = Record<string, Entry>
+
+export interface Entry {
+  epoch: string
+  schema: object
+  defaults?: object
+}

package/source/epoch.test.ts

@@ -0,0 +1,35 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { epoch } from './epoch.js'
+
+it('should be a sha256 hex', () => {
+  assert.match(epoch({ type: 'object' }), /^[a-f0-9]{64}$/)
+})
+
+it('should not depend on key order', () => {
+  const a = {
+    type: 'object',
+    properties: { foo: { type: 'string', default: 'x' }, bar: { type: 'number' } }
+  }
+  const b = {
+    properties: { bar: { type: 'number' }, foo: { default: 'x', type: 'string' } },
+    type: 'object'
+  }
+
+  assert.deepStrictEqual(epoch(a), epoch(b))
+})
+
+it('should depend on values', () => {
+  const a = { type: 'object', properties: { foo: { type: 'string' } } }
+  const b = { type: 'object', properties: { foo: { type: 'number' } } }
+
+  assert.notDeepStrictEqual(epoch(a), epoch(b))
+})
+
+it('should keep array order', () => {
+  const a = { enum: [1, 2] }
+  const b = { enum: [2, 1] }
+
+  assert.notDeepStrictEqual(epoch(a), epoch(b))
+})