@toa.io/extensions.configuration 1.0.0-alpha.273 → 1.0.0-alpha.274

package/source/UI.ts

@@ -0,0 +1,197 @@
+import * as fs from 'node:fs'
+import * as http from 'node:http'
+import * as path from 'node:path'
+import { Connector } from '@toa.io/core'
+import { console } from 'openspan'
+import { UI_PATH } from './const'
+
+/**
+ * Serves the configuration UI: the directory `ui` builds, and nothing else.
+ *
+ * The page is a single-page application whose mount path is baked into the bundle,
+ * so this server routes relative to `UI_PATH` — which is also what the ingress
+ * forwards. `/configuration/*` is left alone: that is the component's own API.
+ */
+export class UI extends Connector {
+  private readonly server: http.Server = http.createServer()
+  private readonly port: number
+  private readonly root: string
+
+  public constructor (port: number, root: string = SITE) {
+    super()
+
+    this.port = port
+    this.root = root
+    this.server.on('request', (request, response) => {
+      this.listen(request, response)
+    })
+  }
+
+  protected override async open (): Promise<void> {
+    /*
+     * A taken port is a real error: uniqueness across services is settled at export
+     * time, so nothing here has to negotiate for one.
+     */
+    await new Promise<void>((resolve, reject) => {
+      const failed = (error: Error): void => {
+        this.server.off('listening', listening)
+        reject(error)
+      }
+
+      const listening = (): void => {
+        this.server.off('error', failed)
+        resolve()
+      }
+
+      this.server.once('error', failed)
+      this.server.once('listening', listening)
+      this.server.listen(this.port)
+    })
+
+    if (!isFile(path.join(this.root, 'index.html')))
+      console.warn('Configuration UI is not built', { root: this.root })
+
+    console.info('Configuration UI started', { port: this.port, path: UI_PATH + '/' })
+  }
+
+  protected override async close (): Promise<void> {
+    const closed = new Promise<void>((resolve) => this.server.once('close', () => {
+      resolve()
+    }))
+
+    this.server.close()
+    this.server.closeAllConnections()
+
+    await closed
+  }
+
+  private listen (request: http.IncomingMessage, response: http.ServerResponse): void {
+    void this.respond(request, response).catch((error: Error) => {
+      console.error('Configuration UI failure', { message: error.message })
+
+      if (!response.writableEnded)
+        response.writeHead(500).end()
+    })
+  }
+
+  private async respond (request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    if (request.method !== 'GET' && request.method !== 'HEAD') {
+      response.writeHead(405, { allow: 'GET, HEAD' }).end()
+
+      return
+    }
+
+    const pathname = decode(request.url ?? '/')
+
+    if (pathname === null) {
+      response.writeHead(400).end()
+
+      return
+    }
+
+    // the port is the explorer's alone, so the root is the way in
+    if (pathname === '/') {
+      response.writeHead(302, { location: UI_PATH + '/' }).end()
+
+      return
+    }
+
+    const file = this.resolve(pathname)
+
+    if (file === null)
+      response.writeHead(404).end()
+    else
+      await this.send(file, request, response)
+  }
+
+  /**
+   * The file a request lands on, or `null` when nothing does. A path that exists is
+   * served as it is; anything else that could be a route falls back to the page,
+   * because the client router — not this server — knows what routes there are.
+   */
+  private resolve (pathname: string): string | null {
+    if (!pathname.startsWith(UI_PATH))
+      return null
+
+    const relative = pathname.slice(UI_PATH.length)
+
+    if (relative !== '' && !relative.startsWith('/'))
+      return null
+
+    const file = path.join(this.root, relative)
+
+    if (file !== this.root && !file.startsWith(this.root + path.sep))
+      return null
+
+    if (isFile(file))
+      return file
+
+    /*
+     * A missing asset is missing, but a route can look like one: `identity.passkeys` is a
+     * component, not a file with an extension. What this server would have served is what
+     * it knows how to serve, so anything else is a route and falls back to the page — as
+     * does anything ending in a slash, which is no name for a file at all.
+     */
+    const asset = !relative.endsWith('/') && path.extname(relative) in TYPES
+
+    return asset ? null : path.join(this.root, 'index.html')
+  }
+
+  private async send (file: string, request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    const stats = await fs.promises.stat(file).catch(() => null)
+
+    if (stats === null) {
+      response.writeHead(503, { 'content-type': 'text/plain' })
+        .end('The configuration UI is not built. Run `npm run build:ui`.\n')
+
+      return
+    }
+
+    response.writeHead(200, {
+      'content-type': TYPES[path.extname(file)] ?? 'application/octet-stream',
+      'content-length': stats.size,
+      'cache-control': file.startsWith(path.join(this.root, IMMUTABLE)) ? FOREVER : 'no-cache'
+    })
+
+    if (request.method === 'HEAD')
+      response.end()
+    else
+      fs.createReadStream(file).pipe(response)
+  }
+}
+
+function decode (url: string): string | null {
+  try {
+    return decodeURIComponent(url.split('?')[0])
+  } catch {
+    return null
+  }
+}
+
+function isFile (file: string): boolean {
+  return fs.existsSync(file) && fs.statSync(file).isFile()
+}
+
+/** Where `npm run build:ui` puts the page, from both `source` and `transpiled`. */
+const SITE = path.resolve(__dirname, '..', 'ui', 'dist')
+
+/** Assets under this prefix carry their build hash in the name. */
+const IMMUTABLE = path.join('_app', 'immutable')
+const FOREVER = 'public, max-age=31536000, immutable'
+
+const TYPES: Record<string, string> = {
+  '.css': 'text/css; charset=utf-8',
+  '.html': 'text/html; charset=utf-8',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.map': 'application/json; charset=utf-8',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml',
+  '.txt': 'text/plain; charset=utf-8',
+  '.webmanifest': 'application/manifest+json',
+  '.webp': 'image/webp',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2'
+}

package/source/configuration.test.ts

@@ -1,6 +1,7 @@
 import { Locator } from '@toa.io/core'
 import { generate } from 'randomstring'
-import { get } from './configuration'
+import { fit, local, overridden } from './configuration'
+import { Secret } from './Secret'
 import { type Manifest } from './manifest'
 
 let locator: Locator
@@ -23,79 +24,110 @@ afterEach(() => {
   used = []
 })
 
-it('should read value', async () => {
-  manifest.schema = {
-    type: 'object',
-    properties: { foo: { type: 'string' } }
-  }
+describe('overridden', () => {
+  it('should be false without the variable', async () => {
+    expect(overridden(locator)).toStrictEqual(false)
+  })
 
-  const value: object = { foo: generate() }
+  it('should be true with the variable', async () => {
+    set({})
 
-  set(value)
+    expect(overridden(locator)).toStrictEqual(true)
+  })
+})
 
-  const result = get(locator, manifest)
+describe('local', () => {
+  it('should read value', async () => {
+    const value: object = { foo: generate() }
 
-  expect(result).toStrictEqual(value)
-})
+    set(value)
 
-it('should return empty object if no value set', async () => {
-  expect(get(locator, manifest)).toStrictEqual({})
-})
+    expect(local(locator, manifest)).toStrictEqual(value)
+  })
 
-it('should substitute secrets', async () => {
-  const value: object = { foo: '$BAR' }
+  it('should return empty object if no value set', async () => {
+    expect(local(locator, manifest)).toStrictEqual({})
+  })
 
-  set(value)
-  set('bar', '_BAR')
+  it('should substitute secrets', async () => {
+    set({ foo: '$BAR' })
+    set('bar', '_BAR')
 
-  const result = get(locator, manifest)
+    const { foo } = local(locator, manifest)
 
-  expect(result).toStrictEqual({ foo: 'bar' })
-})
+    expect(foo).toBeInstanceOf(Secret)
+    expect((foo as Secret).unwrap()).toStrictEqual('bar')
+    expect(String(foo)).toStrictEqual('<REDACTED>')
+  })
 
-it('should use defaults', async () => {
-  manifest.schema = {
-    type: 'object',
-    properties: {
-      foo: { type: 'string' },
-      bar: { type: 'array', items: { type: 'number' } },
-      baz: { type: 'string' }
-    },
-    required: ['foo', 'bar']
-  }
-  manifest.defaults = { foo: 'bar', bar: [1] }
+  it('should substitute secrets in defaults', async () => {
+    manifest.defaults = { foo: '$BAR' }
 
-  const values = { bar: [2], baz: 'foo' }
+    set('bar', '_BAR')
 
-  set(values)
+    expect((local(locator, manifest).foo as Secret).unwrap()).toStrictEqual('bar')
+  })
 
-  const result = get(locator, manifest)
+  it('should use defaults', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'array', items: { type: 'number' } },
+        baz: { type: 'string' }
+      },
+      required: ['foo', 'bar']
+    }
+    manifest.defaults = { foo: 'bar', bar: [1] }
 
-  expect(result).toStrictEqual({
-    foo: 'bar',
-    bar: [2],
-    baz: 'foo'
+    set({ bar: [2], baz: 'foo' })
+
+    expect(local(locator, manifest)).toStrictEqual({ foo: 'bar', bar: [2], baz: 'foo' })
+  })
+
+  it('should validate', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string', default: 'hello' },
+        bar: { type: 'number' }
+      }
+    }
+
+    set({ bar: 5 })
+
+    expect(local(locator, manifest)).toStrictEqual({ foo: 'hello', bar: 5 })
   })
 })
 
-it('should validate', async () => {
-  manifest.schema = {
-    type: 'object',
-    properties: {
-      foo: { type: 'string', default: 'hello' },
-      bar: { type: 'number' }
+describe('fit', () => {
+  it('should substitute secrets and apply the schema', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number', default: 1 }
+      }
     }
-  }
 
-  const values = { bar: 5 }
+    set('secret', '_FOO')
+
+    const raw = { foo: '$FOO' }
+    const values = fit(raw, manifest)
 
-  set(values)
+    expect((values.foo as Secret).unwrap()).toStrictEqual('secret')
+    expect(values.bar).toStrictEqual(1)
+    expect(raw).toStrictEqual({ foo: '$FOO' }) // untouched
+  })
+
+  it('should not apply the manifest defaults', async () => {
+    manifest.defaults = { foo: 'hello' }
 
-  const result = get(locator, manifest)
+    expect(fit({}, manifest)).toStrictEqual({})
+  })
 
-  expect(result).toStrictEqual({
-    foo: 'hello',
-    bar: 5
+  it('should throw on a value not fitting the schema', async () => {
+    expect(() => fit({ foo: { nested: true } }, manifest)).toThrow()
   })
 })
 

package/source/configuration.ts

@@ -1,26 +1,45 @@
 import { type Locator } from '@toa.io/core'
 import { add } from '@toa.io/generic'
 import * as schemas from '@toa.io/schemas'
-import { PREFIX, SECRET_RX } from './deployment'
-import { type Manifest } from './manifest'
+import { PREFIX, SECRET_RX } from './const'
+import { Secret } from './Secret'
 import type { Schema } from '@toa.io/schemas'
+import type { Manifest } from './manifest'
 
-export function get (locator: Locator, manifest: Manifest): Node {
-  const values = getConfiguration(locator.uppercase)
+/** The variable is set, so the values service is not consulted. */
+export function overridden (locator: Locator): boolean {
+  return process.env[PREFIX + locator.uppercase] !== undefined
+}
 
-  substituteSecrets(values)
+/** The variable, the manifest defaults, then the schema. */
+export function local (locator: Locator, manifest: Manifest): Node {
+  const values = read(locator.uppercase)
 
   if (manifest.defaults !== undefined)
     add(values, manifest.defaults)
 
-  const schema: Schema<any> = schemas.schema(manifest.schema)
+  return fit(values, manifest)
+}
 
-  schema.validate(values)
+/** A copy of the value with the schema applied and the secrets substituted. */
+export function fit (raw: object, manifest: Manifest): Node {
+  // a copy of the caller's, and one of this realm: what came over the wire is JSON anyway
+  const values = JSON.parse(JSON.stringify(raw)) as Node
+
+  // the schema sees the references, which are the strings it declares
+  validate(values, manifest)
+  substituteSecrets(values)
 
   return values
 }
 
-function getConfiguration (suffix: string): Node {
+function validate (values: Node, manifest: Manifest): void {
+  const schema: Schema<any> = schemas.schema(manifest.schema)
+
+  schema.validate(values)
+}
+
+function read (suffix: string): Node {
   const variable = PREFIX + suffix
   const string = process.env[variable]
 
@@ -43,7 +62,7 @@ function substituteSecrets (configuration: Node): void {
 
     const name = match.groups?.variable
 
-    configuration[key] = getSecret(name!)
+    configuration[key] = new Secret(getSecret(name!))
   }
 }
 

package/source/const.ts

@@ -0,0 +1,17 @@
+import type { Source } from '@toa.io/core'
+
+/** Per-component variables: the local override and the secrets. */
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+/** The map of every configured component, on the values service. */
+export const VALUES = 'TOA_CONFIGURATION_VALUES'
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+
+/** Where the UI is mounted; `/configuration/*` belongs to the component's own API. */
+export const UI_PATH = '/.configuration'
+export const UI_PORT = 8003
+
+export const EVENT = 'configuration.values.created'
+
+export const SOURCE: Source = { service: 'configuration' }

package/source/deployment.test.ts

@@ -1,7 +1,86 @@
-import { type Annotation, deployment } from './deployment'
+import { Locator } from '@toa.io/core'
+import { type Annotation, type Instance, deployment, describe as map } from './deployment'
+import { epoch } from './epoch'
+import { EVENT, VALUES } from './const'
+
+const schema = {
+  type: 'object',
+  properties: { foo: { type: 'string' }, key: { type: 'string' } }
+}
+
+function instance (name: string, defaults?: Record<string, unknown>): Instance {
+  return {
+    locator: new Locator(name, 'configuration'),
+    manifest: { schema, defaults },
+    component: {} as any
+  }
+}
 
 it('should validate annotation', async () => {
   const wrongType = 'not ok' as unknown as Annotation
 
   expect(() => deployment([], wrongType)).toThrow('object')
 })
+
+it('should reject unknown components', async () => {
+  expect(() => deployment([instance('base')], { 'configuration.nope': {} }))
+    .toThrow("Component 'configuration.nope' does not request configuration")
+})
+
+it('should deploy the values service with the map', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const annotation = { 'configuration.other': { foo: 'set' } }
+
+  const dependency = deployment(instances, annotation)
+
+  expect(dependency.events).toStrictEqual([EVENT])
+  expect(dependency.services).toHaveLength(1)
+
+  const service = dependency.services![0]
+
+  expect(service.group).toStrictEqual('configuration')
+  expect(service.name).toStrictEqual('values')
+  expect(service.components).toStrictEqual(['configuration-values'])
+
+  const variable = service.variables!.find((variable) => variable.name === VALUES)
+
+  expect(variable).toBeDefined()
+  expect(JSON.parse(variable!.value!)).toStrictEqual(map(instances, annotation))
+})
+
+it('should describe every component', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const values = map(instances, { 'configuration.other': { foo: 'set' } })
+
+  expect(values).toStrictEqual({
+    'configuration.base': { epoch: epoch(schema), schema, defaults: { foo: 'hello' } },
+    'configuration.other': { epoch: epoch(schema), schema, defaults: { foo: 'set' } }
+  })
+})
+
+it('should prefer the context over the manifest defaults', async () => {
+  const values = map([instance('base', { foo: 'hello' })], { 'configuration.base': { foo: 'bye' } })
+
+  expect(values['configuration.base'].defaults).toStrictEqual({ foo: 'bye' })
+})
+
+it('should not map values to the component', async () => {
+  const dependency = deployment([instance('base')], { 'configuration.base': { foo: 'set' } })
+
+  expect(dependency.variables).toStrictEqual({})
+})
+
+it('should map secrets to the component', async () => {
+  const dependency = deployment([instance('base')], { 'configuration.base': { key: '$KEY' } })
+
+  expect(dependency.variables).toStrictEqual({
+    'configuration-base': [{
+      name: 'TOA_CONFIGURATION__KEY',
+      secret: { name: 'toa-configuration', key: 'KEY' }
+    }]
+  })
+
+  // the service holds the reference, not the secret
+  expect(map([instance('base')], { 'configuration.base': { key: '$KEY' } })['configuration.base'].defaults)
+    .toStrictEqual({ key: '$KEY' })
+})

package/source/deployment.ts

@@ -1,11 +1,16 @@
 import assert from 'node:assert'
-import { type Dependency, type Variable, type Variables } from '@toa.io/operations'
-import { type Manifest } from './manifest'
+import { type Dependency, type Resources, type Service, type Variable, type Variables } from '@toa.io/operations'
+import { components } from './Composition'
+import { EVENT, PREFIX, SECRET_RX, UI_PATH, UI_PORT, VALUES } from './const'
+import { epoch } from './epoch'
 import * as validators from './schemas'
+import type { Manifest } from './manifest'
 import type { context } from '@toa.io/norm'
 
 export function deployment (instances: Instance[], annotation: Annotation = {}): Dependency {
-  validate(annotation, instances)
+  const { resources, values } = split(annotation)
+
+  annotation = prepare(values, instances)
 
   const variables: Variables = {}
 
@@ -15,17 +20,44 @@ export function deployment (instances: Instance[], annotation: Annotation = {}):
     if (values === undefined)
       continue
 
-    variables[instance.locator.label] = [{
-      name: PREFIX + instance.locator.uppercase,
-      value: JSON.stringify(values)
-    }]
-
     const secrets = createSecrets(values)
 
-    variables[instance.locator.label].push(...secrets)
+    if (secrets.length > 0)
+      variables[instance.locator.label] = secrets
+  }
+
+  const service: Service = {
+    group: 'configuration',
+    name: 'values',
+    version: require('../package.json').version,
+    components: components().labels,
+    resources,
+    // the service that holds the values also serves the page that reads them
+    port: UI_PORT,
+    ingress: { path: UI_PATH },
+    variables: [{
+      name: VALUES,
+      value: JSON.stringify(describe(instances, annotation))
+    }]
   }
 
-  return { variables }
+  return { services: [service], variables, events: [EVENT] }
+}
+
+/** What the values service is given: the epoch, the schema and the defaults of every component. */
+export function describe (instances: Instance[], annotation: Annotation = {}): Values {
+  annotation = prepare(split(annotation).values, instances)
+
+  const values: Values = {}
+
+  for (const { locator, manifest } of instances)
+    values[locator.id] = {
+      epoch: epoch(manifest.schema),
+      schema: manifest.schema,
+      defaults: annotation[locator.id] ?? manifest.defaults
+    }
+
+  return values
 }
 
 function createSecrets (values: object): Variable[] {
@@ -57,27 +89,45 @@ function createSecrets (values: object): Variable[] {
   return secrets
 }
 
-function validate (annotation: Annotation, instances: Instance[]): void {
+/**
+ * The service's own resources, and the component values that are the rest of the annotation.
+ *
+ * Every key here names a component, so the one option the service has of its own needs a
+ * name that cannot be mistaken for one. `resources` is reserved: a component actually
+ * called that is written with its namespace, `default.resources`, which is what an id is
+ * anyway — the bare form is the shorthand.
+ */
+function split (annotation: Annotation): { resources?: Resources, values: Annotation } {
   validators.annotation.validate(annotation)
 
-  const requested = instances.map((instance) => instance.locator.id)
+  const { resources, ...values } = annotation as Annotation & { resources?: Resources }
+
+  return { resources, values }
+}
 
-  for (let id of Object.keys(annotation)) {
-    if (!id.includes('.')) {
-      const newid = 'default.' + id
+/** Validated, keyed by full component ids, and checked against the components that ask. */
+function prepare (annotation: Annotation, instances: Instance[]): Annotation {
+  const normalized: Annotation = {}
+  const requested = instances.map((instance) => instance.locator.id)
 
-      annotation[newid] = annotation[id]
-      delete annotation[id]
-      id = newid
-    }
+  for (const [key, values] of Object.entries(annotation)) {
+    const id = key.includes('.') ? key : 'default.' + key
 
     assert.ok(requested.includes(id),
       `Component '${id}' does not request configuration or does not exist.`)
+
+    normalized[id] = values
   }
-}
 
-export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
-export const PREFIX = 'TOA_CONFIGURATION_'
+  return normalized
+}
 
-export type Annotation = Record<string, object>
+export type Annotation = Record<string, any>
 export type Instance = context.Dependency<Manifest>
+export type Values = Record<string, Entry>
+
+export interface Entry {
+  epoch: string
+  schema: object
+  defaults?: object
+}