@toa.io/extensions.configuration 1.0.0-alpha.28 → 1.0.0-alpha.281

package/source/UI.ts

@@ -0,0 +1,212 @@
+import * as fs from 'node:fs'
+import * as http from 'node:http'
+import * as path from 'node:path'
+import { Connector } from '@toa.io/core'
+import { console } from 'openspan'
+import { UI_PATH } from './const.js'
+
+/**
+ * Serves the configuration UI: the directory `ui` builds, and nothing else.
+ *
+ * The page is a single-page application whose mount path is baked into the bundle,
+ * so this server routes relative to `UI_PATH` — which is also what the ingress
+ * forwards. `/configuration/*` is left alone: that is the component's own API.
+ */
+export class UI extends Connector {
+  private readonly server: http.Server = http.createServer()
+  private readonly port: number
+  private readonly root: string
+
+  public constructor (port: number, root: string = SITE) {
+    super()
+
+    this.port = port
+    this.root = root
+    this.server.on('request', (request, response) => {
+      this.listen(request, response)
+    })
+  }
+
+  protected override async open (): Promise<void> {
+    /*
+     * A taken port is a real error: uniqueness across services is settled at export
+     * time, so nothing here has to negotiate for one.
+     */
+    await new Promise<void>((resolve, reject) => {
+      const failed = (error: Error): void => {
+        this.server.off('listening', listening)
+        reject(error)
+      }
+
+      const listening = (): void => {
+        this.server.off('error', failed)
+        resolve()
+      }
+
+      this.server.once('error', failed)
+      this.server.once('listening', listening)
+      this.server.listen(this.port)
+    })
+
+    if (!isFile(path.join(this.root, 'index.html')))
+      console.warn('Configuration UI is not built', { root: this.root })
+
+    console.info('Configuration UI started', { port: this.port, path: UI_PATH + '/' })
+  }
+
+  protected override async close (): Promise<void> {
+    const closed = new Promise<void>((resolve) => this.server.once('close', () => {
+      resolve()
+    }))
+
+    this.server.close()
+    this.server.closeAllConnections()
+
+    await closed
+  }
+
+  private listen (request: http.IncomingMessage, response: http.ServerResponse): void {
+    void this.respond(request, response).catch((error: Error) => {
+      console.error('Configuration UI failure', { message: error.message })
+
+      if (!response.writableEnded)
+        response.writeHead(500).end()
+    })
+  }
+
+  private async respond (request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    if (request.method !== 'GET' && request.method !== 'HEAD') {
+      response.writeHead(405, { allow: 'GET, HEAD' }).end()
+
+      return
+    }
+
+    const pathname = decode(request.url ?? '/')
+
+    if (pathname === null) {
+      response.writeHead(400).end()
+
+      return
+    }
+
+    // the port is the explorer's alone, so the root is the way in
+    if (pathname === '/') {
+      response.writeHead(302, { location: UI_PATH + '/' }).end()
+
+      return
+    }
+
+    const file = this.resolve(pathname)
+
+    if (file === null)
+      response.writeHead(404).end()
+    else
+      await this.send(file, request, response)
+  }
+
+  /**
+   * The file a request lands on, or `null` when nothing does. A path that exists is
+   * served as it is; anything else that could be a route falls back to the page,
+   * because the client router — not this server — knows what routes there are.
+   */
+  private resolve (pathname: string): string | null {
+    if (!pathname.startsWith(UI_PATH))
+      return null
+
+    const relative = pathname.slice(UI_PATH.length)
+
+    if (relative !== '' && !relative.startsWith('/'))
+      return null
+
+    const file = path.join(this.root, relative)
+
+    if (file !== this.root && !file.startsWith(this.root + path.sep))
+      return null
+
+    if (isFile(file))
+      return file
+
+    /*
+     * A missing asset is missing, but a route can look like one: `identity.passkeys` is a
+     * component, not a file with an extension. What this server would have served is what
+     * it knows how to serve, so anything else is a route and falls back to the page — as
+     * does anything ending in a slash, which is no name for a file at all.
+     */
+    const asset = !relative.endsWith('/') && path.extname(relative) in TYPES
+
+    return asset ? null : path.join(this.root, 'index.html')
+  }
+
+  private async send (file: string, request: http.IncomingMessage, response: http.ServerResponse): Promise<void> {
+    const stats = await fs.promises.stat(file).catch(() => null)
+
+    if (stats === null) {
+      response.writeHead(503, { 'content-type': 'text/plain' })
+        .end('The configuration UI is not built. Run `npm run build:ui`.\n')
+
+      return
+    }
+
+    response.writeHead(200, {
+      'content-type': TYPES[path.extname(file)] ?? 'application/octet-stream',
+      'content-length': stats.size,
+      'cache-control': caching(path.relative(this.root, file))
+    })
+
+    if (request.method === 'HEAD')
+      response.end()
+    else
+      fs.createReadStream(file).pipe(response)
+  }
+}
+
+function decode (url: string): string | null {
+  try {
+    return decodeURIComponent(url.split('?')[0])
+  } catch {
+    return null
+  }
+}
+
+function isFile (file: string): boolean {
+  return fs.existsSync(file) && fs.statSync(file).isFile()
+}
+
+/** Where `npm run build:ui` puts the page, from both `source` and `transpiled`. */
+const SITE = path.resolve(import.meta.dirname, '..', 'ui', 'dist')
+
+/** Assets under this prefix carry their build hash in the name. */
+const IMMUTABLE = path.join('_app', 'immutable')
+const FOREVER = 'public, max-age=31536000, immutable'
+
+/** The icons: named without a hash, so they are re-read, but rarely. */
+const ICONS = new Set(['favicon.ico', 'favicon-96x96.png', 'apple-touch-icon.png'])
+const DAY = 'public, max-age=86400'
+
+/**
+ * How long what is served may be held. The page is never cached — it names the assets,
+ * and their names carry the build. An icon is named for what it is rather than for its
+ * content, so it is asked about again, but not on every page load.
+ */
+function caching (relative: string): string {
+  if (relative.startsWith(IMMUTABLE)) return FOREVER
+
+  return ICONS.has(relative) ? DAY : 'no-cache'
+}
+
+const TYPES: Record<string, string> = {
+  '.css': 'text/css; charset=utf-8',
+  '.html': 'text/html; charset=utf-8',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.map': 'application/json; charset=utf-8',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml',
+  '.txt': 'text/plain; charset=utf-8',
+  '.webmanifest': 'application/manifest+json',
+  '.webp': 'image/webp',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2'
+}

package/source/configuration.test.ts

@@ -1,15 +1,23 @@
-import { encode } from '@toa.io/generic'
+import { describe, it, beforeEach, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+
 import { Locator } from '@toa.io/core'
 import { generate } from 'randomstring'
-import { get } from './configuration'
-import { type Manifest } from './manifest'
+import { fit, local, overridden } from './configuration.js'
+import { Secret } from './Secret.js'
+import { type Manifest } from './manifest.js'
 
 let locator: Locator
 let manifest: Manifest
 
 beforeEach(() => {
   locator = new Locator(generate(), generate())
-  manifest = { schema: { foo: 'string' } }
+  manifest = {
+    schema: {
+      type: 'object',
+      properties: { foo: { type: 'string' } }
+    }
+  }
 })
 
 afterEach(() => {
@@ -19,67 +27,115 @@ afterEach(() => {
   used = []
 })
 
-it('should read value', async () => {
-  manifest.schema = { foo: 'string' }
+describe('overridden', () => {
+  it('should be false without the variable', async () => {
+    assert.deepStrictEqual(overridden(locator), false)
+  })
 
-  const value: object = { foo: generate() }
+  it('should be true with the variable', async () => {
+    set({})
 
-  set(value)
+    assert.deepStrictEqual(overridden(locator), true)
+  })
+})
 
-  const result = get(locator, manifest)
+describe('local', () => {
+  it('should read value', async () => {
+    const value: object = { foo: generate() }
 
-  expect(result).toStrictEqual(value)
-})
+    set(value)
 
-it('should return empty object if no value set', async () => {
-  expect(get(locator, manifest)).toStrictEqual({})
-})
+    assert.deepStrictEqual(local(locator, manifest), value)
+  })
 
-it('should substitute secrets', async () => {
-  const value: object = { foo: '$BAR' }
+  it('should return empty object if no value set', async () => {
+    assert.deepStrictEqual(local(locator, manifest), {})
+  })
 
-  set(value)
-  set('bar', '_BAR')
+  it('should substitute secrets', async () => {
+    set({ foo: '$BAR' })
+    set('bar', '_BAR')
 
-  const result = get(locator, manifest)
+    const { foo } = local(locator, manifest)
 
-  expect(result).toStrictEqual({ foo: 'bar' })
-})
+    assert.ok(foo instanceof Secret)
+    assert.deepStrictEqual((foo as Secret).unwrap(), 'bar')
+    assert.deepStrictEqual(String(foo), '<REDACTED>')
+  })
+
+  it('should substitute secrets in defaults', async () => {
+    manifest.defaults = { foo: '$BAR' }
 
-it('should use defaults', async () => {
-  manifest.schema = { foo: 'string', bar: ['number'], 'baz?': 'string' }
-  manifest.defaults = { foo: 'bar', bar: [1] }
+    set('bar', '_BAR')
 
-  const values = { bar: [2], baz: 'foo' }
+    assert.deepStrictEqual((local(locator, manifest).foo as Secret).unwrap(), 'bar')
+  })
+
+  it('should use defaults', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'array', items: { type: 'number' } },
+        baz: { type: 'string' }
+      },
+      required: ['foo', 'bar']
+    }
+    manifest.defaults = { foo: 'bar', bar: [1] }
+
+    set({ bar: [2], baz: 'foo' })
+
+    assert.deepStrictEqual(local(locator, manifest), { foo: 'bar', bar: [2], baz: 'foo' })
+  })
 
-  set(values)
+  it('should validate', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string', default: 'hello' },
+        bar: { type: 'number' }
+      }
+    }
 
-  const result = get(locator, manifest)
+    set({ bar: 5 })
 
-  expect(result).toStrictEqual({
-    foo: 'bar',
-    bar: [2],
-    baz: 'foo'
+    assert.deepStrictEqual(local(locator, manifest), { foo: 'hello', bar: 5 })
   })
 })
 
-it('should validate', async () => {
-  manifest.schema = { foo: 'hello', bar: 'number' }
+describe('fit', () => {
+  it('should substitute secrets and apply the schema', async () => {
+    manifest.schema = {
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number', default: 1 }
+      }
+    }
 
-  const values = { bar: 5 }
+    set('secret', '_FOO')
 
-  set(values)
+    const raw = { foo: '$FOO' }
+    const values = fit(raw, manifest)
 
-  const result = get(locator, manifest)
+    assert.deepStrictEqual((values.foo as Secret).unwrap(), 'secret')
+    assert.deepStrictEqual(values.bar, 1)
+    assert.deepStrictEqual(raw, { foo: '$FOO' }) // untouched
+  })
+
+  it('should not apply the manifest defaults', async () => {
+    manifest.defaults = { foo: 'hello' }
+
+    assert.deepStrictEqual(fit({}, manifest), {})
+  })
 
-  expect(result).toStrictEqual({
-    foo: 'hello',
-    bar: 5
+  it('should throw on a value not fitting the schema', async () => {
+    assert.throws(() => fit({ foo: { nested: true } }, manifest))
   })
 })
 
 function set (value: object | string, key = locator.uppercase): void {
-  const string = typeof value === 'string' ? value : encode(value)
+  const string = typeof value === 'string' ? value : JSON.stringify(value)
   const name = 'TOA_CONFIGURATION_' + key
 
   process.env[name] = string

package/source/configuration.ts

@@ -1,33 +1,52 @@
 import { type Locator } from '@toa.io/core'
-import { decode, add } from '@toa.io/generic'
+import { add } from '@toa.io/generic'
 import * as schemas from '@toa.io/schemas'
-import { PREFIX, SECRET_RX } from './deployment'
-import { type Manifest } from './manifest'
+import { PREFIX, SECRET_RX } from './const.js'
+import { Secret } from './Secret.js'
 import type { Schema } from '@toa.io/schemas'
+import type { Manifest } from './manifest.js'
 
-export function get (locator: Locator, manifest: Manifest): Node {
-  const values = getConfiguration(locator.uppercase)
+/** The variable is set, so the values service is not consulted. */
+export function overridden (locator: Locator): boolean {
+  return process.env[PREFIX + locator.uppercase] !== undefined
+}
 
-  substituteSecrets(values)
+/** The variable, the manifest defaults, then the schema. */
+export function local (locator: Locator, manifest: Manifest): Node {
+  const values = read(locator.uppercase)
 
   if (manifest.defaults !== undefined)
     add(values, manifest.defaults)
 
-  const schema: Schema<any> = schemas.schema(manifest.schema)
+  return fit(values, manifest)
+}
 
-  schema.validate(values)
+/** A copy of the value with the schema applied and the secrets substituted. */
+export function fit (raw: object, manifest: Manifest): Node {
+  // a copy of the caller's, and one of this realm: what came over the wire is JSON anyway
+  const values = JSON.parse(JSON.stringify(raw)) as Node
+
+  // the schema sees the references, which are the strings it declares
+  validate(values, manifest)
+  substituteSecrets(values)
 
   return values
 }
 
-function getConfiguration (suffix: string): Node {
+function validate (values: Node, manifest: Manifest): void {
+  const schema: Schema<any> = schemas.schema(manifest.schema)
+
+  schema.validate(values)
+}
+
+function read (suffix: string): Node {
   const variable = PREFIX + suffix
   const string = process.env[variable]
 
   if (string === undefined)
     return {}
   else
-    return decode(string)
+    return JSON.parse(string)
 }
 
 function substituteSecrets (configuration: Node): void {
@@ -43,7 +62,7 @@ function substituteSecrets (configuration: Node): void {
 
     const name = match.groups?.variable
 
-    configuration[key] = getSecret(name!)
+    configuration[key] = new Secret(getSecret(name!))
   }
 }
 

package/source/const.ts

@@ -0,0 +1,17 @@
+import type { Source } from '@toa.io/core'
+
+/** Per-component variables: the local override and the secrets. */
+export const PREFIX = 'TOA_CONFIGURATION_'
+
+/** The map of every configured component, on the values service. */
+export const VALUES = 'TOA_CONFIGURATION_VALUES'
+
+export const SECRET_RX = /^\$(?<variable>[A-Z0-9_]{1,32})$/
+
+/** Where the UI is mounted; `/configuration/*` belongs to the component's own API. */
+export const UI_PATH = '/.configuration'
+export const UI_PORT = 8003
+
+export const EVENT = 'configuration.values.created'
+
+export const SOURCE: Source = { service: 'configuration' }

package/source/deployment.test.ts

@@ -1,21 +1,87 @@
-import { type Annotation, deployment, type Instance } from './deployment'
-import { type Manifest } from './manifest'
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { Locator } from '@toa.io/core'
+import { type Annotation, type Instance, deployment, describe as map } from './deployment.js'
+import { epoch } from './epoch.js'
+import { EVENT, VALUES } from './const.js'
+
+const schema = {
+  type: 'object',
+  properties: { foo: { type: 'string' }, key: { type: 'string' } }
+}
+
+function instance (name: string, defaults?: Record<string, unknown>): Instance {
+  return {
+    locator: new Locator(name, 'configuration'),
+    manifest: { schema, defaults },
+    component: {} as any
+  }
+}
 
 it('should validate annotation', async () => {
   const wrongType = 'not ok' as unknown as Annotation
 
-  expect(() => deployment([], wrongType)).toThrow('object')
+  assert.throws(() => deployment([], wrongType), (error: any) => /object/.test(error.message))
 })
 
-it('should validate values', async () => {
-  const manifest: Manifest = {
-    schema: { foo: 'string', bar: 'number' },
-    defaults: { foo: 'ok', bar: 0 }
-  }
+it('should reject unknown components', async () => {
+  assert.throws(() => deployment([instance('base')], { 'configuration.nope': {} }), (error: any) => /Component 'configuration\.nope' does not request configuration/.test(error.message))
+})
+
+it('should deploy the values service with the map', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const annotation = { 'configuration.other': { foo: 'set' } }
+
+  const dependency = deployment(instances, annotation)
+
+  assert.deepStrictEqual(dependency.events, [EVENT])
+  assert.strictEqual(dependency.services?.length, 1)
+
+  const service = dependency.services![0]
+
+  assert.deepStrictEqual(service.group, 'configuration')
+  assert.deepStrictEqual(service.name, 'values')
+  assert.deepStrictEqual(service.components, ['configuration-values'])
+
+  const variable = service.variables!.find((variable) => variable.name === VALUES)
+
+  assert.notStrictEqual(variable, undefined)
+  assert.deepStrictEqual(JSON.parse(variable!.value!), map(instances, annotation))
+})
+
+it('should describe every component', async () => {
+  const instances = [instance('base', { foo: 'hello' }), instance('other')]
+  const values = map(instances, { 'configuration.other': { foo: 'set' } })
+
+  assert.deepStrictEqual(values, {
+    'configuration.base': { epoch: epoch(schema), schema, defaults: { foo: 'hello' } },
+    'configuration.other': { epoch: epoch(schema), schema, defaults: { foo: 'set' } }
+  })
+})
+
+it('should prefer the context over the manifest defaults', async () => {
+  const values = map([instance('base', { foo: 'hello' })], { 'configuration.base': { foo: 'bye' } })
+
+  assert.deepStrictEqual(values['configuration.base'].defaults, { foo: 'bye' })
+})
+
+it('should not map values to the component', async () => {
+  const dependency = deployment([instance('base')], { 'configuration.base': { foo: 'set' } })
+
+  assert.deepStrictEqual(dependency.variables, {})
+})
+
+it('should map secrets to the component', async () => {
+  const dependency = deployment([instance('base')], { 'configuration.base': { key: '$KEY' } })
 
-  const locator = { id: 'component' }
-  const instances = [{ manifest, locator }] as unknown as Instance[]
-  const annotation: Annotation = { [locator.id]: { bar: 'not a number' } }
+  assert.deepStrictEqual(dependency.variables, {
+    'configuration-base': [{
+      name: 'TOA_CONFIGURATION__KEY',
+      secret: { name: 'toa-configuration', key: 'KEY' }
+    }]
+  })
 
-  expect(() => deployment(instances, annotation)).toThrow('number')
+  // the service holds the reference, not the secret
+  assert.deepStrictEqual(map([instance('base')], { 'configuration.base': { key: '$KEY' } })['configuration.base'].defaults, { key: '$KEY' })
 })