@toa.io/extensions.configuration 1.0.0-alpha.28 → 1.0.0-alpha.282

package/source/Aspect.ts

@@ -1,17 +1,37 @@
-import { Connector, type extensions } from '@toa.io/core'
+import { console } from 'openspan'
+import { Connector, type Locator, type extensions } from '@toa.io/core'
+import { fit, local, type Node } from './configuration.js'
+import { epoch } from './epoch.js'
+import type { Client, Value } from './Client.js'
+import type { Manifest } from './manifest.js'
 
 export class Aspect extends Connector implements extensions.Aspect {
   public readonly name = 'configuration'
 
-  private readonly value: object
+  private readonly locator: Locator
+  private readonly manifest: Manifest
+  private readonly client: Client | null
+  private readonly epoch: string
+  private value: Node = {}
+  private created = 0
 
-  public constructor (value: object) {
+  /**
+   * Without a client the value is local: the variable, the defaults and the schema.
+   * With one, the value is what the service holds, and it follows the service.
+   */
+  public constructor (locator: Locator, manifest: Manifest, client: Client | null) {
     super()
 
-    this.value = value
+    this.locator = locator
+    this.manifest = manifest
+    this.client = client
+    this.epoch = epoch(manifest.schema)
+
+    if (client !== null)
+      this.depends(client)
   }
 
-  public invoke (path: string[]): any {
+  public invoke (path?: string[]): any {
     let cursor: any = this.value
 
     if (path !== undefined)
@@ -20,4 +40,43 @@ export class Aspect extends Connector implements extensions.Aspect {
 
     return cursor
   }
+
+  protected override async open (): Promise<void> {
+    if (this.client === null) {
+      this.value = local(this.locator, this.manifest)
+
+      return
+    }
+
+    const { configuration, created } = await this.client.fetch(this.locator.id, this.epoch)
+
+    this.value = fit(configuration, this.manifest)
+    this.created = created
+    this.client.subscribe(this.locator.id, this.epoch, this.listener)
+
+    console.info('Configuration resolved',
+      { component: this.locator.id, epoch: this.epoch, created })
+  }
+
+  protected override async close (): Promise<void> {
+    this.client?.unsubscribe(this.locator.id, this.epoch, this.listener)
+  }
+
+  private readonly listener = ({ configuration, created }: Value): void => {
+    // deliveries may repeat or cross: only what is newer than the held value replaces it
+    if (created <= this.created)
+      return
+
+    try {
+      this.value = fit(configuration, this.manifest)
+      this.created = created
+
+      console.info('Configuration updated',
+        { component: this.locator.id, epoch: this.epoch, created })
+    } catch (error) {
+      // the service validated it against the schema of its epoch, so the two schemas differ
+      console.error('Configuration value does not match the schema',
+        { component: this.locator.id, epoch: this.epoch, error })
+    }
+  }
 }

package/source/Client.test.ts

@@ -0,0 +1,108 @@
+import { it, beforeEach, afterEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+import { isDeepStrictEqual } from 'node:util'
+
+import { Connector, type Locator, type Message, type Receiver } from '@toa.io/core'
+import { timeout } from '@toa.io/generic'
+import { Client, type Fetched } from './Client.js'
+import { EVENT } from './const.js'
+import type { Host } from './Factory.js'
+
+class Remote extends Connector {
+  public readonly invoke = mock.fn(async (_endpoint: string, request: { input: Array<{ component: string, epoch: string }> }) =>
+    request.input.map((pair): Fetched => ({
+      ...pair,
+      configuration: this.values[pair.component] ?? null,
+      created: this.values[pair.component] === undefined ? 0 : 7
+    })))
+
+  public values: Record<string, object | null> = {}
+}
+
+let remote: Remote
+let receiver: Receiver | null
+let host: Host
+let client: Client
+
+beforeEach(() => {
+  remote = new Remote()
+  receiver = null
+
+  host = {
+    remote: mock.fn(async (locator: Locator) => {
+      assert.deepStrictEqual(locator.id, 'configuration.values')
+
+      return remote
+    }),
+    receive: mock.fn(async (label: string, consumer: Receiver) => {
+      assert.deepStrictEqual(label, EVENT)
+
+      receiver = consumer
+
+      return new Connector()
+    })
+  } as unknown as Host
+
+  client = new Client(host, { base: 10, max: 20, warn: 2 })
+})
+
+afterEach(async () => {
+  await client.disconnect()
+})
+
+it('should send the requests of one tick as one call', async () => {
+  remote.values = { 'a.one': { foo: 1 }, 'a.two': { foo: 2 } }
+
+  await client.connect()
+
+  const [one, two] = await Promise.all([client.fetch('a.one', 'e1'), client.fetch('a.two', 'e2')])
+
+  assert.deepStrictEqual(one, { configuration: { foo: 1 }, created: 7 })
+  assert.deepStrictEqual(two, { configuration: { foo: 2 }, created: 7 })
+  assert.strictEqual(remote.invoke.mock.callCount(), 1)
+  assert.deepStrictEqual(remote.invoke.mock.calls[0].arguments[1], {
+    input: [{ component: 'a.one', epoch: 'e1' }, { component: 'a.two', epoch: 'e2' }]
+  })
+})
+
+it('should keep asking until served', async () => {
+  await client.connect()
+
+  const fetching = client.fetch('a.one', 'e1')
+  const deadline = Date.now() + 1000
+
+  while (remote.invoke.mock.calls.length < 2 && Date.now() < deadline)
+    await timeout(5)
+
+  assert.ok(remote.invoke.mock.calls.length >= 2)
+
+  remote.values = { 'a.one': { foo: 1 } }
+
+  assert.deepStrictEqual((await fetching).configuration, { foo: 1 })
+})
+
+it('should hand a created object to its subscribers', async () => {
+  await client.connect()
+
+  const listener = mock.fn()
+  const other = mock.fn()
+
+  client.subscribe('a.one', 'e1', listener)
+  client.subscribe('a.one', 'e0', other)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 2 }, _created: 12 }
+  } satisfies Message)
+
+  assert.ok(listener.mock.calls.some((call: any) => call.arguments.length === 1 && isDeepStrictEqual(call.arguments[0], { configuration: { foo: 2 }, created: 12 })))
+  assert.strictEqual(other.mock.callCount(), 0)
+  assert.strictEqual(remote.invoke.mock.callCount(), 0)
+
+  client.unsubscribe('a.one', 'e1', listener)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 3 }, _created: 13 }
+  } satisfies Message)
+
+  assert.strictEqual(listener.mock.callCount(), 1)
+})

package/source/Client.ts

@@ -0,0 +1,258 @@
+import { console } from 'openspan'
+import { Connector, Locator } from '@toa.io/core'
+import { EVENT, SOURCE } from './const.js'
+import type { Message, Remote } from '@toa.io/core'
+import type { Host } from './Factory.js'
+
+/**
+ * One per process: one remote to the values service and one subscription to its events,
+ * shared by every Aspect. What the Aspects ask for is collected and sent as one call;
+ * what the service creates afterwards is handed to whoever subscribed.
+ */
+export class Client extends Connector {
+  /** Disconnected once, a connector keeps what it depended on, so a gone client is not reused. */
+  public disposed = false
+
+  private readonly host: Host
+  private readonly options: Options
+  private readonly pending = new Map<string, Pending>()
+  private readonly listeners = new Map<string, Set<Listener>>()
+  private remote: Remote | null = null
+  private timer: NodeJS.Timeout | null = null
+  private flushing = false
+  private fresh = false
+  private round = 0
+
+  public constructor (host: Host, options: Partial<Options> = {}) {
+    super()
+
+    this.host = host
+    this.options = { ...DEFAULTS, ...options }
+  }
+
+  /** The configuration of a component for an epoch, once the service has one. */
+  public async fetch (component: string, epoch: string): Promise<Value> {
+    const key = id(component, epoch)
+
+    let entry = this.pending.get(key)
+
+    if (entry === undefined) {
+      entry = { component, epoch, waiters: [] }
+      this.pending.set(key, entry)
+    }
+
+    const promise = new Promise<Value>((resolve) => {
+      entry!.waiters.push(resolve)
+    })
+
+    if (this.flushing)
+      this.fresh = true
+    else
+      this.schedule(0)
+
+    return await promise
+  }
+
+  public subscribe (component: string, epoch: string, listener: Listener): void {
+    const key = id(component, epoch)
+
+    if (!this.listeners.has(key))
+      this.listeners.set(key, new Set())
+
+    this.listeners.get(key)!.add(listener)
+  }
+
+  public unsubscribe (component: string, epoch: string, listener: Listener): void {
+    this.listeners.get(id(component, epoch))?.delete(listener)
+  }
+
+  protected override async open (): Promise<void> {
+    this.remote = await this.host.remote(LOCATOR, SOURCE)
+
+    this.depends(this.remote)
+    await this.remote.connect()
+
+    const subscription = new Subscription(this.deliver.bind(this))
+    const consumer = await this.host.receive(EVENT, subscription)
+
+    this.depends(consumer)
+    await consumer.connect()
+  }
+
+  protected override async close (): Promise<void> {
+    if (this.timer !== null) {
+      clearTimeout(this.timer)
+      this.timer = null
+    }
+  }
+
+  protected override async dispose (): Promise<void> {
+    this.disposed = true
+  }
+
+  private schedule (delay: number): void {
+    if (this.flushing)
+      return
+
+    if (this.timer !== null) {
+      // a request that has just arrived does not wait for the round already planned
+      if (delay > 0)
+        return
+
+      clearTimeout(this.timer)
+    }
+
+    this.timer = setTimeout(() => {
+      void this.flush()
+    }, delay)
+  }
+
+  private async flush (): Promise<void> {
+    this.timer = null
+    this.flushing = true
+    this.fresh = false
+
+    const batch = [...this.pending.values()]
+    const input = batch.map(({ component, epoch }) => ({ component, epoch }))
+
+    try {
+      const output = await this.remote!.invoke<Fetched[] | Error>('fetch', { input })
+
+      if (output instanceof Error)
+        throw output
+
+      this.settle(output)
+    } catch (error) {
+      console.warn('Configuration fetch failed', { error })
+    } finally {
+      this.flushing = false
+    }
+
+    this.report(batch)
+
+    if (this.pending.size === 0)
+      return
+
+    if (this.fresh)
+      this.schedule(0)
+    else
+      this.schedule(this.backoff())
+  }
+
+  /** Those the service has served are told; the rest stay for the next round. */
+  private settle (output: Fetched[]): void {
+    for (const { component, epoch, configuration, created } of output) {
+      if (configuration === null)
+        continue
+
+      const key = id(component, epoch)
+      const entry = this.pending.get(key)
+
+      if (entry === undefined)
+        continue
+
+      this.pending.delete(key)
+
+      for (const resolve of entry.waiters)
+        resolve({ configuration, created })
+    }
+  }
+
+  private report (batch: Pending[]): void {
+    const waiting = batch
+      .filter(({ component, epoch }) => this.pending.has(id(component, epoch)))
+      .map(({ component }) => component)
+
+    if (waiting.length === 0) {
+      this.round = 0
+
+      return
+    }
+
+    this.round++
+
+    if (this.round % this.options.warn === 1)
+      console.warn('Waiting for configuration', { components: waiting, round: this.round })
+  }
+
+  private backoff (): number {
+    return Math.min(this.options.base * Math.pow(FACTOR, this.round), this.options.max)
+  }
+
+  /** A created object goes to the subscribers of its component and epoch, as it is. */
+  private deliver (created: Created): void {
+    const listeners = this.listeners.get(id(created.component, created.epoch))
+
+    if (listeners === undefined)
+      return
+
+    const value: Value = { configuration: created.configuration, created: created._created }
+
+    for (const listener of listeners)
+      listener(value)
+  }
+}
+
+/** What the event consumer hands deliveries to. */
+class Subscription extends Connector {
+  private readonly handler: (created: Created) => void
+
+  public constructor (handler: (created: Created) => void) {
+    super()
+
+    this.handler = handler
+  }
+
+  public async receive (message: Message<Created>): Promise<void> {
+    this.handler(message.payload)
+  }
+}
+
+function id (component: string, epoch: string): string {
+  return component + '\0' + epoch
+}
+
+const LOCATOR = new Locator('values', 'configuration')
+const FACTOR = 1.5
+
+const DEFAULTS: Options = {
+  base: 1000,
+  max: 10000,
+  warn: 5
+}
+
+export interface Options {
+  base: number
+  max: number
+  /** Every n-th round of waiting is reported. */
+  warn: number
+}
+
+/** A configuration and when it was created; `0` for the deployed defaults. */
+export interface Value {
+  configuration: object
+  created: number
+}
+
+export interface Fetched {
+  component: string
+  epoch: string
+  configuration: object | null
+  created: number
+}
+
+/** The `configuration.values.created` payload: the object as stored. */
+export interface Created {
+  component: string
+  epoch: string
+  configuration: object
+  _created: number
+}
+
+export type Listener = (value: Value) => void
+
+interface Pending {
+  component: string
+  epoch: string
+  waiters: Array<(value: Value) => void>
+}

package/source/Composition.ts

@@ -0,0 +1,57 @@
+import { readdirSync, type Dirent } from 'node:fs'
+import { resolve } from 'node:path'
+import { Connector } from '@toa.io/core'
+import { UI_PORT } from './const.js'
+import { UI } from './UI.js'
+import type { Host } from './Factory.js'
+
+/** Hosts the values component in the service process. */
+export class Composition extends Connector {
+  private readonly host: Host
+
+  public constructor (host: Host) {
+    super()
+    this.host = host
+  }
+
+  protected override async open (): Promise<void> {
+    const composition = await this.host.composition(components().paths)
+
+    await composition.connect()
+
+    this.depends(composition)
+
+    // connected here rather than declared as a dependency: dependencies are connected
+    // before `open` runs, so one added from inside it would never be
+    const ui = new UI(UI_PORT)
+
+    await ui.connect()
+
+    this.depends(ui)
+  }
+}
+
+export function components (): Components {
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+function entries (): Dirent[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries.filter((entry) => entry.isDirectory())
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
+const ROOT = resolve(import.meta.dirname, '../components/')

package/source/Factory.ts

@@ -1,12 +1,34 @@
-import { type Locator, type extensions } from '@toa.io/core'
-import { Aspect } from './Aspect'
-import { type Manifest } from './manifest'
-import { get } from './configuration'
+import { type Connector, type Locator, type extensions } from '@toa.io/core'
+import { Aspect } from './Aspect.js'
+import { Client } from './Client.js'
+import { Composition } from './Composition.js'
+import { overridden } from './configuration.js'
+import type { Manifest } from './manifest.js'
 
 export class Factory implements extensions.Factory {
+  private readonly host: Host
+  private client: Client | null = null
+
+  public constructor (host: Host) {
+    this.host = host
+  }
+
   public aspect (locator: Locator, manifest: Manifest): extensions.Aspect {
-    const value = get(locator, manifest)
+    const client = overridden(locator) ? null : this.shared()
+
+    return new Aspect(locator, manifest, client)
+  }
 
-    return new Aspect(value)
+  public service (): Connector {
+    return new Composition(this.host)
+  }
+
+  private shared (): Client {
+    if (this.client === null || this.client.disposed)
+      this.client = new Client(this.host)
+
+    return this.client
   }
 }
+
+export type Host = extensions.Host

package/source/Secret.test.ts

@@ -0,0 +1,32 @@
+import { it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { inspect } from 'node:util'
+import { REDACTED, Secret } from './Secret.js'
+
+const secret = new Secret('s3cret')
+
+it('should unwrap', () => {
+  assert.deepStrictEqual(secret.unwrap(), 's3cret')
+})
+
+it('should not show as a string', () => {
+  assert.deepStrictEqual(String(secret), REDACTED)
+  assert.deepStrictEqual(`${secret}`, REDACTED)
+  assert.deepStrictEqual('' + secret, REDACTED)
+})
+
+it('should not show in JSON', () => {
+  assert.deepStrictEqual(JSON.stringify({ key: secret }), '{"key":"<REDACTED>"}')
+})
+
+it('should not show when inspected', () => {
+  assert.deepStrictEqual(inspect(secret), REDACTED)
+  assert.ok(inspect({ key: secret }).includes(REDACTED))
+  assert.ok(!(inspect({ key: secret }).includes('s3cret')))
+})
+
+it('should not expose the value as a property', () => {
+  assert.deepStrictEqual(Object.keys(secret), [])
+  assert.deepStrictEqual({ ...secret }, {})
+})

package/source/Secret.ts

@@ -0,0 +1,29 @@
+import { inspect } from 'node:util'
+import type { Secret as Contract } from '@toa.io/types'
+
+/** A configuration value that must not leak: a string only to whoever asks for it. */
+export class Secret implements Contract {
+  readonly #value: string
+
+  public constructor (value: string) {
+    this.#value = value
+  }
+
+  public unwrap (): string {
+    return this.#value
+  }
+
+  public toString (): string {
+    return REDACTED
+  }
+
+  public toJSON (): string {
+    return REDACTED
+  }
+
+  public [inspect.custom] (): string {
+    return REDACTED
+  }
+}
+
+export const REDACTED = '<REDACTED>'