@toa.io/extensions.configuration 1.0.0-alpha.273 → 1.0.0-alpha.274

package/source/Client.test.ts

@@ -0,0 +1,104 @@
+import { Connector, type Locator, type Message, type Receiver } from '@toa.io/core'
+import { timeout } from '@toa.io/generic'
+import { Client, type Fetched } from './Client'
+import { EVENT } from './const'
+import type { Bootloader } from './Factory'
+
+class Remote extends Connector {
+  public readonly invoke = jest.fn(async (_endpoint: string, request: { input: Array<{ component: string, epoch: string }> }) =>
+    request.input.map((pair): Fetched => ({
+      ...pair,
+      configuration: this.values[pair.component] ?? null,
+      created: this.values[pair.component] === undefined ? 0 : 7
+    })))
+
+  public values: Record<string, object | null> = {}
+}
+
+let remote: Remote
+let receiver: Receiver | null
+let boot: Bootloader
+let client: Client
+
+beforeEach(() => {
+  remote = new Remote()
+  receiver = null
+
+  boot = {
+    remote: jest.fn(async (locator: Locator) => {
+      expect(locator.id).toStrictEqual('configuration.values')
+
+      return remote
+    }),
+    receive: jest.fn(async (label: string, consumer: Receiver) => {
+      expect(label).toStrictEqual(EVENT)
+
+      receiver = consumer
+
+      return new Connector()
+    })
+  } as unknown as Bootloader
+
+  client = new Client(boot, { base: 10, max: 20, warn: 2 })
+})
+
+afterEach(async () => {
+  await client.disconnect()
+})
+
+it('should send the requests of one tick as one call', async () => {
+  remote.values = { 'a.one': { foo: 1 }, 'a.two': { foo: 2 } }
+
+  await client.connect()
+
+  const [one, two] = await Promise.all([client.fetch('a.one', 'e1'), client.fetch('a.two', 'e2')])
+
+  expect(one).toStrictEqual({ configuration: { foo: 1 }, created: 7 })
+  expect(two).toStrictEqual({ configuration: { foo: 2 }, created: 7 })
+  expect(remote.invoke).toHaveBeenCalledTimes(1)
+  expect(remote.invoke.mock.calls[0][1]).toStrictEqual({
+    input: [{ component: 'a.one', epoch: 'e1' }, { component: 'a.two', epoch: 'e2' }]
+  })
+})
+
+it('should keep asking until served', async () => {
+  await client.connect()
+
+  const fetching = client.fetch('a.one', 'e1')
+  const deadline = Date.now() + 1000
+
+  while (remote.invoke.mock.calls.length < 2 && Date.now() < deadline)
+    await timeout(5)
+
+  expect(remote.invoke.mock.calls.length).toBeGreaterThanOrEqual(2)
+
+  remote.values = { 'a.one': { foo: 1 } }
+
+  expect((await fetching).configuration).toStrictEqual({ foo: 1 })
+})
+
+it('should hand a created object to its subscribers', async () => {
+  await client.connect()
+
+  const listener = jest.fn()
+  const other = jest.fn()
+
+  client.subscribe('a.one', 'e1', listener)
+  client.subscribe('a.one', 'e0', other)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 2 }, _created: 12 }
+  } satisfies Message)
+
+  expect(listener).toHaveBeenCalledWith({ configuration: { foo: 2 }, created: 12 })
+  expect(other).not.toHaveBeenCalled()
+  expect(remote.invoke).not.toHaveBeenCalled()
+
+  client.unsubscribe('a.one', 'e1', listener)
+
+  await receiver!.receive({
+    payload: { component: 'a.one', epoch: 'e1', configuration: { foo: 3 }, _created: 13 }
+  } satisfies Message)
+
+  expect(listener).toHaveBeenCalledTimes(1)
+})

package/source/Client.ts

@@ -0,0 +1,258 @@
+import { console } from 'openspan'
+import { Connector, Locator } from '@toa.io/core'
+import { EVENT, SOURCE } from './const'
+import type { Message, Remote } from '@toa.io/core'
+import type { Bootloader } from './Factory'
+
+/**
+ * One per process: one remote to the values service and one subscription to its events,
+ * shared by every Aspect. What the Aspects ask for is collected and sent as one call;
+ * what the service creates afterwards is handed to whoever subscribed.
+ */
+export class Client extends Connector {
+  /** Disconnected once, a connector keeps what it depended on, so a gone client is not reused. */
+  public disposed = false
+
+  private readonly boot: Bootloader
+  private readonly options: Options
+  private readonly pending = new Map<string, Pending>()
+  private readonly listeners = new Map<string, Set<Listener>>()
+  private remote: Remote | null = null
+  private timer: NodeJS.Timeout | null = null
+  private flushing = false
+  private fresh = false
+  private round = 0
+
+  public constructor (boot: Bootloader, options: Partial<Options> = {}) {
+    super()
+
+    this.boot = boot
+    this.options = { ...DEFAULTS, ...options }
+  }
+
+  /** The configuration of a component for an epoch, once the service has one. */
+  public async fetch (component: string, epoch: string): Promise<Value> {
+    const key = id(component, epoch)
+
+    let entry = this.pending.get(key)
+
+    if (entry === undefined) {
+      entry = { component, epoch, waiters: [] }
+      this.pending.set(key, entry)
+    }
+
+    const promise = new Promise<Value>((resolve) => {
+      entry!.waiters.push(resolve)
+    })
+
+    if (this.flushing)
+      this.fresh = true
+    else
+      this.schedule(0)
+
+    return await promise
+  }
+
+  public subscribe (component: string, epoch: string, listener: Listener): void {
+    const key = id(component, epoch)
+
+    if (!this.listeners.has(key))
+      this.listeners.set(key, new Set())
+
+    this.listeners.get(key)!.add(listener)
+  }
+
+  public unsubscribe (component: string, epoch: string, listener: Listener): void {
+    this.listeners.get(id(component, epoch))?.delete(listener)
+  }
+
+  protected override async open (): Promise<void> {
+    this.remote = await this.boot.remote(LOCATOR, SOURCE)
+
+    this.depends(this.remote)
+    await this.remote.connect()
+
+    const subscription = new Subscription(this.deliver.bind(this))
+    const consumer = await this.boot.receive(EVENT, subscription)
+
+    this.depends(consumer)
+    await consumer.connect()
+  }
+
+  protected override async close (): Promise<void> {
+    if (this.timer !== null) {
+      clearTimeout(this.timer)
+      this.timer = null
+    }
+  }
+
+  protected override async dispose (): Promise<void> {
+    this.disposed = true
+  }
+
+  private schedule (delay: number): void {
+    if (this.flushing)
+      return
+
+    if (this.timer !== null) {
+      // a request that has just arrived does not wait for the round already planned
+      if (delay > 0)
+        return
+
+      clearTimeout(this.timer)
+    }
+
+    this.timer = setTimeout(() => {
+      void this.flush()
+    }, delay)
+  }
+
+  private async flush (): Promise<void> {
+    this.timer = null
+    this.flushing = true
+    this.fresh = false
+
+    const batch = [...this.pending.values()]
+    const input = batch.map(({ component, epoch }) => ({ component, epoch }))
+
+    try {
+      const output = await this.remote!.invoke<Fetched[] | Error>('fetch', { input })
+
+      if (output instanceof Error)
+        throw output
+
+      this.settle(output)
+    } catch (error) {
+      console.warn('Configuration fetch failed', { error })
+    } finally {
+      this.flushing = false
+    }
+
+    this.report(batch)
+
+    if (this.pending.size === 0)
+      return
+
+    if (this.fresh)
+      this.schedule(0)
+    else
+      this.schedule(this.backoff())
+  }
+
+  /** Those the service has served are told; the rest stay for the next round. */
+  private settle (output: Fetched[]): void {
+    for (const { component, epoch, configuration, created } of output) {
+      if (configuration === null)
+        continue
+
+      const key = id(component, epoch)
+      const entry = this.pending.get(key)
+
+      if (entry === undefined)
+        continue
+
+      this.pending.delete(key)
+
+      for (const resolve of entry.waiters)
+        resolve({ configuration, created })
+    }
+  }
+
+  private report (batch: Pending[]): void {
+    const waiting = batch
+      .filter(({ component, epoch }) => this.pending.has(id(component, epoch)))
+      .map(({ component }) => component)
+
+    if (waiting.length === 0) {
+      this.round = 0
+
+      return
+    }
+
+    this.round++
+
+    if (this.round % this.options.warn === 1)
+      console.warn('Waiting for configuration', { components: waiting, round: this.round })
+  }
+
+  private backoff (): number {
+    return Math.min(this.options.base * Math.pow(FACTOR, this.round), this.options.max)
+  }
+
+  /** A created object goes to the subscribers of its component and epoch, as it is. */
+  private deliver (created: Created): void {
+    const listeners = this.listeners.get(id(created.component, created.epoch))
+
+    if (listeners === undefined)
+      return
+
+    const value: Value = { configuration: created.configuration, created: created._created }
+
+    for (const listener of listeners)
+      listener(value)
+  }
+}
+
+/** What the event consumer hands deliveries to. */
+class Subscription extends Connector {
+  private readonly handler: (created: Created) => void
+
+  public constructor (handler: (created: Created) => void) {
+    super()
+
+    this.handler = handler
+  }
+
+  public async receive (message: Message<Created>): Promise<void> {
+    this.handler(message.payload)
+  }
+}
+
+function id (component: string, epoch: string): string {
+  return component + '\0' + epoch
+}
+
+const LOCATOR = new Locator('values', 'configuration')
+const FACTOR = 1.5
+
+const DEFAULTS: Options = {
+  base: 1000,
+  max: 10000,
+  warn: 5
+}
+
+export interface Options {
+  base: number
+  max: number
+  /** Every n-th round of waiting is reported. */
+  warn: number
+}
+
+/** A configuration and when it was created; `0` for the deployed defaults. */
+export interface Value {
+  configuration: object
+  created: number
+}
+
+export interface Fetched {
+  component: string
+  epoch: string
+  configuration: object | null
+  created: number
+}
+
+/** The `configuration.values.created` payload: the object as stored. */
+export interface Created {
+  component: string
+  epoch: string
+  configuration: object
+  _created: number
+}
+
+export type Listener = (value: Value) => void
+
+interface Pending {
+  component: string
+  epoch: string
+  waiters: Array<(value: Value) => void>
+}

package/source/Composition.ts

@@ -0,0 +1,57 @@
+import { readdirSync, type Dirent } from 'node:fs'
+import { resolve } from 'node:path'
+import { Connector } from '@toa.io/core'
+import { UI_PORT } from './const'
+import { UI } from './UI'
+import type { Bootloader } from './Factory'
+
+/** Hosts the values component in the service process. */
+export class Composition extends Connector {
+  private readonly boot: Bootloader
+
+  public constructor (boot: Bootloader) {
+    super()
+    this.boot = boot
+  }
+
+  protected override async open (): Promise<void> {
+    const composition = await this.boot.composition(components().paths)
+
+    await composition.connect()
+
+    this.depends(composition)
+
+    // connected here rather than declared as a dependency: dependencies are connected
+    // before `open` runs, so one added from inside it would never be
+    const ui = new UI(UI_PORT)
+
+    await ui.connect()
+
+    this.depends(ui)
+  }
+}
+
+export function components (): Components {
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+function entries (): Dirent[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries.filter((entry) => entry.isDirectory())
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
+const ROOT = resolve(__dirname, '../components/')

package/source/Factory.ts

@@ -1,12 +1,35 @@
-import { type Locator, type extensions } from '@toa.io/core'
+import { type Connector, type Locator, type extensions } from '@toa.io/core'
 import { Aspect } from './Aspect'
-import { type Manifest } from './manifest'
-import { get } from './configuration'
+import { Client } from './Client'
+import { Composition } from './Composition'
+import { overridden } from './configuration'
+import type { Manifest } from './manifest'
 
 export class Factory implements extensions.Factory {
+  private readonly boot: Bootloader
+  private client: Client | null = null
+
+  public constructor (boot: Bootloader) {
+    this.boot = boot
+  }
+
   public aspect (locator: Locator, manifest: Manifest): extensions.Aspect {
-    const value = get(locator, manifest)
+    const client = overridden(locator) ? null : this.shared()
+
+    return new Aspect(locator, manifest, client)
+  }
 
-    return new Aspect(value)
+  public service (): Connector {
+    return new Composition(this.boot)
+  }
+
+  private shared (): Client {
+    if (this.client === null || this.client.disposed)
+      this.client = new Client(this.boot)
+
+    return this.client
   }
 }
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+export type Bootloader = typeof import('@toa.io/boot')

package/source/Secret.test.ts

@@ -0,0 +1,29 @@
+import { inspect } from 'node:util'
+import { REDACTED, Secret } from './Secret'
+
+const secret = new Secret('s3cret')
+
+it('should unwrap', () => {
+  expect(secret.unwrap()).toStrictEqual('s3cret')
+})
+
+it('should not show as a string', () => {
+  expect(String(secret)).toStrictEqual(REDACTED)
+  expect(`${secret}`).toStrictEqual(REDACTED)
+  expect('' + secret).toStrictEqual(REDACTED)
+})
+
+it('should not show in JSON', () => {
+  expect(JSON.stringify({ key: secret })).toStrictEqual('{"key":"<REDACTED>"}')
+})
+
+it('should not show when inspected', () => {
+  expect(inspect(secret)).toStrictEqual(REDACTED)
+  expect(inspect({ key: secret })).toContain(REDACTED)
+  expect(inspect({ key: secret })).not.toContain('s3cret')
+})
+
+it('should not expose the value as a property', () => {
+  expect(Object.keys(secret)).toStrictEqual([])
+  expect({ ...secret }).toStrictEqual({})
+})

package/source/Secret.ts

@@ -0,0 +1,29 @@
+import { inspect } from 'node:util'
+import type { Secret as Contract } from '@toa.io/types'
+
+/** A configuration value that must not leak: a string only to whoever asks for it. */
+export class Secret implements Contract {
+  readonly #value: string
+
+  public constructor (value: string) {
+    this.#value = value
+  }
+
+  public unwrap (): string {
+    return this.#value
+  }
+
+  public toString (): string {
+    return REDACTED
+  }
+
+  public toJSON (): string {
+    return REDACTED
+  }
+
+  public [inspect.custom] (): string {
+    return REDACTED
+  }
+}
+
+export const REDACTED = '<REDACTED>'