@toa.io/extensions.realtime 1.0.0-alpha.29 → 1.0.0-alpha.291

package/features/steps/Realtime.ts

@@ -1,43 +1,40 @@
 import * as boot from '@toa.io/boot'
-import { encode } from '@toa.io/generic'
 import { type Connector } from '@toa.io/core'
-import { after, binding } from 'cucumber-tsflow'
-import { Factory } from '../../source/Factory'
+import { environment } from '@toa.io/generic'
+
+import tsflow from 'cucumber-tsflow'
+
+import { Factory } from '../../source/index.js'
+import type { Route } from '../../source/extension.js'
+
+const { after, binding } = tsflow
 
 @binding()
 export class Realtime {
   private readonly routes: Route[] = []
-  private readonly service: Connector
+  // the factory loads its binding, so the service is made when it is first served
+  private service: Connector | undefined
   private connected = false
 
-  public constructor () {
-    this.service = new Factory(boot).service()
-  }
-
   @after()
-  private async shutdown (): Promise<void> {
+  public async shutdown(): Promise<void> {
     this.connected = false
 
-    await this.service.disconnect()
+    await this.service?.disconnect()
   }
 
-  public declare (event: string, properties: string[]): void {
-    this.routes.push({ event, properties })
+  public declare(event: string, properties: string[], expose?: string[]): void {
+    this.routes.push({ event, properties, expose })
   }
 
-  public async serve (): Promise<void> {
-    if (this.connected)
-      return
+  public async serve(): Promise<void> {
+    if (this.connected) return
 
-    process.env.TOA_REALTIME = encode(this.routes)
+    environment.set('TOA_REALTIME', JSON.stringify(this.routes))
 
     this.connected = true
+    this.service = await new Factory(boot.host()).service()
 
     await this.service.connect()
   }
 }
-
-interface Route {
-  event: string
-  properties: string[]
-}

package/features/steps/Streams.ts

@@ -1,50 +1,85 @@
-import { type Readable } from 'node:stream'
-import { after, binding, given, then } from 'cucumber-tsflow'
-import { match } from '@toa.io/generic'
+import * as assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
+import tsflow from 'cucumber-tsflow'
 
-import { type Component } from '@toa.io/core'
-import { parse } from '@toa.io/yaml'
+import { match } from '@toa.io/generic'
+import { load as parse } from 'js-yaml'
 import * as stage from '@toa.io/userland/stage'
-import { Realtime } from './Realtime'
+import { Realtime } from './Realtime.js'
+import type { Readable } from 'node:stream'
+import type { Component } from '@toa.io/core'
+
+const { after, binding, given, then } = tsflow
 
 @binding([Realtime])
 export class Streams {
   private readonly realtime: Realtime
   private remote: Component | null = null
   private streams: Record<string, Readable> = {}
-  private events: Record<string, object[]> = {}
+  private events: Record<string, Event[]> = {}
 
-  public constructor (realtime: Realtime) {
+  public constructor(realtime: Realtime) {
     this.realtime = realtime
   }
 
-  @given('the stream `{word}` is consumed')
-  public async consume (key: string): Promise<void> {
+  @given('the stream `{word}` is consumed', { timeout: 30_000 })
+  public async consume(key: string): Promise<void> {
     await this.realtime.serve()
 
     this.remote ??= await stage.remote('realtime.streams')
     this.events[key] = []
-    this.streams[key] = await this.remote.invoke('create', { input: { key } })
-    this.streams[key].on('data', (data: object) => this.events[key].push(data))
+
+    await this.createStream(key)
   }
 
   @then('an event is received from the stream `{word}`:')
-  public received (key: string, yaml: string): void {
-    const expected = parse<object>(yaml)
+  public async received(key: string, yaml: string): Promise<void> {
+    await setTimeout(100)
 
-    for (const event of this.events[key])
-      if (match(event, expected))
-        return
+    const expected = parse(yaml) as object
+
+    for (const event of this.events[key]) if (match(event, expected)) return
 
     throw new Error('No matching event received')
   }
 
+  @then('the consumer `{word}` is disconnected')
+  public disconnected(key: string): void {
+    this.streams[key]?.destroy()
+    delete this.streams[key]
+  }
+
+  @then('the consumer `{word}` is reconnected')
+  public async reconnected(key: string): Promise<void> {
+    const last = this.events[key].findLast((event) => event.event === 'token')
+
+    assert.ok(last, `No last event found for stream ${key}`)
+
+    await this.createStream(key, last.data as string)
+  }
+
   @after()
-  private shutdown (): void {
-    for (const stream of Object.values(this.streams))
-      stream.destroy()
+  public async shutdown(): Promise<void> {
+    for (const stream of Object.values(this.streams)) stream.destroy()
 
     this.streams = {}
     this.events = {}
+
+    await setTimeout(100)
   }
+
+  private async createStream(key: string, token?: string): Promise<void> {
+    this.streams[key] = await this.remote!.invoke('create', { input: { key, token } })
+    this.streams[key].on('data', (event: Event) => {
+      console.log('[TEST] Received event', event)
+      this.events[key]?.push(event)
+    })
+  }
+}
+
+interface Event {
+  key: string
+  token: string
+  event: string
+  data?: unknown
 }

package/features/steps/components/messages/manifest.toa.yaml

@@ -1,16 +1,45 @@
 name: messages
 
 entity:
-  schema:
-    sender*: string(32)
-    recipient*: string(32)
-    text*: string
+  properties:
+    watchers:
+      type: array
+      items:
+        type: string
+        maxLength: 32
+    sender:
+      type: string
+      maxLength: 32
+    recipient:
+      type: string
+      maxLength: 32
+    text:
+      type: string
+  required:
+    - sender
+    - recipient
+    - text
 
 operations:
   create:
     query: false
     forward: transit
     input:
-      sender*: .
-      recipient*: .
-      text*: .
+      properties:
+        watchers:
+          type: string
+          default: .
+        sender:
+          type: string
+          default: .
+        recipient:
+          type: string
+          default: .
+        text:
+          type: string
+          default: .
+      type: object
+      required:
+        - sender
+        - recipient
+        - text

package/features/steps/config.ts

@@ -1 +1,15 @@
-process.env.TOA_DEV = '1'
+import { setDefaultTimeout } from '@cucumber/cucumber'
+import { environment } from '@toa.io/generic'
+
+environment.set('TOA_DEV', '1')
+
+// a reply is checked against what the operation declares, so the suite runs Toa under the
+// contract it asks applications to keep
+if (!environment.has('TOA_ENV')) environment.set('TOA_ENV', 'local')
+
+// the streams component boots inside the service, and without a variable it would wait
+// for the values service, which these features do not run
+if (!environment.has('TOA_CONFIGURATION_REALTIME_STREAMS'))
+  environment.set('TOA_CONFIGURATION_REALTIME_STREAMS', '{}')
+
+setDefaultTimeout(60 * 1000)

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/extensions.realtime",
-  "version": "1.0.0-alpha.29",
+  "version": "1.0.0-alpha.291",
+  "type": "module",
   "description": "Toa Realtime",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -16,13 +17,14 @@
   "publishConfig": {
     "access": "public"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node"
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.291",
+    "@toa.io/generic": "1.0.0-alpha.291",
+    "openspan": "1.0.0-alpha.286"
   },
   "scripts": {
-    "transpile": "npx tsc && npx tsc -p ./components/streams",
-    "features": "npx cucumber-js"
+    "transpile": "npx tsc && npx tsc -p ./components/realtime.streams",
+    "features": "TSX_TSCONFIG_PATH=features/steps/tsconfig.json NODE_OPTIONS=--import=tsx cucumber-js"
   },
-  "gitHead": "186d0e17c12a9016b8076edf925963cbb32b9219"
+  "gitHead": "bcd8d8d07bd3e64fc826af8719472404662cc37f"
 }

package/readme.md

@@ -10,17 +10,20 @@
 </a>
 
 Realtime extension combines application events into streams according to defined routes.
-Clients may consume these streams [via Exposition](#exposition).
+Clients may consume these streams [via Exposition](/extensions/exposition).
+
+If stream is idle for 16 seconds, a `heartbeat` message is sent.
 
 ## Static routes
 
 Static route specifies an event that should be combined into a stream using specified property of
-event's payload as a stream key.
+event's payload as a stream key or an array of stream keys.
 
 Static routes may be defined in Component manifest or the Context annotation.
 
 ```yaml
 # manifest.toa.yaml
+
 name: users
 
 realtime:
@@ -29,13 +32,24 @@ realtime:
 
 ```yaml
 # context.toa.yaml
+
 realtime:
   users.updated: id
-  orders.created: custromer_id
+  orders.created: customer_id
 ```
 
 In case of conflict, the Context annotation takes precedence.
 
+Multiple stream keys may be defined for a single event.
+
+```yaml
+# manifest.toa.yaml
+name: messages
+
+realtime:
+  updated: [sender_id, recipient_id]
+```
+
 ### Static route examples
 
 Given two rules: `users.updated: id` and `orders.created: customer_id`,
@@ -87,13 +101,25 @@ Dynamic routes are managed by the `realtime.routes` component, running in the Re
 
 ## Exposition
 
-Streams are exposed by the [`realtime.streams`](components/streams) component, running in the
+Streams are exposed by the [`realtime.streams`](components/realtime.streams) component, running in the
 Realtime extension, and are
 accessible via the `/realtime/streams/:key/` resource with
 the [`auth:id: key`](/extensions/exposition/documentation/access.md#id) authorization rule.
 
-Refer to the [Exposition extension](/extensions/exposition) for more
-details:
+Refer to the [Exposition extension](/extensions/exposition) for more details:
 
-- [Streams](/extensions/exposition/documentation/protocol.md#streams)
+- [Multipart responses](/extensions/exposition/documentation/protocol.md#multipart-types)
 - [Access authorization](/extensions/exposition/documentation/access.md)
+
+## Resources management
+
+Resource requests and limits can be specified by `resources` annotation:
+
+```yaml
+# context.toa.yaml
+
+realtime:
+  resources:
+    cpu: [100m, 500m]
+    memory: [100Mi, 200Mi]
+```

package/source/Composition.ts

@@ -1,40 +1,51 @@
 import { readdirSync, type Dirent } from 'node:fs'
 import { resolve } from 'node:path'
 import { Connector } from '@toa.io/core'
-import { type Bootloader } from './Factory'
+import { type Host } from './Factory.js'
 
 export class Composition extends Connector {
-  private readonly boot: Bootloader
+  private readonly host: Host
 
-  public constructor (boot: Bootloader) {
+  public constructor(host: Host) {
     super()
-    this.boot = boot
+    this.host = host
   }
 
-  protected override async open (): Promise<void> {
+  protected override async open(): Promise<void> {
     const paths = find()
-    const composition = await this.boot.composition(paths)
+    const composition = await this.host.composition(paths)
 
     await composition.connect()
 
     this.depends(composition)
-
-    console.info('Composition complete.')
-  }
-
-  protected override dispose (): void {
-    console.info('Composition shutdown complete.')
   }
 }
 
-function find (): string[] {
+export function find(): string[] {
   return entries().map((entry) => resolve(ROOT, entry.name))
 }
 
-function entries (): Dirent[] {
+function entries(): Dirent[] {
   const entries = readdirSync(ROOT, { withFileTypes: true })
 
   return entries.filter((entry) => entry.isDirectory())
 }
 
-const ROOT = resolve(__dirname, '../components/')
+export function components(): Components {
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
+const ROOT = resolve(import.meta.dirname, '../components/')

package/source/Factory.ts

@@ -1,20 +1,20 @@
-import { type Component, type extensions, Locator } from '@toa.io/core'
-import { Realtime } from './Realtime'
-import { Composition } from './Composition'
-import { Routes } from './Routes'
+import { type Component, Locator } from '@toa.io/core'
+import type { extensions } from '@toa.io/core/types'
+import { Realtime } from './Realtime.js'
+import { Composition } from './Composition.js'
+import { Routes } from './Routes.js'
 
 export class Factory implements extensions.Factory {
-  private readonly boot: Bootloader
+  private readonly host: Host
 
-  public constructor (boot: Bootloader) {
-    this.boot = boot
+  public constructor(host: Host) {
+    this.host = host
   }
 
-  public service (): Realtime {
-    const discovery = this.discovery()
-    const routes = new Routes(this.boot)
-    const composition = new Composition(this.boot)
-    const realtime = new Realtime(routes, discovery)
+  public service(): Realtime {
+    const routes = new Routes(this.host)
+    const composition = new Composition(this.host)
+    const realtime = new Realtime(routes, async () => await this.discovery())
 
     realtime.depends(routes)
     realtime.depends(composition)
@@ -22,12 +22,11 @@ export class Factory implements extensions.Factory {
     return realtime
   }
 
-  private async discovery (): Promise<Component> {
+  private async discovery(): Promise<Component> {
     const locator = new Locator('streams', 'realtime')
 
-    return await this.boot.remote(locator)
+    return await this.host.remote(locator, { service: 'realtime' })
   }
 }
 
-// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-export type Bootloader = typeof import('@toa.io/boot')
+export type Host = extensions.Host

package/source/Realtime.ts

@@ -1,11 +1,13 @@
+import { console, run } from 'openspan'
 import { type Component, Connector } from '@toa.io/core'
-import { type Routes } from './Routes'
+import { type Routes } from './Routes.js'
+import type { Push } from './Receiver.js'
 
 export class Realtime extends Connector {
-  private readonly discovery: Promise<Component>
+  private readonly discovery: () => Promise<Component>
   private streams: Component | null = null
 
-  public constructor (routes: Routes, discovery: Promise<Component>) {
+  public constructor(routes: Routes, discovery: () => Promise<Component>) {
     super()
 
     this.discovery = discovery
@@ -13,26 +15,57 @@ export class Realtime extends Connector {
     routes.events.on('data', this.push.bind(this))
   }
 
-  protected override async open (): Promise<void> {
-    this.streams = await this.discovery
+  protected override async open(): Promise<void> {
+    // the lookup belongs here, not in the constructor: dependencies connect before
+    // open, so the composition that serves realtime.streams is up to answer it. Asked
+    // any earlier the lookup goes unanswered, and it waits without a bound.
+    this.streams = await this.discovery()
     this.depends(this.streams)
 
     await this.streams.connect()
 
-    console.log('Realtime has started.')
+    console.info('Realtime service started')
   }
 
-  protected override dispose (): void {
-    console.log('Realtime shutdown complete.')
+  protected override dispose(): void {
+    console.info('Realtime service shutdown complete')
   }
 
-  private push (event: Event): void {
-    void this.streams?.invoke('push', { input: event })
+  private push({ telemetry, ...event }: Push): void {
+    const processing =
+      telemetry === null
+        ? this.deliver(event)
+        : run(telemetry, async () => await this.deliver(event))
+
+    void processing.catch((error) => console.error('Realtime push failed', error))
   }
-}
 
-interface Event {
-  key: string
-  event: string
-  data: string
+  private async deliver(event: Omit<Push, 'telemetry'>): Promise<void> {
+    /*
+     * The delivery span is created on behalf of the messaging destination
+     * (same as the core Receiver), so that service graphs display the fan-out:
+     * producer -> destination -> realtime
+     */
+    const delivery = {
+      name: `${event.event} deliver`,
+      kind: 'producer' as const,
+      service: event.event,
+      attributes: { 'messaging.destination.name': event.event }
+    }
+
+    const options = {
+      name: `${event.event} push`,
+      kind: 'consumer' as const,
+      service: 'realtime',
+      attributes: { 'messaging.destination.name': event.event }
+    }
+
+    await console.span(
+      delivery,
+      async () =>
+        await console.span(options, async () => {
+          await this.streams?.invoke('push', { input: event })
+        })
+    )
+  }
 }

package/source/Receiver.ts

@@ -0,0 +1,95 @@
+import { console, decode, run } from 'openspan'
+import { Connector } from '@toa.io/core'
+import type { Message } from '@toa.io/core/types'
+import type { SpanContext } from 'openspan'
+import type { Readable } from 'node:stream'
+
+export class Receiver extends Connector {
+  private readonly event: string
+  private readonly properties: string[]
+  private readonly expose?: string[]
+  private readonly stream: Readable
+
+  public constructor({
+    event,
+    properties,
+    stream,
+    expose
+  }: {
+    event: string
+    properties: string[]
+    stream: Readable
+    expose?: string[]
+  }) {
+    super()
+
+    this.event = event
+    this.properties = properties
+    this.expose = expose
+    this.stream = stream
+  }
+
+  public async receive(message: Message<Record<string, string>>): Promise<void> {
+    // the push continues the trace from the producer
+    const telemetry = message.telemetry === undefined ? null : decode(message.telemetry)
+
+    if (telemetry === null) this.process(message, telemetry)
+    else run(telemetry, () => this.process(message, telemetry))
+  }
+
+  private process(
+    message: Message<Record<string, string>>,
+    telemetry: SpanContext | null
+  ): void {
+    const data = this.fit(message.payload)
+
+    for (const property of this.properties) {
+      const key = message.payload[property]
+
+      if (key === undefined) {
+        console.debug('Event does not contain key property', {
+          property,
+          event: this.event
+        })
+
+        continue
+      }
+
+      if (Array.isArray(key))
+        // eslint-disable-next-line max-depth
+        for (const k of key as string[]) this.push(k, data, telemetry)
+      else this.push(key, data, telemetry)
+    }
+  }
+
+  private fit(payload: Record<string, string>): Record<string, string> {
+    if (this.expose === undefined) return payload
+
+    const entries = Object.entries(payload).filter(([key]) => this.expose!.includes(key))
+
+    return Object.fromEntries(entries)
+  }
+
+  private push(
+    key: string | null,
+    data: Record<string, string>,
+    telemetry: SpanContext | null
+  ): void {
+    if (key === null || typeof key === 'undefined') {
+      console.debug('Key is null or undefined, skipping', { key, event: this.event })
+
+      return
+    }
+
+    console.debug('Pushing event to stream', { key, event: this.event, data })
+
+    this.stream.push({ key, event: this.event, data, telemetry } satisfies Push)
+  }
+}
+
+export interface Push {
+  key: string
+  event: string
+  data: Record<string, string>
+  telemetry: SpanContext | null
+}