@toa.io/extensions.realtime 1.0.0-alpha.26 → 1.0.0-alpha.263

package/features/static.feature

@@ -1,5 +1,12 @@
 Feature: Static routes
 
+  Scenario: Debug
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+
   Scenario: Routing an event
     Given the `messages` component is running with routes:
       """yaml
@@ -30,3 +37,116 @@ Feature: Static routes
         recipient: 004e02a959c04cecaf111827f91caa36
         text: Hello!
       """
+
+  Scenario: Exposing event properties
+    Given the `messages` component is running with routes:
+      """yaml
+      created:
+        key: [sender, recipient]
+        expose: [text, sender]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And the stream `96db5a47a8244eb3b21820781b7d596e` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        text: Hello!
+      """
+    And an event is received from the stream `96db5a47a8244eb3b21820781b7d596e`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        text: Hello!
+      """
+
+  Scenario: Routing an event using array
+    Given the `messages` component is running with routes:
+      """yaml
+      created: watchers
+      """
+    And the stream `51c15a7290ce47e0af8ec41d60dccb32` is consumed
+    And the stream `bb27366509a64178a39313aac42435ae` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        watchers:
+          - 51c15a7290ce47e0af8ec41d60dccb32
+          - bb27366509a64178a39313aac42435ae
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `51c15a7290ce47e0af8ec41d60dccb32`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `bb27366509a64178a39313aac42435ae`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+
+  Scenario: Continuation token
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """
+
+    When the consumer `004e02a959c04cecaf111827f91caa36` is disconnected
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And the consumer `004e02a959c04cecaf111827f91caa36` is reconnected
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """

package/features/steps/Components.ts

@@ -5,6 +5,7 @@ import { parse } from '@toa.io/yaml'
 import * as stage from '@toa.io/userland/stage'
 import { type Component, type Request } from '@toa.io/core'
 import { timeout } from '@toa.io/generic'
+import { parse as parseRoutes, type Declaration } from '../../source/extension'
 import { Realtime } from './Realtime'
 
 @binding([Realtime])
@@ -18,14 +19,14 @@ export class Components {
 
   @given('the `{word}` component is running with routes:')
   public async run (component: string, yaml: string): Promise<void> {
-    const routes = parse<Manifest>(yaml)
+    const declaration = parse<Declaration>(yaml)
     const [name, namespace = 'default'] = component.split('.').reverse()
+    const routes = parseRoutes(declaration)
 
-    for (const [event, value] of Object.entries(routes)) {
-      const label = `${namespace}.${name}.${event}`
-      const properties = Array.isArray(value) ? value : [value]
+    for (const route of routes) {
+      const label = `${namespace}.${name}.${route.event}`
 
-      this.realtime.declare(label, properties)
+      this.realtime.declare(label, route.properties, route.expose)
     }
   }
 
@@ -65,7 +66,3 @@ function componentPaths (): string[] {
 }
 
 const ROOT = resolve(__dirname, 'components')
-
-interface Manifest {
-  realtime: Map<string, string | string[]>
-}

package/features/steps/Realtime.ts

@@ -2,7 +2,8 @@ import * as boot from '@toa.io/boot'
 import { encode } from '@toa.io/generic'
 import { type Connector } from '@toa.io/core'
 import { after, binding } from 'cucumber-tsflow'
-import { Factory } from '../../source/Factory'
+import { Factory } from '../../source'
+import type { Route } from '../../source/extension'
 
 @binding()
 export class Realtime {
@@ -15,14 +16,14 @@ export class Realtime {
   }
 
   @after()
-  private async shutdown (): Promise<void> {
+  public async shutdown (): Promise<void> {
     this.connected = false
 
     await this.service.disconnect()
   }
 
-  public declare (event: string, properties: string[]): void {
-    this.routes.push({ event, properties })
+  public declare (event: string, properties: string[], expose?: string[]): void {
+    this.routes.push({ event, properties, expose })
   }
 
   public async serve (): Promise<void> {
@@ -36,8 +37,3 @@ export class Realtime {
     await this.service.connect()
   }
 }
-
-interface Route {
-  event: string
-  properties: string[]
-}

package/features/steps/Streams.ts

@@ -1,35 +1,38 @@
-import { type Readable } from 'node:stream'
+import * as assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
 import { after, binding, given, then } from 'cucumber-tsflow'
 import { match } from '@toa.io/generic'
-
-import { type Component } from '@toa.io/core'
 import { parse } from '@toa.io/yaml'
 import * as stage from '@toa.io/userland/stage'
 import { Realtime } from './Realtime'
+import type { Readable } from 'node:stream'
+import type { Component } from '@toa.io/core'
 
 @binding([Realtime])
 export class Streams {
   private readonly realtime: Realtime
   private remote: Component | null = null
   private streams: Record<string, Readable> = {}
-  private events: Record<string, object[]> = {}
+  private events: Record<string, Event[]> = {}
 
   public constructor (realtime: Realtime) {
     this.realtime = realtime
   }
 
-  @given('the stream `{word}` is consumed')
+  @given('the stream `{word}` is consumed', { timeout: 30_000 })
   public async consume (key: string): Promise<void> {
     await this.realtime.serve()
 
     this.remote ??= await stage.remote('realtime.streams')
     this.events[key] = []
-    this.streams[key] = await this.remote.invoke('create', { input: { key } })
-    this.streams[key].on('data', (data: object) => this.events[key].push(data))
+
+    await this.createStream(key)
   }
 
   @then('an event is received from the stream `{word}`:')
-  public received (key: string, yaml: string): void {
+  public async received (key: string, yaml: string): Promise<void> {
+    await setTimeout(100)
+
     const expected = parse<object>(yaml)
 
     for (const event of this.events[key])
@@ -39,12 +42,44 @@ export class Streams {
     throw new Error('No matching event received')
   }
 
+  @then('the consumer `{word}` is disconnected')
+  public disconnected (key: string): void {
+    this.streams[key]?.destroy()
+    delete this.streams[key]
+  }
+
+  @then('the consumer `{word}` is reconnected')
+  public async reconnected (key: string): Promise<void> {
+    const last = this.events[key].findLast((event) => event.event === 'token')
+
+    assert.ok(last, `No last event found for stream ${key}`)
+
+    await this.createStream(key, last.data as string)
+  }
+
   @after()
-  private shutdown (): void {
+  public async shutdown (): Promise<void> {
     for (const stream of Object.values(this.streams))
       stream.destroy()
 
     this.streams = {}
     this.events = {}
+
+    await setTimeout(100)
   }
+
+  private async createStream (key: string, token?: string): Promise<void> {
+    this.streams[key] = await this.remote!.invoke('create', { input: { key, token } })
+    this.streams[key].on('data', (event: Event) => {
+      console.log('[TEST] Received event', event)
+      this.events[key]?.push(event)
+    })
+  }
+}
+
+interface Event {
+  key: string
+  token: string
+  event: string
+  data?: unknown
 }

package/features/steps/components/messages/manifest.toa.yaml

@@ -5,6 +5,7 @@ entity:
     sender*: string(32)
     recipient*: string(32)
     text*: string
+    watchers: [string(32)]
 
 operations:
   create:
@@ -14,3 +15,4 @@ operations:
       sender*: .
       recipient*: .
       text*: .
+      watchers: .

package/features/steps/config.ts

@@ -1 +1,5 @@
+import { setDefaultTimeout } from '@cucumber/cucumber'
+
 process.env.TOA_DEV = '1'
+
+setDefaultTimeout(60 * 1000)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.realtime",
-  "version": "1.0.0-alpha.26",
+  "version": "1.0.0-alpha.263",
   "description": "Toa Realtime",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -16,13 +16,17 @@
   "publishConfig": {
     "access": "public"
   },
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.262",
+    "openspan": "1.0.0-alpha.254"
+  },
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node"
   },
   "scripts": {
-    "transpile": "npx tsc && npx tsc -p ./components/streams",
+    "transpile": "npx tsc && npx tsc -p ./components/realtime.streams",
     "features": "npx cucumber-js"
   },
-  "gitHead": "0ddd9260762d1fe2b8be88eb7b9725e3f7838299"
+  "gitHead": "165daa284bb7a75907a593fde359e3553b083417"
 }

package/readme.md

@@ -10,17 +10,20 @@
 </a>
 
 Realtime extension combines application events into streams according to defined routes.
-Clients may consume these streams [via Exposition](#exposition).
+Clients may consume these streams [via Exposition](/extensions/exposition).
+
+If stream is idle for 16 seconds, a `heartbeat` message is sent.
 
 ## Static routes
 
 Static route specifies an event that should be combined into a stream using specified property of
-event's payload as a stream key.
+event's payload as a stream key or an array of stream keys.
 
 Static routes may be defined in Component manifest or the Context annotation.
 
 ```yaml
 # manifest.toa.yaml
+
 name: users
 
 realtime:
@@ -29,13 +32,24 @@ realtime:
 
 ```yaml
 # context.toa.yaml
+
 realtime:
   users.updated: id
-  orders.created: custromer_id
+  orders.created: customer_id
 ```
 
 In case of conflict, the Context annotation takes precedence.
 
+Multiple stream keys may be defined for a single event.
+
+```yaml
+# manifest.toa.yaml
+name: messages
+
+realtime:
+  updated: [sender_id, recipient_id]
+```
+
 ### Static route examples
 
 Given two rules: `users.updated: id` and `orders.created: customer_id`,
@@ -87,13 +101,25 @@ Dynamic routes are managed by the `realtime.routes` component, running in the Re
 
 ## Exposition
 
-Streams are exposed by the [`realtime.streams`](components/streams) component, running in the
+Streams are exposed by the [`realtime.streams`](components/realtime.streams) component, running in the
 Realtime extension, and are
 accessible via the `/realtime/streams/:key/` resource with
 the [`auth:id: key`](/extensions/exposition/documentation/access.md#id) authorization rule.
 
-Refer to the [Exposition extension](/extensions/exposition) for more
-details:
+Refer to the [Exposition extension](/extensions/exposition) for more details:
 
-- [Streams](/extensions/exposition/documentation/protocol.md#streams)
+- [Multipart responses](/extensions/exposition/documentation/protocol.md#multipart-types)
 - [Access authorization](/extensions/exposition/documentation/access.md)
+
+## Resources management
+
+Resource requests and limits can be specified by `resources` annotation:
+
+```yaml
+# context.toa.yaml
+
+realtime:
+  resources:
+    cpu: [100m, 500m]
+    memory: [100Mi, 200Mi]
+```

package/source/Composition.ts

@@ -18,16 +18,10 @@ export class Composition extends Connector {
     await composition.connect()
 
     this.depends(composition)
-
-    console.info('Composition complete.')
-  }
-
-  protected override dispose (): void {
-    console.info('Composition shutdown complete.')
   }
 }
 
-function find (): string[] {
+export function find (): string[] {
   return entries().map((entry) => resolve(ROOT, entry.name))
 }
 
@@ -37,4 +31,21 @@ function entries (): Dirent[] {
   return entries.filter((entry) => entry.isDirectory())
 }
 
+export function components (): Components {
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
 const ROOT = resolve(__dirname, '../components/')

package/source/Factory.ts

@@ -25,7 +25,7 @@ export class Factory implements extensions.Factory {
   private async discovery (): Promise<Component> {
     const locator = new Locator('streams', 'realtime')
 
-    return await this.boot.remote(locator)
+    return await this.boot.remote(locator, { service: 'realtime' })
   }
 }
 

package/source/Realtime.ts

@@ -1,5 +1,7 @@
+import { console, run } from 'openspan'
 import { type Component, Connector } from '@toa.io/core'
 import { type Routes } from './Routes'
+import type { Push } from './Receiver'
 
 export class Realtime extends Connector {
   private readonly discovery: Promise<Component>
@@ -19,20 +21,43 @@ export class Realtime extends Connector {
 
     await this.streams.connect()
 
-    console.log('Realtime has started.')
+    console.info('Realtime service started')
   }
 
   protected override dispose (): void {
-    console.log('Realtime shutdown complete.')
+    console.info('Realtime service shutdown complete')
   }
 
-  private push (event: Event): void {
-    void this.streams?.invoke('push', { input: event })
+  private push ({ telemetry, ...event }: Push): void {
+    const processing = telemetry === null
+      ? this.deliver(event)
+      : run(telemetry, async () => await this.deliver(event))
+
+    void processing.catch((error) => console.error('Realtime push failed', error))
   }
-}
 
-interface Event {
-  key: string
-  event: string
-  data: string
+  private async deliver (event: Omit<Push, 'telemetry'>): Promise<void> {
+    /*
+     * The delivery span is created on behalf of the messaging destination
+     * (same as the core Receiver), so that service graphs display the fan-out:
+     * producer -> destination -> realtime
+     */
+    const delivery = {
+      name: `${event.event} deliver`,
+      kind: 'producer' as const,
+      service: event.event,
+      attributes: { 'messaging.destination.name': event.event }
+    }
+
+    const options = {
+      name: `${event.event} push`,
+      kind: 'consumer' as const,
+      service: 'realtime',
+      attributes: { 'messaging.destination.name': event.event }
+    }
+
+    await console.span(delivery, async () => await console.span(options, async () => {
+      await this.streams?.invoke('push', { input: event })
+    }))
+  }
 }

package/source/Receiver.ts

@@ -0,0 +1,86 @@
+import { console, decode, run } from 'openspan'
+import { Connector, type Message } from '@toa.io/core'
+import type { SpanContext } from 'openspan'
+import type { Readable } from 'node:stream'
+
+export class Receiver extends Connector {
+  private readonly event: string
+  private readonly properties: string[]
+  private readonly expose?: string[]
+  private readonly stream: Readable
+
+  public constructor ({ event, properties, stream, expose }: {
+    event: string
+    properties: string[]
+    stream: Readable
+    expose?: string[]
+  }) {
+    super()
+
+    this.event = event
+    this.properties = properties
+    this.expose = expose
+    this.stream = stream
+  }
+
+  public receive (message: Message<Record<string, string>>): void {
+    // the push continues the trace from the producer
+    const telemetry = message.telemetry === undefined ? null : decode(message.telemetry)
+
+    if (telemetry === null)
+      this.process(message, telemetry)
+    else
+      run(telemetry, () => this.process(message, telemetry))
+  }
+
+  private process (message: Message<Record<string, string>>, telemetry: SpanContext | null): void {
+    const data = this.fit(message.payload)
+
+    for (const property of this.properties) {
+      const key = message.payload[property]
+
+      if (key === undefined) {
+        console.debug('Event does not contain key property',
+          { property, event: this.event })
+
+        continue
+      }
+
+      if (Array.isArray(key))
+        // eslint-disable-next-line max-depth
+        for (const k of key as string[])
+          this.push(k, data, telemetry)
+      else
+        this.push(key, data, telemetry)
+    }
+  }
+
+  private fit (payload: Record<string, string>): Record<string, string> {
+    if (this.expose === undefined)
+      return payload
+
+    const entries = Object.entries(payload)
+      .filter(([key]) => this.expose!.includes(key))
+
+    return Object.fromEntries(entries)
+  }
+
+  private push (key: string | null, data: Record<string, string>, telemetry: SpanContext | null): void {
+    if (key === null || typeof key === 'undefined') {
+      console.debug('Key is null or undefined, skipping', { key, event: this.event })
+
+      return
+    }
+
+    console.debug('Pushing event to stream', { key, event: this.event, data })
+
+    this.stream.push({ key, event: this.event, data, telemetry } satisfies Push)
+  }
+}
+
+export interface Push {
+  key: string
+  event: string
+  data: Record<string, string>
+  telemetry: SpanContext | null
+}

package/source/Routes.ts

@@ -1,7 +1,10 @@
 import { Readable } from 'node:stream'
-import { Connector, type Message } from '@toa.io/core'
+import { console } from 'openspan'
+import { Connector } from '@toa.io/core'
 import { decode } from '@toa.io/generic'
-import { type Bootloader } from './Factory'
+import { Receiver } from './Receiver'
+import type { Route } from './extension'
+import type { Bootloader } from './Factory'
 
 export class Routes extends Connector {
   public events = new Events()
@@ -25,8 +28,8 @@ export class Routes extends Connector {
     const routes = Routes.read()
     const creating = []
 
-    for (const { event, properties } of routes) {
-      const consumer = this.boot.receive(event, this.getReceiver(event, properties))
+    for (const { event, properties, expose } of routes) {
+      const consumer = this.boot.receive(event, new Receiver({ event, properties, stream: this.events, expose }))
 
       creating.push(consumer)
     }
@@ -39,29 +42,11 @@ export class Routes extends Connector {
     await Promise.all(connecting)
     this.depends(consumers)
 
-    console.log(`Event sources (${creating.length}) connected.`)
+    console.info('Event sources connected', { count: creating.length })
   }
 
   public override async close (): Promise<void> {
-    console.log('Event sources disconnected.')
-  }
-
-  private getReceiver (event: string, properties: string[]): Receiver {
-    return {
-      receive: (message: Message<Record<string, string>>) => {
-        for (const property of properties) {
-          const key = message.payload[property]
-
-          if (key === undefined) {
-            console.error(`Event '${event}' does not contain the '${property}' property.`)
-
-            return
-          }
-
-          this.events.push({ key, event, data: message.payload })
-        }
-      }
-    }
+    console.info('Event sources disconnected')
   }
 }
 
@@ -74,11 +59,4 @@ class Events extends Readable {
   }
 }
 
-export interface Route {
-  event: string
-  properties: string[]
-}
-
-interface Receiver {
-  receive: (message: Message<Record<string, string>>) => void
-}
+export type { Route }