@toa.io/extensions.realtime 1.0.0-alpha.21 → 1.0.0-alpha.212

package/components/streams/source/lib/Stash.ts

@@ -0,0 +1,105 @@
+import type { Redis } from 'ioredis'
+
+export class Stash {
+  private readonly stash: Redis
+  private readonly configuration: Configuration
+
+  public constructor (stash: any, configuration: Configuration) {
+    this.stash = stash
+    this.configuration = configuration
+  }
+
+  public async connect (key: string): Promise<string | Error> {
+    return await this.xadd(key, 'connect')
+  }
+
+  public async push (key: string, event: string, data: unknown): Promise<string | Error> {
+    return await this.xadd(key, event, data)
+  }
+
+  public async pop (key: string, token: string): Promise<[string, Event[]] | null | Error> {
+    const stamp = this.decode(token)
+
+    const results = await this.stash
+      .xread('STREAMS', key, stamp)
+      .catch((error: Error) => error)
+
+    if (results === null)
+      return ERR_NO_RESULTS
+
+    if (results instanceof Error)
+      return results
+
+    if (results.length === 0)
+      return null
+
+    const [, items] = results[0]
+    const events: Event[] = []
+
+    let lastStamp: string | null = null
+
+    for (const item of items) {
+      const [, event, , json] = item[1]
+
+      lastStamp = item[0]
+
+      events.push({ event, data: JSON.parse(json) })
+    }
+
+    if (lastStamp === null)
+      return null
+
+    return [this.encode(lastStamp), events]
+  }
+
+  private async xadd (key: string, event: string, data?: unknown): Promise<string | Error> {
+    const args = ['MAXLEN', '~', this.configuration.maxlen, '*', 'type', event]
+
+    if (data !== undefined)
+      args.push('data', JSON.stringify(data))
+
+    const results = await this.stash
+      .multi()
+      .xadd(key, ...args)
+      .expire(key, this.configuration.expire)
+      .exec()
+      .catch((error: Error) => error)
+
+    if (results === null)
+      return ERR_NO_RESULTS
+
+    if (results instanceof Error)
+      return results
+
+    const [[error, stamp]] = results
+
+    if (error !== null)
+      return error
+
+    return this.encode(stamp as string)
+  }
+
+  private encode (token: string): string {
+    return Buffer.from(token).toString('base64url')
+  }
+
+  private decode (token: string): string {
+    return Buffer.from(token, 'base64url').toString()
+  }
+}
+
+interface Configuration {
+  maxlen: number
+  expire: number
+}
+
+interface Event {
+  event: string
+  data: unknown
+}
+
+class NoResultsError extends Error {
+  public readonly code = 'NO_RESULTS'
+}
+
+const ERR_NO_RESULTS = new NoResultsError()

package/components/streams/source/lib/Stream.ts

@@ -0,0 +1,40 @@
+import { EventEmitter, Readable } from 'node:stream'
+
+export class Stream extends Readable {
+  public events = new EventEmitter()
+
+  private interval: NodeJS.Timeout | null = null
+
+  public constructor () {
+    super(objectMode)
+  }
+
+  // has to be here
+  public override _read (): void {
+    if (this.interval === null)
+      this.interval = setInterval(() => this.heartbeat(), HEARTBEAT_INTERVAL)
+  }
+
+  public override _destroy (error: Error | null, callback: (error?: (Error | null)) => void): void {
+    if (this.interval !== null)
+      clearInterval(this.interval)
+
+    this.events.emit('destroy')
+
+    super._destroy(error, callback)
+  }
+
+  public heartbeat (stream: Readable = this): boolean {
+    const resume = stream.push('heartbeat ' + Date.now())
+
+    if (!resume && this.interval !== null) {
+      clearInterval(this.interval)
+      this.interval = null
+    }
+
+    return resume
+  }
+}
+
+const HEARTBEAT_INTERVAL = 16_000 // why?
+const objectMode = { objectMode: true }

package/components/streams/source/lib/index.ts

@@ -0,0 +1,2 @@
+export { Stash } from './Stash'
+export { Stream } from './Stream'

package/components/streams/source/lib/types.ts

@@ -0,0 +1,24 @@
+import { type Stream } from './Stream'
+import { type Stash } from './Stash'
+
+export interface Context {
+  stash: any
+  state: {
+    streams: Map<string, Stream>
+    stash: Stash
+  }
+  logs: {
+    info: (m: string, att?: object) => void
+    error: (m: string, att?: object) => void
+  }
+  configuration: {
+    maxlen: number
+    expire: number
+  }
+}
+
+export interface PushInput {
+  key: string
+  event: string
+  data: unknown
+}

package/components/streams/source/push.ts

@@ -1,5 +1,12 @@
-import { type Context, type PushInput } from './types'
+import { type Context, type PushInput } from './lib/types'
 
 export async function effect ({ key, event, data }: PushInput, context: Context): Promise<void> {
   context.state.streams.get(key)?.push({ event, data })
+
+  void context.state.stash.push(key, event, data).then((token) => {
+    if (token instanceof Error)
+      context.logs.error('Failed to push to stash', { key, error: token })
+    else
+      context.state.streams.get(key)?.push({ event: 'token', data: token })
+  })
 }

package/features/static.feature

@@ -1,5 +1,12 @@
 Feature: Static routes
 
+  Scenario: Debug
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+
   Scenario: Routing an event
     Given the `messages` component is running with routes:
       """yaml
@@ -30,3 +37,85 @@ Feature: Static routes
         recipient: 004e02a959c04cecaf111827f91caa36
         text: Hello!
       """
+
+  Scenario: Routing an event using array
+    Given the `messages` component is running with routes:
+      """yaml
+      created: watchers
+      """
+    And the stream `51c15a7290ce47e0af8ec41d60dccb32` is consumed
+    And the stream `bb27366509a64178a39313aac42435ae` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        watchers:
+          - 51c15a7290ce47e0af8ec41d60dccb32
+          - bb27366509a64178a39313aac42435ae
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `51c15a7290ce47e0af8ec41d60dccb32`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `bb27366509a64178a39313aac42435ae`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+
+  Scenario: Continuation token
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """
+
+    When the consumer `004e02a959c04cecaf111827f91caa36` is disconnected
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And the consumer `004e02a959c04cecaf111827f91caa36` is reconnected
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """

package/features/steps/Realtime.ts

@@ -2,7 +2,7 @@ import * as boot from '@toa.io/boot'
 import { encode } from '@toa.io/generic'
 import { type Connector } from '@toa.io/core'
 import { after, binding } from 'cucumber-tsflow'
-import { Factory } from '../../source/Factory'
+import { Factory } from '../../source'
 
 @binding()
 export class Realtime {
@@ -15,7 +15,7 @@ export class Realtime {
   }
 
   @after()
-  private async shutdown (): Promise<void> {
+  public async shutdown (): Promise<void> {
     this.connected = false
 
     await this.service.disconnect()

package/features/steps/Streams.ts

@@ -1,35 +1,38 @@
-import { type Readable } from 'node:stream'
+import * as assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
 import { after, binding, given, then } from 'cucumber-tsflow'
 import { match } from '@toa.io/generic'
-
-import { type Component } from '@toa.io/core'
 import { parse } from '@toa.io/yaml'
 import * as stage from '@toa.io/userland/stage'
 import { Realtime } from './Realtime'
+import type { Readable } from 'node:stream'
+import type { Component } from '@toa.io/core'
 
 @binding([Realtime])
 export class Streams {
   private readonly realtime: Realtime
   private remote: Component | null = null
   private streams: Record<string, Readable> = {}
-  private events: Record<string, object[]> = {}
+  private events: Record<string, Event[]> = {}
 
   public constructor (realtime: Realtime) {
     this.realtime = realtime
   }
 
-  @given('the stream `{word}` is consumed')
+  @given('the stream `{word}` is consumed', { timeout: 30_000 })
   public async consume (key: string): Promise<void> {
     await this.realtime.serve()
 
     this.remote ??= await stage.remote('realtime.streams')
     this.events[key] = []
-    this.streams[key] = await this.remote.invoke('create', { input: { key } })
-    this.streams[key].on('data', (data: object) => this.events[key].push(data))
+
+    await this.createStream(key)
   }
 
   @then('an event is received from the stream `{word}`:')
-  public received (key: string, yaml: string): void {
+  public async received (key: string, yaml: string): Promise<void> {
+    await setTimeout(100)
+
     const expected = parse<object>(yaml)
 
     for (const event of this.events[key])
@@ -39,12 +42,44 @@ export class Streams {
     throw new Error('No matching event received')
   }
 
+  @then('the consumer `{word}` is disconnected')
+  public disconnected (key: string): void {
+    this.streams[key]?.destroy()
+    delete this.streams[key]
+  }
+
+  @then('the consumer `{word}` is reconnected')
+  public async reconnected (key: string): Promise<void> {
+    const last = this.events[key].findLast((event) => event.event === 'token')
+
+    assert.ok(last, `No last event found for stream ${key}`)
+
+    await this.createStream(key, last.data as string)
+  }
+
   @after()
-  private shutdown (): void {
+  public async shutdown (): Promise<void> {
     for (const stream of Object.values(this.streams))
       stream.destroy()
 
     this.streams = {}
     this.events = {}
+
+    await setTimeout(100)
   }
+
+  private async createStream (key: string, token?: string): Promise<void> {
+    this.streams[key] = await this.remote!.invoke('create', { input: { key, token } })
+    this.streams[key].on('data', (event: Event) => {
+      console.log('[TEST] Received event', event)
+      this.events[key]?.push(event)
+    })
+  }
+}
+
+interface Event {
+  key: string
+  token: string
+  event: string
+  data?: unknown
 }

package/features/steps/components/messages/manifest.toa.yaml

@@ -5,6 +5,7 @@ entity:
     sender*: string(32)
     recipient*: string(32)
     text*: string
+    watchers: [string(32)]
 
 operations:
   create:
@@ -14,3 +15,4 @@ operations:
       sender*: .
       recipient*: .
       text*: .
+      watchers: .

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.realtime",
-  "version": "1.0.0-alpha.21",
+  "version": "1.0.0-alpha.212",
   "description": "Toa Realtime",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -16,6 +16,10 @@
   "publishConfig": {
     "access": "public"
   },
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.212",
+    "openspan": "1.0.0-alpha.173"
+  },
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node"
@@ -24,5 +28,5 @@
     "transpile": "npx tsc && npx tsc -p ./components/streams",
     "features": "npx cucumber-js"
   },
-  "gitHead": "da9f4c278f6ab02a28f65c6e25c713c013cbfce9"
+  "gitHead": "d8aefb3b37df15be74bb63e1447f76c27e88d9be"
 }

package/readme.md

@@ -10,17 +10,20 @@
 </a>
 
 Realtime extension combines application events into streams according to defined routes.
-Clients may consume these streams [via Exposition](#exposition).
+Clients may consume these streams [via Exposition](/extensions/exposition).
+
+If stream is idle for 16 seconds, a `heartbeat` message is sent.
 
 ## Static routes
 
 Static route specifies an event that should be combined into a stream using specified property of
-event's payload as a stream key.
+event's payload as a stream key or an array of stream keys.
 
 Static routes may be defined in Component manifest or the Context annotation.
 
 ```yaml
 # manifest.toa.yaml
+
 name: users
 
 realtime:
@@ -29,13 +32,24 @@ realtime:
 
 ```yaml
 # context.toa.yaml
+
 realtime:
   users.updated: id
-  orders.created: custromer_id
+  orders.created: customer_id
 ```
 
 In case of conflict, the Context annotation takes precedence.
 
+Multiple stream keys may be defined for a single event.
+
+```yaml
+# manifest.toa.yaml
+name: messages
+
+realtime:
+  updated: [sender_id, recipient_id]
+```
+
 ### Static route examples
 
 Given two rules: `users.updated: id` and `orders.created: customer_id`,
@@ -92,8 +106,20 @@ Realtime extension, and are
 accessible via the `/realtime/streams/:key/` resource with
 the [`auth:id: key`](/extensions/exposition/documentation/access.md#id) authorization rule.
 
-Refer to the [Exposition extension](/extensions/exposition) for more
-details:
+Refer to the [Exposition extension](/extensions/exposition) for more details:
 
-- [Streams](/extensions/exposition/documentation/protocol.md#streams)
+- [Multipart responses](/extensions/exposition/documentation/protocol.md#multipart-types)
 - [Access authorization](/extensions/exposition/documentation/access.md)
+
+## Resources management
+
+Resource requests and limits can be specified by `resources` annotation:
+
+```yaml
+# context.toa.yaml
+
+realtime:
+  resources:
+    cpu: [100m, 500m]
+    memory: [100Mi, 200Mi]
+```

package/source/Composition.ts

@@ -18,16 +18,10 @@ export class Composition extends Connector {
     await composition.connect()
 
     this.depends(composition)
-
-    console.info('Composition complete.')
-  }
-
-  protected override dispose (): void {
-    console.info('Composition shutdown complete.')
   }
 }
 
-function find (): string[] {
+export function find (): string[] {
   return entries().map((entry) => resolve(ROOT, entry.name))
 }
 

package/source/Realtime.ts

@@ -1,3 +1,4 @@
+import { console } from 'openspan'
 import { type Component, Connector } from '@toa.io/core'
 import { type Routes } from './Routes'
 
@@ -19,15 +20,16 @@ export class Realtime extends Connector {
 
     await this.streams.connect()
 
-    console.log('Realtime has started.')
+    console.info('Realtime service started')
   }
 
   protected override dispose (): void {
-    console.log('Realtime shutdown complete.')
+    console.info('Realtime service shutdown complete')
   }
 
   private push (event: Event): void {
     void this.streams?.invoke('push', { input: event })
+      .catch((error) => console.error('Realtime push failed', error))
   }
 }
 

package/source/Receiver.ts

@@ -0,0 +1,49 @@
+import { console } from 'openspan'
+import { Connector, type Message } from '@toa.io/core'
+import type { Readable } from 'node:stream'
+
+export class Receiver extends Connector {
+  private readonly event: string
+  private readonly properties: string[]
+  private readonly stream: Readable
+
+  public constructor (event: string, properties: string[], stream: Readable) {
+    super()
+
+    this.event = event
+    this.properties = properties
+    this.stream = stream
+  }
+
+  public receive (message: Message<Record<string, string>>): void {
+    for (const property of this.properties) {
+      const key = message.payload[property]
+
+      if (key === undefined) {
+        console.debug('Event does not contain key property',
+          { property, event: this.event })
+
+        continue
+      }
+
+      if (Array.isArray(key))
+        // eslint-disable-next-line max-depth
+        for (const k of key as string[])
+          this.push(k, message.payload)
+      else
+        this.push(key, message.payload)
+    }
+  }
+
+  private push (key: string | null, data: Record<string, string>): void {
+    if (key === null || typeof key === 'undefined') {
+      console.debug('Key is null or undefined, skipping', { key, event: this.event })
+
+      return
+    }
+
+    console.debug('Pushing event to stream', { key, event: this.event, data })
+
+    this.stream.push({ key, event: this.event, data })
+  }
+}

package/source/Routes.ts

@@ -1,7 +1,9 @@
 import { Readable } from 'node:stream'
-import { Connector, type Message } from '@toa.io/core'
+import { console } from 'openspan'
+import { Connector } from '@toa.io/core'
 import { decode } from '@toa.io/generic'
-import { type Bootloader } from './Factory'
+import { Receiver } from './Receiver'
+import type { Bootloader } from './Factory'
 
 export class Routes extends Connector {
   public events = new Events()
@@ -26,7 +28,7 @@ export class Routes extends Connector {
     const creating = []
 
     for (const { event, properties } of routes) {
-      const consumer = this.boot.receive(event, this.getReceiver(event, properties))
+      const consumer = this.boot.receive(event, new Receiver(event, properties, this.events))
 
       creating.push(consumer)
     }
@@ -39,29 +41,11 @@ export class Routes extends Connector {
     await Promise.all(connecting)
     this.depends(consumers)
 
-    console.log(`Event sources (${creating.length}) connected.`)
+    console.info('Event sources connected', { count: creating.length })
   }
 
   public override async close (): Promise<void> {
-    console.log('Event sources disconnected.')
-  }
-
-  private getReceiver (event: string, properties: string[]): Receiver {
-    return {
-      receive: (message: Message<Record<string, string>>) => {
-        for (const property of properties) {
-          const key = message.payload[property]
-
-          if (key === undefined) {
-            console.error(`Event '${event}' does not contain the '${property}' property.`)
-
-            return
-          }
-
-          this.events.push({ key, event, data: message.payload })
-        }
-      }
-    }
+    console.info('Event sources disconnected')
   }
 }
 
@@ -78,7 +62,3 @@ export interface Route {
   event: string
   properties: string[]
 }
-
-interface Receiver {
-  receive: (message: Message<Record<string, string>>) => void
-}