@toa.io/extensions.realtime 1.0.0-alpha.28 → 1.0.0-alpha.283

package/cucumber.mjs

@@ -0,0 +1,5 @@
+export default {
+  paths: ['features/**/*.feature'],
+  import: ['./features/**/*.ts'],
+  failFast: true
+}

package/features/static.feature

@@ -1,5 +1,12 @@
 Feature: Static routes
 
+  Scenario: Debug
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+
   Scenario: Routing an event
     Given the `messages` component is running with routes:
       """yaml
@@ -30,3 +37,116 @@ Feature: Static routes
         recipient: 004e02a959c04cecaf111827f91caa36
         text: Hello!
       """
+
+  Scenario: Exposing event properties
+    Given the `messages` component is running with routes:
+      """yaml
+      created:
+        key: [sender, recipient]
+        expose: [text, sender]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And the stream `96db5a47a8244eb3b21820781b7d596e` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        text: Hello!
+      """
+    And an event is received from the stream `96db5a47a8244eb3b21820781b7d596e`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        text: Hello!
+      """
+
+  Scenario: Routing an event using array
+    Given the `messages` component is running with routes:
+      """yaml
+      created: watchers
+      """
+    And the stream `51c15a7290ce47e0af8ec41d60dccb32` is consumed
+    And the stream `bb27366509a64178a39313aac42435ae` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        watchers:
+          - 51c15a7290ce47e0af8ec41d60dccb32
+          - bb27366509a64178a39313aac42435ae
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `51c15a7290ce47e0af8ec41d60dccb32`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `bb27366509a64178a39313aac42435ae`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+
+  Scenario: Continuation token
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """
+
+    When the consumer `004e02a959c04cecaf111827f91caa36` is disconnected
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And the consumer `004e02a959c04cecaf111827f91caa36` is reconnected
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 004e02a959c04cecaf111827f91caa36
+        recipient: 96db5a47a8244eb3b21820781b7d596e
+        text: Hi!
+      """
+    And an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: token
+      """

package/features/steps/Components.ts

@@ -1,11 +1,15 @@
 import { resolve } from 'node:path'
 import { readdirSync } from 'node:fs'
-import { after, before, binding, given, when } from 'cucumber-tsflow'
-import { parse } from '@toa.io/yaml'
+import tsflow from 'cucumber-tsflow'
+
+import { load as parse } from 'js-yaml'
 import * as stage from '@toa.io/userland/stage'
 import { type Component, type Request } from '@toa.io/core'
 import { timeout } from '@toa.io/generic'
-import { Realtime } from './Realtime'
+import { parse as parseRoutes, type Declaration } from '../../source/extension.js'
+import { Realtime } from './Realtime.js'
+
+const { after, before, binding, given, when } = tsflow
 
 @binding([Realtime])
 export class Components {
@@ -18,20 +22,20 @@ export class Components {
 
   @given('the `{word}` component is running with routes:')
   public async run (component: string, yaml: string): Promise<void> {
-    const routes = parse<Manifest>(yaml)
+    const declaration = parse(yaml) as Declaration
     const [name, namespace = 'default'] = component.split('.').reverse()
+    const routes = parseRoutes(declaration)
 
-    for (const [event, value] of Object.entries(routes)) {
-      const label = `${namespace}.${name}.${event}`
-      const properties = Array.isArray(value) ? value : [value]
+    for (const route of routes) {
+      const label = `${namespace}.${name}.${route.event}`
 
-      this.realtime.declare(label, properties)
+      this.realtime.declare(label, route.properties, route.expose)
     }
   }
 
   @when('the `{word}` is called with:')
   public async call (endpoint: string, yaml: string): Promise<void> {
-    const request = parse<Request>(yaml)
+    const request = parse(yaml) as Request
     const [operation, component, namespace = 'default'] = endpoint.split('.').reverse()
     const id = `${namespace}.${component}`
 
@@ -64,8 +68,4 @@ function componentPaths (): string[] {
     .map((entry) => resolve(ROOT, entry.name))
 }
 
-const ROOT = resolve(__dirname, 'components')
-
-interface Manifest {
-  realtime: Map<string, string | string[]>
-}
+const ROOT = resolve(import.meta.dirname, 'components')

package/features/steps/Realtime.ts

@@ -1,43 +1,39 @@
 import * as boot from '@toa.io/boot'
-import { encode } from '@toa.io/generic'
 import { type Connector } from '@toa.io/core'
-import { after, binding } from 'cucumber-tsflow'
-import { Factory } from '../../source/Factory'
+import tsflow from 'cucumber-tsflow'
+
+import { Factory } from '../../source/index.js'
+import type { Route } from '../../source/extension.js'
+
+const { after, binding } = tsflow
 
 @binding()
 export class Realtime {
   private readonly routes: Route[] = []
-  private readonly service: Connector
+  // the factory loads its binding, so the service is made when it is first served
+  private service: Connector | undefined
   private connected = false
 
-  public constructor () {
-    this.service = new Factory(boot).service()
-  }
-
   @after()
-  private async shutdown (): Promise<void> {
+  public async shutdown (): Promise<void> {
     this.connected = false
 
-    await this.service.disconnect()
+    await this.service?.disconnect()
   }
 
-  public declare (event: string, properties: string[]): void {
-    this.routes.push({ event, properties })
+  public declare (event: string, properties: string[], expose?: string[]): void {
+    this.routes.push({ event, properties, expose })
   }
 
   public async serve (): Promise<void> {
     if (this.connected)
       return
 
-    process.env.TOA_REALTIME = encode(this.routes)
+    process.env.TOA_REALTIME = JSON.stringify(this.routes)
 
     this.connected = true
+    this.service = await new Factory(boot.host()).service()
 
     await this.service.connect()
   }
 }
-
-interface Route {
-  event: string
-  properties: string[]
-}

package/features/steps/Streams.ts

@@ -1,36 +1,42 @@
-import { type Readable } from 'node:stream'
-import { after, binding, given, then } from 'cucumber-tsflow'
-import { match } from '@toa.io/generic'
+import * as assert from 'node:assert'
+import { setTimeout } from 'node:timers/promises'
+import tsflow from 'cucumber-tsflow'
 
-import { type Component } from '@toa.io/core'
-import { parse } from '@toa.io/yaml'
+import { match } from '@toa.io/generic'
+import { load as parse } from 'js-yaml'
 import * as stage from '@toa.io/userland/stage'
-import { Realtime } from './Realtime'
+import { Realtime } from './Realtime.js'
+import type { Readable } from 'node:stream'
+import type { Component } from '@toa.io/core'
+
+const { after, binding, given, then } = tsflow
 
 @binding([Realtime])
 export class Streams {
   private readonly realtime: Realtime
   private remote: Component | null = null
   private streams: Record<string, Readable> = {}
-  private events: Record<string, object[]> = {}
+  private events: Record<string, Event[]> = {}
 
   public constructor (realtime: Realtime) {
     this.realtime = realtime
   }
 
-  @given('the stream `{word}` is consumed')
+  @given('the stream `{word}` is consumed', { timeout: 30_000 })
   public async consume (key: string): Promise<void> {
     await this.realtime.serve()
 
     this.remote ??= await stage.remote('realtime.streams')
     this.events[key] = []
-    this.streams[key] = await this.remote.invoke('create', { input: { key } })
-    this.streams[key].on('data', (data: object) => this.events[key].push(data))
+
+    await this.createStream(key)
   }
 
   @then('an event is received from the stream `{word}`:')
-  public received (key: string, yaml: string): void {
-    const expected = parse<object>(yaml)
+  public async received (key: string, yaml: string): Promise<void> {
+    await setTimeout(100)
+
+    const expected = parse(yaml) as object
 
     for (const event of this.events[key])
       if (match(event, expected))
@@ -39,12 +45,44 @@ export class Streams {
     throw new Error('No matching event received')
   }
 
+  @then('the consumer `{word}` is disconnected')
+  public disconnected (key: string): void {
+    this.streams[key]?.destroy()
+    delete this.streams[key]
+  }
+
+  @then('the consumer `{word}` is reconnected')
+  public async reconnected (key: string): Promise<void> {
+    const last = this.events[key].findLast((event) => event.event === 'token')
+
+    assert.ok(last, `No last event found for stream ${key}`)
+
+    await this.createStream(key, last.data as string)
+  }
+
   @after()
-  private shutdown (): void {
+  public async shutdown (): Promise<void> {
     for (const stream of Object.values(this.streams))
       stream.destroy()
 
     this.streams = {}
     this.events = {}
+
+    await setTimeout(100)
   }
+
+  private async createStream (key: string, token?: string): Promise<void> {
+    this.streams[key] = await this.remote!.invoke('create', { input: { key, token } })
+    this.streams[key].on('data', (event: Event) => {
+      console.log('[TEST] Received event', event)
+      this.events[key]?.push(event)
+    })
+  }
+}
+
+interface Event {
+  key: string
+  token: string
+  event: string
+  data?: unknown
 }

package/features/steps/components/messages/manifest.toa.yaml

@@ -2,15 +2,46 @@ name: messages
 
 entity:
   schema:
-    sender*: string(32)
-    recipient*: string(32)
-    text*: string
+    properties:
+      watchers:
+        type: array
+        items:
+          type: string
+          maxLength: 32
+      sender:
+        type: string
+        maxLength: 32
+      recipient:
+        type: string
+        maxLength: 32
+      text:
+        type: string
+    type: object
+    required:
+      - sender
+      - recipient
+      - text
 
 operations:
   create:
     query: false
     forward: transit
     input:
-      sender*: .
-      recipient*: .
-      text*: .
+      properties:
+        watchers:
+          type: string
+          default: .
+        sender:
+          type: string
+          default: .
+        recipient:
+          type: string
+          default: .
+        text:
+          type: string
+          default: .
+      type: object
+      required:
+        - sender
+        - recipient
+        - text

package/features/steps/config.ts

@@ -1 +1,13 @@
+import { setDefaultTimeout } from '@cucumber/cucumber'
+
 process.env.TOA_DEV = '1'
+
+// a reply is checked against what the operation declares, so the suite runs Toa under the
+// contract it asks applications to keep
+process.env.TOA_ENV ??= 'local'
+
+// the streams component boots inside the service, and without a variable it would wait
+// for the values service, which these features do not run
+process.env.TOA_CONFIGURATION_REALTIME_STREAMS ??= '{}'
+
+setDefaultTimeout(60 * 1000)

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/extensions.realtime",
-  "version": "1.0.0-alpha.28",
+  "version": "1.0.0-alpha.283",
+  "type": "module",
   "description": "Toa Realtime",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -16,13 +17,13 @@
   "publishConfig": {
     "access": "public"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node"
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.283",
+    "openspan": "1.0.0-alpha.277"
   },
   "scripts": {
-    "transpile": "npx tsc && npx tsc -p ./components/streams",
-    "features": "npx cucumber-js"
+    "transpile": "npx tsc && npx tsc -p ./components/realtime.streams",
+    "features": "TSX_TSCONFIG_PATH=features/steps/tsconfig.json NODE_OPTIONS=--import=tsx cucumber-js"
   },
-  "gitHead": "89c3b7d1314e357c9e33df8d95f8d46954cd3bcf"
+  "gitHead": "2f52b023bed0ccb2ecdb2b82fe59fbe38bf282e1"
 }

package/readme.md

@@ -10,17 +10,20 @@
 </a>
 
 Realtime extension combines application events into streams according to defined routes.
-Clients may consume these streams [via Exposition](#exposition).
+Clients may consume these streams [via Exposition](/extensions/exposition).
+
+If stream is idle for 16 seconds, a `heartbeat` message is sent.
 
 ## Static routes
 
 Static route specifies an event that should be combined into a stream using specified property of
-event's payload as a stream key.
+event's payload as a stream key or an array of stream keys.
 
 Static routes may be defined in Component manifest or the Context annotation.
 
 ```yaml
 # manifest.toa.yaml
+
 name: users
 
 realtime:
@@ -29,13 +32,24 @@ realtime:
 
 ```yaml
 # context.toa.yaml
+
 realtime:
   users.updated: id
-  orders.created: custromer_id
+  orders.created: customer_id
 ```
 
 In case of conflict, the Context annotation takes precedence.
 
+Multiple stream keys may be defined for a single event.
+
+```yaml
+# manifest.toa.yaml
+name: messages
+
+realtime:
+  updated: [sender_id, recipient_id]
+```
+
 ### Static route examples
 
 Given two rules: `users.updated: id` and `orders.created: customer_id`,
@@ -87,13 +101,25 @@ Dynamic routes are managed by the `realtime.routes` component, running in the Re
 
 ## Exposition
 
-Streams are exposed by the [`realtime.streams`](components/streams) component, running in the
+Streams are exposed by the [`realtime.streams`](components/realtime.streams) component, running in the
 Realtime extension, and are
 accessible via the `/realtime/streams/:key/` resource with
 the [`auth:id: key`](/extensions/exposition/documentation/access.md#id) authorization rule.
 
-Refer to the [Exposition extension](/extensions/exposition) for more
-details:
+Refer to the [Exposition extension](/extensions/exposition) for more details:
 
-- [Streams](/extensions/exposition/documentation/protocol.md#streams)
+- [Multipart responses](/extensions/exposition/documentation/protocol.md#multipart-types)
 - [Access authorization](/extensions/exposition/documentation/access.md)
+
+## Resources management
+
+Resource requests and limits can be specified by `resources` annotation:
+
+```yaml
+# context.toa.yaml
+
+realtime:
+  resources:
+    cpu: [100m, 500m]
+    memory: [100Mi, 200Mi]
+```

package/source/Composition.ts

@@ -1,33 +1,27 @@
 import { readdirSync, type Dirent } from 'node:fs'
 import { resolve } from 'node:path'
 import { Connector } from '@toa.io/core'
-import { type Bootloader } from './Factory'
+import { type Host } from './Factory.js'
 
 export class Composition extends Connector {
-  private readonly boot: Bootloader
+  private readonly host: Host
 
-  public constructor (boot: Bootloader) {
+  public constructor (host: Host) {
     super()
-    this.boot = boot
+    this.host = host
   }
 
   protected override async open (): Promise<void> {
     const paths = find()
-    const composition = await this.boot.composition(paths)
+    const composition = await this.host.composition(paths)
 
     await composition.connect()
 
     this.depends(composition)
-
-    console.info('Composition complete.')
-  }
-
-  protected override dispose (): void {
-    console.info('Composition shutdown complete.')
   }
 }
 
-function find (): string[] {
+export function find (): string[] {
   return entries().map((entry) => resolve(ROOT, entry.name))
 }
 
@@ -37,4 +31,21 @@ function entries (): Dirent[] {
   return entries.filter((entry) => entry.isDirectory())
 }
 
-const ROOT = resolve(__dirname, '../components/')
+export function components (): Components {
+  const labels: string[] = []
+  const paths: string[] = []
+
+  for (const entry of entries()) {
+    labels.push(entry.name.replace('.', '-'))
+    paths.push(resolve(ROOT, entry.name))
+  }
+
+  return { labels, paths }
+}
+
+interface Components {
+  labels: string[]
+  paths: string[]
+}
+
+const ROOT = resolve(import.meta.dirname, '../components/')

package/source/Factory.ts

@@ -1,20 +1,19 @@
 import { type Component, type extensions, Locator } from '@toa.io/core'
-import { Realtime } from './Realtime'
-import { Composition } from './Composition'
-import { Routes } from './Routes'
+import { Realtime } from './Realtime.js'
+import { Composition } from './Composition.js'
+import { Routes } from './Routes.js'
 
 export class Factory implements extensions.Factory {
-  private readonly boot: Bootloader
+  private readonly host: Host
 
-  public constructor (boot: Bootloader) {
-    this.boot = boot
+  public constructor (host: Host) {
+    this.host = host
   }
 
   public service (): Realtime {
-    const discovery = this.discovery()
-    const routes = new Routes(this.boot)
-    const composition = new Composition(this.boot)
-    const realtime = new Realtime(routes, discovery)
+    const routes = new Routes(this.host)
+    const composition = new Composition(this.host)
+    const realtime = new Realtime(routes, async () => await this.discovery())
 
     realtime.depends(routes)
     realtime.depends(composition)
@@ -25,9 +24,8 @@ export class Factory implements extensions.Factory {
   private async discovery (): Promise<Component> {
     const locator = new Locator('streams', 'realtime')
 
-    return await this.boot.remote(locator)
+    return await this.host.remote(locator, { service: 'realtime' })
   }
 }
 
-// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-export type Bootloader = typeof import('@toa.io/boot')
+export type Host = extensions.Host