@toa.io/extensions.realtime 0.0.0-alpha.2

package/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020-present Artem Gurtovoi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/components/context.toa.yaml

@@ -0,0 +1,11 @@
+# used for development only
+
+name: realtime
+packages: '*'
+registry: localhost
+
+amqp: amqp://localhost
+mongodb: mongodb://localhost
+
+realtime:
+  messages.created: [sender, recipient]

package/components/streams/manifest.toa.yaml

@@ -0,0 +1,20 @@
+namespace: realtime
+name: streams
+
+operations:
+  create:
+    input:
+      key: string
+  push:
+    bindings: ~ # only for internal use
+    input:
+      key*: string
+      event*: string
+      data: ~
+
+state: ~
+
+exposition:
+  /:key:
+    auth:id: key
+    GET: create

package/components/streams/source/create.ts

@@ -0,0 +1,16 @@
+import { type Readable } from 'node:stream'
+import { type Context } from './types'
+import { addStream } from './lib/streams'
+
+export async function effect (input: Input, context: Context): Promise<Readable> {
+  const key = input.key
+
+  if (!(key in context.state))
+    addStream(key, context.state)
+
+  return context.state[key].fork()
+}
+
+interface Input {
+  key: string
+}

package/components/streams/source/lib/streams.ts

@@ -0,0 +1,45 @@
+import { PassThrough, Readable } from 'node:stream'
+
+export function addStream (key: string, map: Record<string, Stream>): void {
+  const stream = new Stream()
+
+  map[key] = stream
+
+  stream.once('close', () => delete map[key])
+}
+
+export class Stream extends Readable {
+  private forks: number = 0
+
+  public constructor () {
+    super(objectMode)
+  }
+
+  public fork (): PassThrough {
+    const through = new PassThrough(objectMode)
+
+    through.once('close', this.decrement.bind(this))
+
+    this.increment()
+    this.pipe(through)
+
+    return through
+  }
+
+  // has to be here
+  public override _read (): void {
+  }
+
+  private increment (): void {
+    this.forks++
+  }
+
+  private decrement (): void {
+    this.forks--
+
+    if (this.forks === 0)
+      this.destroy()
+  }
+}
+
+const objectMode = { objectMode: true }

package/components/streams/source/push.ts

@@ -0,0 +1,5 @@
+import { type Context, type PushInput } from './types'
+
+export async function effect ({ key, event, data }: PushInput, context: Context): Promise<void> {
+  context.state[key]?.push({ event, data })
+}

package/components/streams/source/types.ts

@@ -0,0 +1,11 @@
+import { type Stream } from './lib/streams'
+
+export interface Context {
+  state: Record<string, Stream>
+}
+
+export interface PushInput {
+  key: string
+  event: string
+  data: unknown
+}

package/components/streams/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./operations",
+  },
+  "include": [
+    "source"
+  ]
+}

package/cucumber.js

@@ -0,0 +1,9 @@
+module.exports = {
+  default: {
+    paths: ['features/**/*.feature'],
+    requireModule: ['ts-node/register'],
+    require: ['./features/**/*.ts'],
+    publishQuiet: true,
+    failFast: true
+  }
+}

package/features/static.feature

@@ -0,0 +1,32 @@
+Feature: Static routes
+
+  Scenario: Routing an event
+    Given the `messages` component is running with routes:
+      """yaml
+      created: [sender, recipient]
+      """
+    And the stream `004e02a959c04cecaf111827f91caa36` is consumed
+    And the stream `96db5a47a8244eb3b21820781b7d596e` is consumed
+    When the `messages.create` is called with:
+      """yaml
+      input:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    Then an event is received from the stream `004e02a959c04cecaf111827f91caa36`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """
+    And an event is received from the stream `96db5a47a8244eb3b21820781b7d596e`:
+      """yaml
+      event: default.messages.created
+      data:
+        sender: 96db5a47a8244eb3b21820781b7d596e
+        recipient: 004e02a959c04cecaf111827f91caa36
+        text: Hello!
+      """

package/features/steps/Components.ts

@@ -0,0 +1,71 @@
+import { resolve } from 'node:path'
+import { readdirSync } from 'node:fs'
+import { after, before, binding, given, when } from 'cucumber-tsflow'
+import { parse } from '@toa.io/yaml'
+import * as stage from '@toa.io/userland/stage'
+import { type Component, type Request } from '@toa.io/core'
+import { timeout } from '@toa.io/generic'
+import { Realtime } from './Realtime'
+
+@binding([Realtime])
+export class Components {
+  private readonly realtime: Realtime
+  private remotes: Record<string, Component> = {}
+
+  public constructor (realtime: Realtime) {
+    this.realtime = realtime
+  }
+
+  @given('the `{word}` component is running with routes:')
+  public async run (component: string, yaml: string): Promise<void> {
+    const routes = parse<Manifest>(yaml)
+    const [name, namespace = 'default'] = component.split('.').reverse()
+
+    for (const [event, value] of Object.entries(routes)) {
+      const label = `${namespace}.${name}.${event}`
+      const properties = Array.isArray(value) ? value : [value]
+
+      this.realtime.declare(label, properties)
+    }
+  }
+
+  @when('the `{word}` is called with:')
+  public async call (endpoint: string, yaml: string): Promise<void> {
+    const request = parse<Request>(yaml)
+    const [operation, component, namespace = 'default'] = endpoint.split('.').reverse()
+    const id = `${namespace}.${component}`
+
+    this.remotes[id] ??= await stage.remote(id)
+
+    await this.remotes[id].invoke(operation, request)
+    await timeout(500)
+  }
+
+  @before()
+  private async compose (): Promise<void> {
+    const paths = componentPaths()
+
+    await stage.compose(paths)
+  }
+
+  @after()
+  private async shutdown (): Promise<void> {
+    this.remotes = {}
+
+    await stage.shutdown()
+  }
+}
+
+function componentPaths (): string[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => resolve(ROOT, entry.name))
+}
+
+const ROOT = resolve(__dirname, 'components')
+
+interface Manifest {
+  realtime: Map<string, string | string[]>
+}

package/features/steps/Realtime.ts

@@ -0,0 +1,43 @@
+import * as boot from '@toa.io/boot'
+import { encode } from '@toa.io/generic'
+import { type Connector } from '@toa.io/core'
+import { after, binding } from 'cucumber-tsflow'
+import { Factory } from '../../source/Factory'
+
+@binding()
+export class Realtime {
+  private readonly routes: Route[] = []
+  private readonly service: Connector
+  private connected = false
+
+  public constructor () {
+    this.service = new Factory(boot).service()
+  }
+
+  @after()
+  private async shutdown (): Promise<void> {
+    this.connected = false
+
+    await this.service.disconnect()
+  }
+
+  public declare (event: string, properties: string[]): void {
+    this.routes.push({ event, properties })
+  }
+
+  public async serve (): Promise<void> {
+    if (this.connected)
+      return
+
+    process.env.TOA_REALTIME = encode(this.routes)
+
+    this.connected = true
+
+    await this.service.connect()
+  }
+}
+
+interface Route {
+  event: string
+  properties: string[]
+}

package/features/steps/Streams.ts

@@ -0,0 +1,50 @@
+import { type Readable } from 'node:stream'
+import { after, binding, given, then } from 'cucumber-tsflow'
+import { match } from '@toa.io/generic'
+
+import { type Component } from '@toa.io/core'
+import { parse } from '@toa.io/yaml'
+import * as stage from '@toa.io/userland/stage'
+import { Realtime } from './Realtime'
+
+@binding([Realtime])
+export class Streams {
+  private readonly realtime: Realtime
+  private remote: Component | null = null
+  private streams: Record<string, Readable> = {}
+  private events: Record<string, object[]> = {}
+
+  public constructor (realtime: Realtime) {
+    this.realtime = realtime
+  }
+
+  @given('the stream `{word}` is consumed')
+  public async consume (key: string): Promise<void> {
+    await this.realtime.serve()
+
+    this.remote ??= await stage.remote('realtime.streams')
+    this.events[key] = []
+    this.streams[key] = await this.remote.invoke('create', { input: { key } })
+    this.streams[key].on('data', (data: object) => this.events[key].push(data))
+  }
+
+  @then('an event is received from the stream `{word}`:')
+  public received (key: string, yaml: string): void {
+    const expected = parse<object>(yaml)
+
+    for (const event of this.events[key])
+      if (match(event, expected))
+        return
+
+    throw new Error('No matching event received')
+  }
+
+  @after()
+  private shutdown (): void {
+    for (const stream of Object.values(this.streams))
+      stream.destroy()
+
+    this.streams = {}
+    this.events = {}
+  }
+}

package/features/steps/components/messages/manifest.toa.yaml

@@ -0,0 +1,16 @@
+name: messages
+
+entity:
+  schema:
+    sender*: string(32)
+    recipient*: string(32)
+    text*: string
+
+operations:
+  create:
+    query: false
+    forward: transit
+    input:
+      sender*: .
+      recipient*: .
+      text*: .

package/features/steps/config.ts

@@ -0,0 +1 @@
+process.env.TOA_DEV = '1'

package/features/steps/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "outDir": "/dev/null",
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true
+  }
+}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@toa.io/extensions.realtime",
+  "version": "0.0.0-alpha.2",
+  "description": "Toa Realtime",
+  "author": "temich <tema.gurtovoy@gmail.com>",
+  "homepage": "https://github.com/toa-io/toa#readme",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toa-io/toa.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toa-io/toa/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "nopeable": "*"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
+  },
+  "scripts": {
+    "transpile": "npx tsc && npx tsc -p ./components/streams",
+    "features": "npx cucumber-js"
+  },
+  "gitHead": "d6e8c5bdae26e2beb66f4d7f37f71a3494fe9fcb"
+}

package/readme.md

@@ -0,0 +1,99 @@
+# Toa Realtime
+
+## Overview
+
+<a href="https://miro.com/app/board/uXjVOoy0ImU=/?moveToWidget=3458764566111478378&cot=14">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset=".readme/overview-dark.jpg">
+    <img alt="Realtime" width="700" height="202" src=".readme/overview-light.jpg">
+  </picture>
+</a>
+
+Realtime extension combines application events into streams according to defined routes.
+Clients may consume these streams [via Exposition](#exposition).
+
+## Static routes
+
+Static route specifies an event that should be combined into a stream using specified property of
+event's payload as a stream key.
+
+Static routes may be defined in Component manifest or the Context annotation.
+
+```yaml
+# manifest.toa.yaml
+name: users
+
+realtime:
+  updated: id
+```
+
+```yaml
+# context.toa.yaml
+realtime:
+  users.updated: id
+  orders.created: custromer_id
+```
+
+In case of conflict, the Context annotation takes precedence.
+
+### Static route examples
+
+Given two rules: `users.updated: id` and `orders.created: customer_id`,
+the following events will be routed into a stream with `a4b8e7e8` key:
+
+```yaml
+# users.updated
+id: a4b8e7e8 # id property is used as a stream key
+name: John Doe
+```
+
+```yaml
+# orders.created
+id: 1
+customer_id: a4b8e7e8 # customer_id property is used as a stream key
+amount: 100
+```
+
+## Dynamic routes
+
+![Not implemented](https://img.shields.io/badge/Not%20implemented-red)
+
+Dynamic routes address the cases when a stream key is not a property of an event.
+
+Among with an `event` and a `stream` key, a dynamic route has `property` and `value` properties,
+which define a condition that should be met for an event to be combined into a stream with the
+specified key.
+
+### Dynamic route example
+
+For instance, when there are chat rooms with a list of users, and a user joins or leaves the room.
+When a message is sent to a room, an event will have a `room_id` property, but not a `user_id`.
+In this case, when the user `a4b8e7e8` enters the room with id `general`,
+a dynamic route may be created as follows:
+
+```yaml
+event: message.sent
+property: room_id
+value: general
+stream: a4b8e7e8
+```
+
+Each time the message is sent to the room with id `general`, the event will be routed into a
+stream with `a4b8e7e8` key.
+
+### Managing dynamic routes
+
+Dynamic routes are managed by the `realtime.routes` component, running in the Realtime extension.
+
+## Exposition
+
+Streams are exposed by the [`realtime.streams`](components/streams) component, running in the
+Realtime extension, and are
+accessible via the `/realtime/streams/:key/` resource with
+the [`auth:id: key`](/extensions/exposition/documentation/access.md#id) authorization rule.
+
+Refer to the [Exposition extension](/extensions/exposition) for more
+details:
+
+- [Streams](/extensions/exposition/documentation/protocol.md#streams)
+- [Access authorization](/extensions/exposition/documentation/access.md)

package/source/Composition.ts

@@ -0,0 +1,40 @@
+import { readdirSync, type Dirent } from 'node:fs'
+import { resolve } from 'node:path'
+import { Connector } from '@toa.io/core'
+import { type Bootloader } from './Factory'
+
+export class Composition extends Connector {
+  private readonly boot: Bootloader
+
+  public constructor (boot: Bootloader) {
+    super()
+    this.boot = boot
+  }
+
+  protected override async open (): Promise<void> {
+    const paths = find()
+    const composition = await this.boot.composition(paths)
+
+    await composition.connect()
+
+    this.depends(composition)
+
+    console.info('Composition complete.')
+  }
+
+  protected override dispose (): void {
+    console.info('Composition shutdown complete.')
+  }
+}
+
+function find (): string[] {
+  return entries().map((entry) => resolve(ROOT, entry.name))
+}
+
+function entries (): Dirent[] {
+  const entries = readdirSync(ROOT, { withFileTypes: true })
+
+  return entries.filter((entry) => entry.isDirectory())
+}
+
+const ROOT = resolve(__dirname, '../components/')

package/source/Factory.ts

@@ -0,0 +1,33 @@
+import { type Component, type extensions, Locator } from '@toa.io/core'
+import { Realtime } from './Realtime'
+import { Composition } from './Composition'
+import { Routes } from './Routes'
+
+export class Factory implements extensions.Factory {
+  private readonly boot: Bootloader
+
+  public constructor (boot: Bootloader) {
+    this.boot = boot
+  }
+
+  public service (): Realtime {
+    const discovery = this.discovery()
+    const routes = new Routes(this.boot)
+    const composition = new Composition(this.boot)
+    const realtime = new Realtime(routes, discovery)
+
+    realtime.depends(routes)
+    realtime.depends(composition)
+
+    return realtime
+  }
+
+  private async discovery (): Promise<Component> {
+    const locator = new Locator('streams', 'realtime')
+
+    return await this.boot.remote(locator)
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+export type Bootloader = typeof import('@toa.io/boot')

package/source/Realtime.ts

@@ -0,0 +1,38 @@
+import { type Component, Connector } from '@toa.io/core'
+import { type Routes } from './Routes'
+
+export class Realtime extends Connector {
+  private readonly discovery: Promise<Component>
+  private streams: Component | null = null
+
+  public constructor (routes: Routes, discovery: Promise<Component>) {
+    super()
+
+    this.discovery = discovery
+
+    routes.events.on('data', this.push.bind(this))
+  }
+
+  protected override async open (): Promise<void> {
+    this.streams = await this.discovery
+    this.depends(this.streams)
+
+    await this.streams.connect()
+
+    console.log('Realtime has started.')
+  }
+
+  protected override dispose (): void {
+    console.log('Realtime shutdown complete.')
+  }
+
+  private push (event: Event): void {
+    void this.streams?.invoke('push', { input: event })
+  }
+}
+
+interface Event {
+  key: string
+  event: string
+  data: string
+}

package/source/Routes.ts

@@ -0,0 +1,84 @@
+import { Readable } from 'node:stream'
+import { Connector, type Message } from '@toa.io/core'
+import { decode } from '@toa.io/generic'
+import { type Bootloader } from './Factory'
+
+export class Routes extends Connector {
+  public events = new Events()
+
+  private readonly boot: Bootloader
+
+  public constructor (boot: Bootloader) {
+    super()
+
+    this.boot = boot
+  }
+
+  private static read (): Route[] {
+    if (process.env.TOA_REALTIME === undefined)
+      throw new Error('TOA_REALTIME is not defined')
+
+    return decode<Route[]>(process.env.TOA_REALTIME)
+  }
+
+  public override async open (): Promise<void> {
+    const routes = Routes.read()
+    const creating = []
+
+    for (const { event, properties } of routes) {
+      const consumer = this.boot.receive(event, this.getReceiver(event, properties))
+
+      creating.push(consumer)
+    }
+
+    const consumers = await Promise.all(creating)
+
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    const connecting = consumers.map((consumer) => consumer.connect())
+
+    await Promise.all(connecting)
+    this.depends(consumers)
+
+    console.log(`Event sources (${creating.length}) connected.`)
+  }
+
+  public override async close (): Promise<void> {
+    console.log('Event sources disconnected.')
+  }
+
+  private getReceiver (event: string, properties: string[]): Receiver {
+    return {
+      receive: (message: Message<Record<string, string>>) => {
+        for (const property of properties) {
+          const key = message.payload[property]
+
+          if (key === undefined) {
+            console.error(`Event '${event}' does not contain the '${property}' property.`)
+
+            return
+          }
+
+          this.events.push({ key, event, data: message.payload })
+        }
+      }
+    }
+  }
+}
+
+class Events extends Readable {
+  public constructor () {
+    super({ objectMode: true })
+  }
+
+  public override _read (): void {
+  }
+}
+
+export interface Route {
+  event: string
+  properties: string[]
+}
+
+interface Receiver {
+  receive: (message: Message<Record<string, string>>) => void
+}

package/source/index.ts

@@ -0,0 +1 @@
+export { Factory } from './Factory'

package/stage/streams.test.ts

@@ -0,0 +1,128 @@
+import { resolve } from 'node:path'
+import { type Readable } from 'node:stream'
+import { generate } from 'randomstring'
+import * as stage from '@toa.io/userland/stage'
+import { type Component } from '@toa.io/core'
+import { newid, timeout } from '@toa.io/generic'
+
+let component: Component
+let stream: Readable
+let events: Record<string, any[]> = {}
+
+beforeEach(async () => {
+  await run()
+})
+
+afterEach(async () => {
+  stream?.destroy()
+  await stop()
+  events = {}
+})
+
+it('should route events', async () => {
+  const event = generate()
+  const data = generate()
+  const key1 = newid()
+  const key2 = newid()
+
+  await create(key1)
+  await create(key2)
+  await push(key1, event, data)
+  await push(key2, event, data)
+
+  expect(events[key1]).toEqual([{ event, data }])
+  expect(events[key2]).toEqual([{ event, data }])
+})
+
+it('should create fresh stream', async () => {
+  const data = { foo: generate() }
+  const key = newid()
+
+  await create(key)
+  await push(key, 'first', data)
+
+  expect(events[key]).toEqual([{ event: 'first', data }])
+
+  await create(key)
+  await timeout(0)
+
+  expect(events[key]).toEqual([])
+
+  await push(key, 'second', data)
+
+  expect(events[key]).toEqual([{ event: 'second', data }])
+})
+
+it('should ignore pushes to non-created streams', async () => {
+  const data = { foo: generate() }
+  const key = newid()
+
+  await push(key, 'first', data)
+  await create(key)
+
+  expect(events[key]).toEqual([])
+
+  await push(key, 'second', data)
+
+  expect(events[key]).toEqual([{ event: 'second', data }])
+})
+
+it('should not destroy source stream if one of the forks is destroyed', async () => {
+  const data = generate()
+  const key = newid()
+
+  await create(key)
+  await push(key, 'first', data)
+
+  const firstStream = stream
+  const firstStreamEvents = events[key]
+
+  await create(key)
+  await push(key, 'second', data)
+
+  expect(firstStreamEvents).toEqual([{ event: 'first', data }, { event: 'second', data }])
+  expect(stream === firstStream).toBe(false)
+
+  stream.destroy()
+
+  await push(key, 'third', data)
+
+  expect(firstStreamEvents).toEqual([
+    { event: 'first', data },
+    { event: 'second', data },
+    { event: 'third', data }
+  ])
+
+  firstStream.destroy()
+})
+
+/// region component
+
+async function run (name: string = 'streams'): Promise<void> {
+  const path = resolve(__dirname, `../components/${name}`)
+
+  component = await stage.component(path)
+}
+
+async function create (key: string): Promise<void> {
+  const chunks: any[] = []
+
+  events[key] = chunks
+
+  stream = await component.invoke('create', { input: { key } })
+  stream.on('data', (chunk) => chunks.push(chunk))
+}
+
+async function stop (): Promise<void> {
+  await component?.disconnect()
+}
+
+async function push (key: string, event: string, data: any): Promise<void> {
+  const reply = await component.invoke<any>('push', { input: { key, event, data } })
+
+  expect(reply?.exception).toBeUndefined()
+
+  await timeout(0) // wait for stream internals
+}
+
+/// endregion

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./transpiled"
+  },
+  "include": [
+    "source"
+  ],
+  "exclude": [
+    "**/*.test.ts"
+  ]
+}