@toa.io/extensions.realtime 1.0.0-alpha.291 → 1.0.0-alpha.293

package/features/steps/Components.ts

@@ -1,72 +0,0 @@
-import { resolve } from 'node:path'
-import { readdirSync } from 'node:fs'
-import tsflow from 'cucumber-tsflow'
-
-import { load as parse } from 'js-yaml'
-import * as stage from '@toa.io/userland/stage'
-import { type Component } from '@toa.io/core'
-import type { Request } from '@toa.io/core/types'
-import { timeout } from '@toa.io/generic'
-import { parse as parseRoutes, type Declaration } from '../../source/extension.js'
-import { Realtime } from './Realtime.js'
-
-const { after, before, binding, given, when } = tsflow
-
-@binding([Realtime])
-export class Components {
-  private readonly realtime: Realtime
-  private remotes: Record<string, Component> = {}
-
-  public constructor(realtime: Realtime) {
-    this.realtime = realtime
-  }
-
-  @given('the `{word}` component is running with routes:')
-  public async run(component: string, yaml: string): Promise<void> {
-    const declaration = parse(yaml) as Declaration
-    const [name, namespace = 'default'] = component.split('.').reverse()
-    const routes = parseRoutes(declaration)
-
-    for (const route of routes) {
-      const label = `${namespace}.${name}.${route.event}`
-
-      this.realtime.declare(label, route.properties, route.expose)
-    }
-  }
-
-  @when('the `{word}` is called with:')
-  public async call(endpoint: string, yaml: string): Promise<void> {
-    const request = parse(yaml) as Request
-    const [operation, component, namespace = 'default'] = endpoint.split('.').reverse()
-    const id = `${namespace}.${component}`
-
-    this.remotes[id] ??= await stage.remote(id)
-
-    await this.remotes[id].invoke(operation, request)
-    await timeout(500)
-  }
-
-  @before()
-  private async compose(): Promise<void> {
-    const paths = componentPaths()
-
-    await stage.compose(paths)
-  }
-
-  @after()
-  private async shutdown(): Promise<void> {
-    this.remotes = {}
-
-    await stage.shutdown()
-  }
-}
-
-function componentPaths(): string[] {
-  const entries = readdirSync(ROOT, { withFileTypes: true })
-
-  return entries
-    .filter((entry) => entry.isDirectory())
-    .map((entry) => resolve(ROOT, entry.name))
-}
-
-const ROOT = resolve(import.meta.dirname, 'components')

package/features/steps/Realtime.ts

@@ -1,40 +0,0 @@
-import * as boot from '@toa.io/boot'
-import { type Connector } from '@toa.io/core'
-import { environment } from '@toa.io/generic'
-
-import tsflow from 'cucumber-tsflow'
-
-import { Factory } from '../../source/index.js'
-import type { Route } from '../../source/extension.js'
-
-const { after, binding } = tsflow
-
-@binding()
-export class Realtime {
-  private readonly routes: Route[] = []
-  // the factory loads its binding, so the service is made when it is first served
-  private service: Connector | undefined
-  private connected = false
-
-  @after()
-  public async shutdown(): Promise<void> {
-    this.connected = false
-
-    await this.service?.disconnect()
-  }
-
-  public declare(event: string, properties: string[], expose?: string[]): void {
-    this.routes.push({ event, properties, expose })
-  }
-
-  public async serve(): Promise<void> {
-    if (this.connected) return
-
-    environment.set('TOA_REALTIME', JSON.stringify(this.routes))
-
-    this.connected = true
-    this.service = await new Factory(boot.host()).service()
-
-    await this.service.connect()
-  }
-}

package/features/steps/Streams.ts

@@ -1,85 +0,0 @@
-import * as assert from 'node:assert'
-import { setTimeout } from 'node:timers/promises'
-import tsflow from 'cucumber-tsflow'
-
-import { match } from '@toa.io/generic'
-import { load as parse } from 'js-yaml'
-import * as stage from '@toa.io/userland/stage'
-import { Realtime } from './Realtime.js'
-import type { Readable } from 'node:stream'
-import type { Component } from '@toa.io/core'
-
-const { after, binding, given, then } = tsflow
-
-@binding([Realtime])
-export class Streams {
-  private readonly realtime: Realtime
-  private remote: Component | null = null
-  private streams: Record<string, Readable> = {}
-  private events: Record<string, Event[]> = {}
-
-  public constructor(realtime: Realtime) {
-    this.realtime = realtime
-  }
-
-  @given('the stream `{word}` is consumed', { timeout: 30_000 })
-  public async consume(key: string): Promise<void> {
-    await this.realtime.serve()
-
-    this.remote ??= await stage.remote('realtime.streams')
-    this.events[key] = []
-
-    await this.createStream(key)
-  }
-
-  @then('an event is received from the stream `{word}`:')
-  public async received(key: string, yaml: string): Promise<void> {
-    await setTimeout(100)
-
-    const expected = parse(yaml) as object
-
-    for (const event of this.events[key]) if (match(event, expected)) return
-
-    throw new Error('No matching event received')
-  }
-
-  @then('the consumer `{word}` is disconnected')
-  public disconnected(key: string): void {
-    this.streams[key]?.destroy()
-    delete this.streams[key]
-  }
-
-  @then('the consumer `{word}` is reconnected')
-  public async reconnected(key: string): Promise<void> {
-    const last = this.events[key].findLast((event) => event.event === 'token')
-
-    assert.ok(last, `No last event found for stream ${key}`)
-
-    await this.createStream(key, last.data as string)
-  }
-
-  @after()
-  public async shutdown(): Promise<void> {
-    for (const stream of Object.values(this.streams)) stream.destroy()
-
-    this.streams = {}
-    this.events = {}
-
-    await setTimeout(100)
-  }
-
-  private async createStream(key: string, token?: string): Promise<void> {
-    this.streams[key] = await this.remote!.invoke('create', { input: { key, token } })
-    this.streams[key].on('data', (event: Event) => {
-      console.log('[TEST] Received event', event)
-      this.events[key]?.push(event)
-    })
-  }
-}
-
-interface Event {
-  key: string
-  token: string
-  event: string
-  data?: unknown
-}

package/features/steps/components/messages/manifest.toa.yaml

@@ -1,45 +0,0 @@
-name: messages
-
-entity:
-  properties:
-    watchers:
-      type: array
-      items:
-        type: string
-        maxLength: 32
-    sender:
-      type: string
-      maxLength: 32
-    recipient:
-      type: string
-      maxLength: 32
-    text:
-      type: string
-  required:
-    - sender
-    - recipient
-    - text
-
-operations:
-  create:
-    query: false
-    forward: transit
-    input:
-      properties:
-        watchers:
-          type: string
-          default: .
-        sender:
-          type: string
-          default: .
-        recipient:
-          type: string
-          default: .
-        text:
-          type: string
-          default: .
-      type: object
-      required:
-        - sender
-        - recipient
-        - text

package/features/steps/config.ts

@@ -1,15 +0,0 @@
-import { setDefaultTimeout } from '@cucumber/cucumber'
-import { environment } from '@toa.io/generic'
-
-environment.set('TOA_DEV', '1')
-
-// a reply is checked against what the operation declares, so the suite runs Toa under the
-// contract it asks applications to keep
-if (!environment.has('TOA_ENV')) environment.set('TOA_ENV', 'local')
-
-// the streams component boots inside the service, and without a variable it would wait
-// for the values service, which these features do not run
-if (!environment.has('TOA_CONFIGURATION_REALTIME_STREAMS'))
-  environment.set('TOA_CONFIGURATION_REALTIME_STREAMS', '{}')
-
-setDefaultTimeout(60 * 1000)

package/features/steps/tsconfig.json

@@ -1,9 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "noEmit": true,
-    "moduleResolution": "node",
-    "experimentalDecorators": true,
-    "emitDecoratorMetadata": true
-  }
-}

package/source/Composition.ts

@@ -1,51 +0,0 @@
-import { readdirSync, type Dirent } from 'node:fs'
-import { resolve } from 'node:path'
-import { Connector } from '@toa.io/core'
-import { type Host } from './Factory.js'
-
-export class Composition extends Connector {
-  private readonly host: Host
-
-  public constructor(host: Host) {
-    super()
-    this.host = host
-  }
-
-  protected override async open(): Promise<void> {
-    const paths = find()
-    const composition = await this.host.composition(paths)
-
-    await composition.connect()
-
-    this.depends(composition)
-  }
-}
-
-export function find(): string[] {
-  return entries().map((entry) => resolve(ROOT, entry.name))
-}
-
-function entries(): Dirent[] {
-  const entries = readdirSync(ROOT, { withFileTypes: true })
-
-  return entries.filter((entry) => entry.isDirectory())
-}
-
-export function components(): Components {
-  const labels: string[] = []
-  const paths: string[] = []
-
-  for (const entry of entries()) {
-    labels.push(entry.name.replace('.', '-'))
-    paths.push(resolve(ROOT, entry.name))
-  }
-
-  return { labels, paths }
-}
-
-interface Components {
-  labels: string[]
-  paths: string[]
-}
-
-const ROOT = resolve(import.meta.dirname, '../components/')

package/source/Factory.ts

@@ -1,32 +0,0 @@
-import { type Component, Locator } from '@toa.io/core'
-import type { extensions } from '@toa.io/core/types'
-import { Realtime } from './Realtime.js'
-import { Composition } from './Composition.js'
-import { Routes } from './Routes.js'
-
-export class Factory implements extensions.Factory {
-  private readonly host: Host
-
-  public constructor(host: Host) {
-    this.host = host
-  }
-
-  public service(): Realtime {
-    const routes = new Routes(this.host)
-    const composition = new Composition(this.host)
-    const realtime = new Realtime(routes, async () => await this.discovery())
-
-    realtime.depends(routes)
-    realtime.depends(composition)
-
-    return realtime
-  }
-
-  private async discovery(): Promise<Component> {
-    const locator = new Locator('streams', 'realtime')
-
-    return await this.host.remote(locator, { service: 'realtime' })
-  }
-}
-
-export type Host = extensions.Host

package/source/Realtime.ts

@@ -1,71 +0,0 @@
-import { console, run } from 'openspan'
-import { type Component, Connector } from '@toa.io/core'
-import { type Routes } from './Routes.js'
-import type { Push } from './Receiver.js'
-
-export class Realtime extends Connector {
-  private readonly discovery: () => Promise<Component>
-  private streams: Component | null = null
-
-  public constructor(routes: Routes, discovery: () => Promise<Component>) {
-    super()
-
-    this.discovery = discovery
-
-    routes.events.on('data', this.push.bind(this))
-  }
-
-  protected override async open(): Promise<void> {
-    // the lookup belongs here, not in the constructor: dependencies connect before
-    // open, so the composition that serves realtime.streams is up to answer it. Asked
-    // any earlier the lookup goes unanswered, and it waits without a bound.
-    this.streams = await this.discovery()
-    this.depends(this.streams)
-
-    await this.streams.connect()
-
-    console.info('Realtime service started')
-  }
-
-  protected override dispose(): void {
-    console.info('Realtime service shutdown complete')
-  }
-
-  private push({ telemetry, ...event }: Push): void {
-    const processing =
-      telemetry === null
-        ? this.deliver(event)
-        : run(telemetry, async () => await this.deliver(event))
-
-    void processing.catch((error) => console.error('Realtime push failed', error))
-  }
-
-  private async deliver(event: Omit<Push, 'telemetry'>): Promise<void> {
-    /*
-     * The delivery span is created on behalf of the messaging destination
-     * (same as the core Receiver), so that service graphs display the fan-out:
-     * producer -> destination -> realtime
-     */
-    const delivery = {
-      name: `${event.event} deliver`,
-      kind: 'producer' as const,
-      service: event.event,
-      attributes: { 'messaging.destination.name': event.event }
-    }
-
-    const options = {
-      name: `${event.event} push`,
-      kind: 'consumer' as const,
-      service: 'realtime',
-      attributes: { 'messaging.destination.name': event.event }
-    }
-
-    await console.span(
-      delivery,
-      async () =>
-        await console.span(options, async () => {
-          await this.streams?.invoke('push', { input: event })
-        })
-    )
-  }
-}

package/source/Receiver.ts

@@ -1,95 +0,0 @@
-import { console, decode, run } from 'openspan'
-import { Connector } from '@toa.io/core'
-import type { Message } from '@toa.io/core/types'
-import type { SpanContext } from 'openspan'
-import type { Readable } from 'node:stream'
-
-export class Receiver extends Connector {
-  private readonly event: string
-  private readonly properties: string[]
-  private readonly expose?: string[]
-  private readonly stream: Readable
-
-  public constructor({
-    event,
-    properties,
-    stream,
-    expose
-  }: {
-    event: string
-    properties: string[]
-    stream: Readable
-    expose?: string[]
-  }) {
-    super()
-
-    this.event = event
-    this.properties = properties
-    this.expose = expose
-    this.stream = stream
-  }
-
-  public async receive(message: Message<Record<string, string>>): Promise<void> {
-    // the push continues the trace from the producer
-    const telemetry = message.telemetry === undefined ? null : decode(message.telemetry)
-
-    if (telemetry === null) this.process(message, telemetry)
-    else run(telemetry, () => this.process(message, telemetry))
-  }
-
-  private process(
-    message: Message<Record<string, string>>,
-    telemetry: SpanContext | null
-  ): void {
-    const data = this.fit(message.payload)
-
-    for (const property of this.properties) {
-      const key = message.payload[property]
-
-      if (key === undefined) {
-        console.debug('Event does not contain key property', {
-          property,
-          event: this.event
-        })
-
-        continue
-      }
-
-      if (Array.isArray(key))
-        // eslint-disable-next-line max-depth
-        for (const k of key as string[]) this.push(k, data, telemetry)
-      else this.push(key, data, telemetry)
-    }
-  }
-
-  private fit(payload: Record<string, string>): Record<string, string> {
-    if (this.expose === undefined) return payload
-
-    const entries = Object.entries(payload).filter(([key]) => this.expose!.includes(key))
-
-    return Object.fromEntries(entries)
-  }
-
-  private push(
-    key: string | null,
-    data: Record<string, string>,
-    telemetry: SpanContext | null
-  ): void {
-    if (key === null || typeof key === 'undefined') {
-      console.debug('Key is null or undefined, skipping', { key, event: this.event })
-
-      return
-    }
-
-    console.debug('Pushing event to stream', { key, event: this.event, data })
-
-    this.stream.push({ key, event: this.event, data, telemetry } satisfies Push)
-  }
-}
-
-export interface Push {
-  key: string
-  event: string
-  data: Record<string, string>
-  telemetry: SpanContext | null
-}

package/source/Routes.ts

@@ -1,65 +0,0 @@
-import { Readable } from 'node:stream'
-import { console } from 'openspan'
-import { Connector } from '@toa.io/core'
-import { Receiver } from './Receiver.js'
-import { environment } from '@toa.io/generic'
-import type { Route } from './extension.js'
-import type { Host } from './Factory.js'
-
-export class Routes extends Connector {
-  public events = new Events()
-
-  private readonly host: Host
-
-  public constructor(host: Host) {
-    super()
-
-    this.host = host
-  }
-
-  private static read(): Route[] {
-    const value = environment.get('TOA_REALTIME')
-
-    if (value === undefined) throw new Error('TOA_REALTIME is not defined')
-
-    return JSON.parse(value) as Route[]
-  }
-
-  public override async open(): Promise<void> {
-    const routes = Routes.read()
-    const creating = []
-
-    for (const { event, properties, expose } of routes) {
-      const consumer = this.host.receive(
-        event,
-        new Receiver({ event, properties, stream: this.events, expose })
-      )
-
-      creating.push(consumer)
-    }
-
-    const consumers = await Promise.all(creating)
-
-    // eslint-disable-next-line @typescript-eslint/promise-function-async
-    const connecting = consumers.map((consumer) => consumer.connect())
-
-    await Promise.all(connecting)
-    this.depends(consumers)
-
-    console.info('Event sources connected', { count: creating.length })
-  }
-
-  public override async close(): Promise<void> {
-    console.info('Event sources disconnected')
-  }
-}
-
-class Events extends Readable {
-  public constructor() {
-    super({ objectMode: true })
-  }
-
-  public override _read(): void {}
-}
-
-export type { Route }

package/source/extension.ts

@@ -1,92 +0,0 @@
-import { readFileSync } from 'node:fs'
-import { components } from './Composition.js'
-import type { Dependency, Instances, Resources, Service } from '@toa.io/operations'
-
-/** Where Toa's release publishes this service's image. An application takes it
- *  instead of building one when its context says `registry.services: published`. */
-export const image = 'ghcr.io/toa-io/extension-realtime-streams'
-
-export const standalone = true
-export { components } from './Composition.js'
-
-export function deployment(
-  instances: Instances<Declaration>,
-  annotation?: Declaration & Annotation
-): Dependency {
-  const routes = []
-  const { resources, ...annotatedRoutes } = annotation ?? {}
-  const labels = components().labels
-
-  if (annotatedRoutes !== undefined) routes.push(...parse(annotatedRoutes))
-
-  for (const instance of instances) {
-    const completed: Declaration = {}
-
-    for (const [key, value] of Object.entries(instance.manifest)) {
-      const event = instance.locator.id + '.' + key
-
-      completed[event] = value
-    }
-
-    routes.push(...parse(completed))
-  }
-
-  const service: Service = {
-    group: 'realtime',
-    name: 'streams',
-    image,
-
-    version: JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
-      .version,
-    components: labels,
-    resources,
-    variables: [
-      {
-        name: 'TOA_REALTIME',
-        value: JSON.stringify(routes)
-      }
-    ]
-  }
-
-  return { services: [service], events: routes.map((route) => route.event) }
-}
-
-export function parse(declaration: Declaration): Route[] {
-  const routes: Route[] = []
-
-  for (const [event, value] of Object.entries(declaration))
-    if (isObject(value)) {
-      const properties = Array.isArray(value.key) ? value.key : [value.key]
-
-      routes.push({ event, properties, expose: value.expose })
-    } else {
-      const properties = Array.isArray(value) ? value : [value]
-
-      routes.push({ event, properties })
-    }
-
-  return routes
-}
-
-function isObject(value: Entry): value is RouteDeclaration {
-  return typeof value === 'object' && !Array.isArray(value)
-}
-
-export type Entry = string | string[] | RouteDeclaration
-
-export type Declaration = Record<string, Entry>
-
-export interface RouteDeclaration {
-  key: string | string[]
-  expose?: string[]
-}
-
-export interface Route {
-  event: string
-  properties: string[]
-  expose?: string[]
-}
-
-interface Annotation {
-  resources?: Resources
-}

package/source/index.ts

@@ -1,2 +0,0 @@
-export { Factory } from './Factory.js'
-export * from './extension.js'