@toa.io/core 1.0.0-alpha.284 → 1.0.0-alpha.287

package/{src/outbox/outbox.js → source/outbox/outbox.ts}

@@ -1,6 +1,17 @@
 import { console } from 'openspan'
 import { Connector } from '../connector.js'
 import { newid } from '../entities/newid.js'
+import type { Emission } from '../emission.js'
+import type { Atom } from '../types/atomicity.js'
+import type { Storage } from '../types/storages.js'
+import type { Row } from '../types/outbox.js'
+import type { Event } from '../types/state.js'
+
+export interface Options {
+  interval?: number
+  batch?: number
+  gap?: number
+}
 
 /**
  * Owns the intent to publish. A row is built before the write so that the storage can commit
@@ -14,30 +25,36 @@ import { newid } from '../entities/newid.js'
  * inline emission it replaces.
  */
 export class Outbox extends Connector {
-  #emission
-  #storage
-  #atom
+  readonly #emission: Emission
+  readonly #storage: Storage | undefined
+  readonly #atom: Atom
 
-  #gap
-  #interval
-  #batch
-  #defer
+  readonly #gap: number
+  readonly #interval: number
+  readonly #batch: number
+  readonly #defer: boolean
 
   /** ids this process has published, held until a cycle marks them */
-  #published = new Set()
+  readonly #published = new Set<string>()
 
   /** in-flight publications, awaited (with a bound) on close */
-  #inflight = new Set()
+  readonly #inflight = new Set<Promise<void>>()
 
   /** rows this replica is publishing right now, so a cycle does not pick them up again */
-  #publishing = new Set()
+  readonly #publishing = new Set<string>()
 
-  #timer
-  #off
+  #timer: NodeJS.Timeout | undefined
+  #off: (() => void) | undefined
   #pumping = false
   #closing = false
 
-  constructor (emission, storage, atom, options = {}) {
+  // eslint-disable-next-line max-params
+  public constructor(
+    emission: Emission,
+    storage: Storage | undefined,
+    atom: Atom,
+    options: Options = {}
+  ) {
     super()
 
     this.#emission = emission
@@ -56,21 +73,21 @@ export class Outbox extends Connector {
   }
 
   /** whether the storage can commit a row atomically with the entity */
-  get durable () {
+  public get durable(): boolean {
     return this.#storage?.outbox !== undefined
   }
 
   /**
-   * @param event {toa.core.transition.Event}
-   * @returns {object}
+   * An assignment's images are the write's own, so it hands over an event with neither, and
+   * the storage fills them in.
    */
-  row (event) {
+  public row(event: Partial<Event>): Row {
     return {
       id: newid(),
       lane: this.#lane(),
       published: false,
       pending: Date.now() + this.#gap,
-      event
+      event: event as Event
     }
   }
 
@@ -78,29 +95,37 @@ export class Outbox extends Connector {
    * Hands a committed row over. Awaited by the caller only on the legacy path — with an
    * outbox this returns at once and the broker leaves the operation's path.
    */
-  publish (row) {
-    if (!this.durable)
-      return this.#emission.emit(row.event)
+  public publish(row: Row): Promise<void> | void {
+    // without a durable outbox this is the inline path, and the caller awaits the emission
+    if (!this.durable) return this.#emission.emit(row.event)
 
     /*
      * A publication started while the pump is closing would outlive the emitters it needs,
      * and `comq` waits on a connection that is going rather than failing. The row is already
      * durable, so leaving it is exactly what it is for.
      */
-    if (this.#closing || this.#defer ||
-      this.#inflight.size >= INFLIGHT || this.#published.size >= PUBLISHED)
+    if (
+      this.#closing ||
+      this.#defer ||
+      this.#inflight.size >= INFLIGHT ||
+      this.#published.size >= PUBLISHED
+    )
       return
 
     void this.#publish(row)
   }
 
-  async open () {
+  protected override async open(): Promise<void> {
     if (!this.durable) return
 
     if (this.#defer)
-      console.warn('Outbox immediate publication is deferred; events are published by the pump only')
+      console.warn(
+        'Outbox immediate publication is deferred; events are published by the pump only'
+      )
 
-    this.#timer = setInterval(() => this.#tick(), this.#interval)
+    this.#timer = setInterval(() => {
+      this.#tick()
+    }, this.#interval)
     this.#timer.unref()
 
     /*
@@ -108,10 +133,12 @@ export class Outbox extends Connector {
      * publish, and the cycle would not notice for up to an interval. Being told costs a cycle
      * that finds nothing in the usual case, where the claim arrives once and never changes.
      */
-    this.#off = this.#atom.onassigned(() => this.#tick())
+    this.#off = this.#atom.onassigned(() => {
+      this.#tick()
+    })
   }
 
-  async close () {
+  protected override async close(): Promise<void> {
     this.#closing = true
 
     this.#off?.()
@@ -131,9 +158,8 @@ export class Outbox extends Connector {
    * it would not stop it, it would only mean the row is published twice once it lands. What
    * bounds this instead is the in-flight cap and the drain on close.
    *
-   * @private
    */
-  async #publish (row) {
+  async #publish(row: Row): Promise<void> {
     this.#publishing.add(row.id)
 
     const publishing = this.#emission.emit(row.event)
@@ -156,9 +182,8 @@ export class Outbox extends Connector {
    * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
    * unbounded drain outlives any grace period.
    *
-   * @private
    */
-  async #drain () {
+  async #drain(): Promise<void> {
     if (this.#inflight.size === 0) return
 
     await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)])
@@ -168,9 +193,8 @@ export class Outbox extends Connector {
    * Reads what is due, publishes it, and marks everything this process has sent — what it just
    * published and what the immediate path published since the last cycle. One cycle at a time.
    *
-   * @private
    */
-  #tick () {
+  #tick(): void {
     if (this.#pumping) return
 
     this.#pumping = true
@@ -178,29 +202,33 @@ export class Outbox extends Connector {
     void this.#pump().finally(() => (this.#pumping = false))
   }
 
-  /** @private */
-  async #pump () {
-    let page
+  async #pump(): Promise<void> {
+    let page: Row[]
+    let after: string | undefined
 
     do {
-      page = await this.#read(page?.[page.length - 1]?.id)
+      page = await this.#read(after)
 
       if (page.length === 0) break
 
+      // so a page is never read twice
+      after = page[page.length - 1]?.id
+
       /*
        * A row is unpublished in the database until a cycle marks it, so a page includes what
        * this replica is sending right now and what a failed marking left behind. Only this
        * process knows either.
        */
-      const rows = page.filter((row) =>
-        !this.#published.has(row.id) && !this.#publishing.has(row.id))
+      const rows = page.filter(
+        (row) => !this.#published.has(row.id) && !this.#publishing.has(row.id)
+      )
 
       if (rows.length > 0) {
         console.info('Outbox recovering unpublished events', { count: rows.length })
 
         // every row is given its chance; what the broker refused stays unpublished and comes
         // back on a later cycle
-        await Promise.allSettled(rows.map((row) => this.#publish(row)))
+        await Promise.allSettled(rows.map(async (row) => this.#publish(row)))
       }
 
       // a full page is a page that may have been cut short
@@ -218,15 +246,15 @@ export class Outbox extends Connector {
    * assignment arrives. Reading without an assignment would be a different guarantee, where
    * every replica publishes every stranded row.
    *
-   * @private
-   * @param {string} [after] the last id of the page before, so a page is never read twice
+   * @param after the last id of the page before, so a page is never read twice
    */
-  async #read (after) {
+  async #read(after?: string): Promise<Row[]> {
     const lanes = this.#atom.slots(LANES)
 
     if (lanes === null || lanes.length === 0) return []
 
-    return this.#storage.outbox.pending(lanes, Date.now(), this.#batch, after)
+    return this.#storage!
+      .outbox!.pending(lanes, Date.now(), this.#batch, after)
       .catch((error) => {
         console.warn('Outbox read failed', { error })
 
@@ -239,15 +267,14 @@ export class Outbox extends Connector {
    * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
    * marked is simply published again, which is within the contract.
    *
-   * @private
    */
-  async #mark () {
+  async #mark(): Promise<void> {
     if (this.#published.size === 0) return
 
     const ids = [...this.#published]
 
     try {
-      await this.#storage.outbox.settle(ids)
+      await this.#storage!.outbox!.settle(ids)
 
       for (const id of ids) this.#published.delete(id)
     } catch (error) {
@@ -255,14 +282,13 @@ export class Outbox extends Connector {
     }
   }
 
-   /**
+  /**
    * A lane this replica currently owns, so that in steady state it settles its own rows
    * before it ever reads them. Any lane at all when it owns none: the row still has to be
    * written, and whoever ends up owning that lane will pump it.
    *
-   * @private
    */
-  #lane () {
+  #lane(): number {
     const owned = this.#atom.slots(LANES)
 
     return owned === null || owned.length === 0
@@ -271,7 +297,11 @@ export class Outbox extends Connector {
   }
 }
 
-function number (variable, declared, fallback) {
+function number(
+  variable: string,
+  declared: number | undefined,
+  fallback: number
+): number {
   if (declared !== undefined) return declared
 
   const value = Number(process.env[variable])
@@ -279,7 +309,11 @@ function number (variable, declared, fallback) {
   return Number.isNaN(value) || value <= 0 ? fallback : value
 }
 
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref())
+async function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms).unref()
+  })
+}
 
 /**
  * Constant, never configuration: rows carry their lane, so lowering this would leave rows in

package/source/query/criteria.ts

@@ -0,0 +1,90 @@
+import { parse } from '@rsql/parser'
+import { QuerySyntaxException } from '../exceptions.js'
+import type { Node } from '../types/storages.js'
+
+/** What a component declares about the properties a criteria may select on. */
+export type Properties = Record<string, { type: string }>
+
+export function criteria(expression: string, properties?: Properties): Node {
+  let ast: Node
+
+  try {
+    ast = parse(expression) as unknown as Node
+  } catch (e) {
+    throw new QuerySyntaxException((e as Error).message)
+  }
+
+  if (properties !== undefined) read(ast, properties)
+
+  return ast
+}
+
+function read(node: Node, properties: Properties): void {
+  if (
+    node.type === 'COMPARISON' &&
+    node.left?.type === 'SELECTOR' &&
+    node.right?.type === 'VALUE'
+  ) {
+    const selector = node.left.selector as string
+    const property = properties[selector]
+
+    if (property === undefined) {
+      throw new QuerySyntaxException(`Criteria selector '${selector}' is not defined`)
+    }
+
+    const cast = CAST[property.type]
+
+    // `=in=` and `=out=` carry a list, and casting that as one value gives whatever
+    // a comma-separated string reads as
+    if (cast !== undefined)
+      node.right.value = Array.isArray(node.right.value)
+        ? node.right.value.map((value: string) => cast(value, selector))
+        : cast(node.right.value as string, selector)
+  } else {
+    if (node.left !== undefined) read(node.left, properties)
+    if (node.right !== undefined) read(node.right, properties)
+  }
+}
+
+/**
+ * A criteria arrives as text, so a value is read as what the property it selects on holds. What
+ * cannot be read as that is refused rather than passed on: `parseInt` answers `NaN` for a word
+ * and `12` for `12kg`, and a storage asked to match either finds nothing and says nothing —
+ * a mistyped filter reads as an empty result.
+ *
+ * A string is left as it came, because every text is one: what a criteria may compare a string
+ * to is not the type's to say.
+ */
+const CAST: Record<string, (value: string, selector: string) => unknown> = {
+  number: (value, selector) => finite(value, selector, 'a number'),
+
+  integer: (value, selector) => {
+    const number = finite(value, selector, 'an integer')
+
+    if (!Number.isInteger(number)) refuse(selector, 'an integer', value)
+
+    return number
+  },
+
+  boolean: (value, selector) => {
+    if (value === 'true') return true
+    if (value === 'false') return false
+
+    return refuse(selector, 'a boolean', value)
+  }
+}
+
+function finite(value: string, selector: string, expected: string): number {
+  // `Number` reads a blank string as zero, and a criteria that says nothing says nothing
+  const number = value.trim() === '' ? Number.NaN : Number(value)
+
+  if (!Number.isFinite(number)) refuse(selector, expected, value)
+
+  return number
+}
+
+function refuse(selector: string, expected: string, value: string): never {
+  throw new QuerySyntaxException(
+    `Criteria selector '${selector}' takes ${expected}, and '${value}' is not one`
+  )
+}

package/source/query/options.ts

@@ -0,0 +1,35 @@
+import { QuerySyntaxException } from '../exceptions.js'
+import type { Options } from '../types/storages.js'
+import type { Properties } from './criteria.js'
+
+export function options(given: Record<string, any>, properties: Properties): Options {
+  if (given.sort !== undefined) given.sort = sort(given.sort, properties)
+
+  if (given.projection !== undefined) projection(given.projection, properties)
+
+  return given as Options
+}
+
+function sort(sort: string[], properties: Properties): Array<[string, string]> {
+  const result: Array<[string, string]> = []
+
+  for (const sorting of sort) {
+    const [property, direction] = sorting.split(':')
+
+    if (properties[property] === undefined)
+      throw new QuerySyntaxException(`Sort property '${property}' is not defined`)
+
+    result.push([property, direction ?? 'asc'])
+  }
+
+  return result
+}
+
+function projection(projection: string[], properties: Properties): void {
+  for (const property of projection)
+    if (properties[property] === undefined)
+      throw new QuerySyntaxException(`Projection property '${property}' is not defined`)
+
+  for (const property of ['VERSION', 'CREATED', 'UPDATED', 'DELETED'])
+    if (!projection.includes(property)) projection.push(property)
+}

package/{src/query.js → source/query.ts}

@@ -1,28 +1,25 @@
 import { empty } from '@toa.io/generic'
 import * as criteria from './query/criteria.js'
 import * as options from './query/options.js'
+import type { Properties } from './query/criteria.js'
+import type { Node, Options, Query as Parsed } from './types/storages.js'
+import type { Query as Requested } from './types/request.js'
 
 const parse = { ...criteria, ...options }
 
+/** Translates the query a request carries into the one a storage is given. */
 export class Query {
-  #properties
-  #system
+  readonly #properties: Properties
 
-  /** @type {Map<string, object>} parsed criteria by their expression */
-  #asts = new Map()
+  /** parsed criteria by their expression */
+  readonly #asts = new Map<string, Node>()
 
-  constructor (properties) {
+  public constructor(properties: Properties) {
     this.#properties = properties
-    this.#system = Object.keys(properties).filter((key) => properties[key].system === true)
   }
 
-  /**
-   * @param {toa.core.request.Query} query
-   * @returns {toa.core.storages.Query}
-   */
-  parse (query) {
-    /** @type {toa.core.storages.Query} */
-    const result = {}
+  public parse(query: Requested): Parsed {
+    const result: Parsed = {}
     const { id, ids, version, criteria, search, ...rest } = query
 
     const options = this.#options(rest)
@@ -37,10 +34,10 @@ export class Query {
     return result
   }
 
-  #options (options) {
-    if (empty(options)) return
+  #options(given: Record<string, any>): Options | undefined {
+    if (empty(given)) return undefined
 
-    return parse.options(options, this.#properties, this.#system)
+    return parse.options(given, this.#properties)
   }
 
   /**
@@ -52,7 +49,7 @@ export class Query {
    * Expressions come from the client, hence the bound. An invalid one throws before it
    * reaches the cache.
    */
-  #criteria (criteria) {
+  #criteria(criteria: string): Node {
     const known = this.#asts.get(criteria)
 
     if (known !== undefined) return known

package/source/receiver.ts

@@ -0,0 +1,120 @@
+import { console, decode, run, type SpanOptions } from 'openspan'
+import { add } from '@toa.io/generic'
+import { Connector } from './connector.js'
+import type { Component } from './component.js'
+import type { Receiver as Bridge } from './types/bridges.js'
+import type { Message } from './types/message.js'
+import type { Request, Source } from './types/request.js'
+
+export interface Definition {
+  conditioned?: boolean
+  adaptive?: boolean
+  operation: string
+  label?: string
+  destination?: string
+  arguments?: unknown[]
+  origin?: Source
+}
+
+export class Receiver extends Connector {
+  readonly #conditioned: boolean | undefined
+  readonly #adaptive: boolean | undefined
+  readonly #endpoint: string
+  readonly #label: string
+  readonly #destination: string
+  readonly #arguments: unknown[] | undefined
+  readonly #origin: Source | undefined
+  readonly #local: Component
+  readonly #bridge: Bridge | undefined
+  readonly #delivery: SpanOptions
+  readonly #processing: SpanOptions
+
+  public constructor(definition: Definition, local: Component, bridge?: Bridge) {
+    super()
+
+    const { conditioned, adaptive, operation } = definition
+
+    this.#conditioned = conditioned
+    this.#adaptive = adaptive
+    this.#endpoint = operation
+    this.#label = definition.label ?? operation
+    this.#destination = definition.destination ?? this.#label
+    this.#arguments = definition.arguments
+    this.#origin = definition.origin
+
+    this.#local = local
+    this.#bridge = bridge
+
+    this.depends(local)
+    if (bridge !== undefined) this.depends(bridge)
+
+    /*
+     * The delivery span is created on behalf of the messaging destination, so that
+     * each consumer forms its own complete producer/consumer pair, and service graphs
+     * display fan-out correctly: producer -> destination -> each consumer
+     * (Tempo pairs spans one-to-one, thus multiple consumers can not pair
+     * with a single producer span, see grafana/tempo#5408)
+     */
+    this.#delivery = {
+      name: `${this.#label} deliver`,
+      kind: 'producer',
+      service: this.#destination,
+      attributes: { 'messaging.destination.name': this.#destination }
+    }
+
+    this.#processing = {
+      name: `${this.#label} process`,
+      kind: 'consumer',
+      service: local.locator.id,
+      attributes: { 'messaging.destination.name': this.#destination }
+    }
+  }
+
+  /** @hot */
+  public async receive(message: Message): Promise<void> {
+    const { payload, telemetry, ...extensions } = message
+
+    if (this.#conditioned === true && (await this.#bridge?.condition(payload)) === false)
+      return
+
+    const request = await this.#request(payload)
+
+    add(request, extensions)
+
+    // set after `add`, so that a message field can not spoof the origin
+    if (this.#origin !== undefined) request.source = this.#origin
+
+    // continue the trace from the producer span
+    const remote = telemetry === undefined ? null : decode(telemetry)
+    const task = async (): Promise<void> => {
+      await this.#process(request)
+    }
+
+    if (remote === null) await task()
+    else await run(remote, task)
+  }
+
+  async #process(request: Request): Promise<void> {
+    return console.span(this.#delivery, async () =>
+      console.span(this.#processing, async () => {
+        try {
+          await this.#local.invoke(this.#endpoint, request)
+        } catch (error) {
+          console.error('Receiver error', {
+            component: this.#local.locator.id,
+            endpoint: this.#endpoint,
+            error
+          })
+
+          throw error
+        }
+      })
+    )
+  }
+
+  async #request(payload: object): Promise<Request> {
+    return this.#adaptive === true
+      ? await this.#bridge!.request(payload, ...(this.#arguments ?? []))
+      : { input: payload }
+  }
+}

package/source/reflection.ts

@@ -0,0 +1,20 @@
+import { Connector } from './connector.js'
+
+/** What a reflection reads, once, as it connects. */
+export type Source<T = any> = () => Promise<T>
+
+export class Reflection<T = any> extends Connector {
+  public value: T | undefined
+
+  readonly #source: Source<T>
+
+  public constructor(source: Source<T>) {
+    super()
+
+    this.#source = source
+  }
+
+  protected override async open(): Promise<void> {
+    this.value = await this.#source()
+  }
+}

package/source/remote.ts

@@ -0,0 +1,20 @@
+import assert from 'node:assert'
+import { Component } from './component.js'
+import type { Invocable } from './component.js'
+
+/** A call knows what its endpoint declares, which is what an explanation is read from. */
+export interface Explicable extends Invocable {
+  explain(): any
+}
+
+export class Remote extends Component<Explicable> {
+  protected override kind = 'client' as const
+
+  public explain(endpoint: string): any {
+    if (!(endpoint in this.operations))
+      // `assert.fail`, not `assert.ok`: the message is built only when it is needed
+      assert.fail(`Endpoint '${endpoint}' is not provided by '${this.locator.id}'`)
+
+    return this.operations[endpoint].explain()
+  }
+}