@toa.io/core 1.0.0-alpha.292 → 1.0.0-alpha.293

package/source/outbox/outbox.ts

@@ -1,341 +0,0 @@
-import { console } from 'openspan'
-import { Connector } from '../connector.js'
-import { newid } from '../entities/newid.js'
-import { environment } from '@toa.io/generic'
-import type { Emission } from '../emission.js'
-import type { Atom } from '../types/atomicity.js'
-import type { Storage } from '../types/storages.js'
-import type { Row } from '../types/outbox.js'
-import type { Event } from '../types/state.js'
-
-export interface Options {
-  interval?: number
-  batch?: number
-  gap?: number
-}
-
-/**
- * Owns the intent to publish. A row is built before the write so that the storage can commit
- * it in the same transaction as the entity; publication then happens off the operation's path,
- * and anything that fails to publish is recovered from the row.
- *
- * The mechanism is a safety net: in a healthy system the row is written, published within
- * milliseconds by the same process, and marked published on that process's next tick.
- *
- * A storage that cannot commit a row atomically has no outbox, and this degrades to the
- * inline emission it replaces.
- */
-export class Outbox extends Connector {
-  readonly #emission: Emission
-  readonly #storage: Storage | undefined
-  readonly #atom: Atom
-
-  readonly #gap: number
-  readonly #interval: number
-  readonly #batch: number
-  readonly #defer: boolean
-
-  /** ids this process has published, held until a cycle marks them */
-  readonly #published = new Set<string>()
-
-  /** in-flight publications, awaited (with a bound) on close */
-  readonly #inflight = new Set<Promise<void>>()
-
-  /** rows this replica is publishing right now, so a cycle does not pick them up again */
-  readonly #publishing = new Set<string>()
-
-  #timer: NodeJS.Timeout | undefined
-  #off: (() => void) | undefined
-  #pumping = false
-  #closing = false
-
-  // eslint-disable-next-line max-params
-  public constructor(
-    emission: Emission,
-    storage: Storage | undefined,
-    atom: Atom,
-    options: Options = {}
-  ) {
-    super()
-
-    this.#emission = emission
-    this.#storage = storage
-    this.#atom = atom
-
-    this.#interval = number('TOA_OUTBOX_INTERVAL', options.interval, INTERVAL)
-    this.#batch = number('TOA_OUTBOX_BATCH', options.batch, BATCH)
-    this.#gap = options.gap ?? this.#interval * K
-    this.#defer = environment.get('TOA_OUTBOX_DEFER') === '1'
-
-    this.depends(emission)
-    this.depends(atom)
-
-    if (storage !== undefined) this.depends(storage)
-  }
-
-  /** whether the storage can commit a row atomically with the entity */
-  public get durable(): boolean {
-    return this.#storage?.outbox !== undefined
-  }
-
-  /**
-   * An assignment's images are the write's own, so it hands over an event with neither, and
-   * the storage fills them in.
-   */
-  public row(event: Partial<Event>): Row {
-    return {
-      id: newid(),
-      lane: this.#lane(),
-      published: false,
-      pending: Date.now() + this.#gap,
-      event: event as Event
-    }
-  }
-
-  /**
-   * Hands a committed row over. Awaited by the caller only on the legacy path — with an
-   * outbox this returns at once and the broker leaves the operation's path.
-   */
-  public publish(row: Row): Promise<void> | void {
-    // without a durable outbox this is the inline path, and the caller awaits the emission
-    if (!this.durable) return this.#emission.emit(row.event)
-
-    /*
-     * A publication started while the pump is closing would outlive the emitters it needs,
-     * and `comq` waits on a connection that is going rather than failing. The row is already
-     * durable, so leaving it is exactly what it is for.
-     */
-    if (
-      this.#closing ||
-      this.#defer ||
-      this.#inflight.size >= INFLIGHT ||
-      this.#published.size >= PUBLISHED
-    )
-      return
-
-    void this.#publish(row)
-  }
-
-  protected override async open(): Promise<void> {
-    if (!this.durable) return
-
-    if (this.#defer)
-      console.warn(
-        'Outbox immediate publication is deferred; events are published by the pump only'
-      )
-
-    this.#timer = setInterval(() => {
-      this.#tick()
-    }, this.#interval)
-    this.#timer.unref()
-
-    /*
-     * A lane changing hands is exactly when rows stranded in it become this replica's to
-     * publish, and the cycle would not notice for up to an interval. Being told costs a cycle
-     * that finds nothing in the usual case, where the claim arrives once and never changes.
-     */
-    this.#off = this.#atom.onassigned(() => {
-      this.#tick()
-    })
-  }
-
-  protected override async close(): Promise<void> {
-    this.#closing = true
-
-    this.#off?.()
-
-    if (this.#timer !== undefined) clearInterval(this.#timer)
-
-    await this.#drain()
-    await this.#mark()
-  }
-
-  /**
-   * Publishes one row and swallows the failure: the row stays unpublished and comes back on a
-   * later cycle, which is the whole point of having written it.
-   *
-   * There is no timeout here on purpose. A publication is a confirmed write to a durable
-   * exchange, and `comq` waits for the broker to come back rather than failing — abandoning
-   * it would not stop it, it would only mean the row is published twice once it lands. What
-   * bounds this instead is the in-flight cap and the drain on close.
-   *
-   */
-  async #publish(row: Row): Promise<void> {
-    this.#publishing.add(row.id)
-
-    const publishing = this.#emission.emit(row.event)
-
-    this.#inflight.add(publishing)
-
-    try {
-      await publishing
-
-      this.#published.add(row.id)
-    } catch (error) {
-      console.warn('Event publication failed', { row: row.id, error })
-    } finally {
-      this.#inflight.delete(publishing)
-      this.#publishing.delete(row.id)
-    }
-  }
-
-  /**
-   * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
-   * unbounded drain outlives any grace period.
-   *
-   */
-  async #drain(): Promise<void> {
-    if (this.#inflight.size === 0) return
-
-    await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)])
-  }
-
-  /**
-   * Reads what is due, publishes it, and marks everything this process has sent — what it just
-   * published and what the immediate path published since the last cycle. One cycle at a time.
-   *
-   */
-  #tick(): void {
-    if (this.#pumping) return
-
-    this.#pumping = true
-
-    void this.#pump().finally(() => (this.#pumping = false))
-  }
-
-  async #pump(): Promise<void> {
-    let page: Row[]
-    let after: string | undefined
-
-    do {
-      page = await this.#read(after)
-
-      if (page.length === 0) break
-
-      // so a page is never read twice
-      after = page[page.length - 1]?.id
-
-      /*
-       * A row is unpublished in the database until a cycle marks it, so a page includes what
-       * this replica is sending right now and what a failed marking left behind. Only this
-       * process knows either.
-       */
-      const rows = page.filter(
-        (row) => !this.#published.has(row.id) && !this.#publishing.has(row.id)
-      )
-
-      if (rows.length > 0) {
-        console.info('Outbox recovering unpublished events', { count: rows.length })
-
-        // every row is given its chance; what the broker refused stays unpublished and comes
-        // back on a later cycle
-        await Promise.allSettled(rows.map(async (row) => this.#publish(row)))
-      }
-
-      // a full page is a page that may have been cut short
-    } while (page.length === this.#batch)
-
-    await this.#mark()
-  }
-
-  /**
-   * One page of what is due. In a healthy system the first one is empty, every cycle — a row is
-   * due only if the process that wrote it failed to publish or died before marking it.
-   *
-   * Reading is suspended, not stopped, while this replica does not know which lanes are its
-   * own: the cycle keeps running and keeps marking, and reading resumes as soon as an
-   * assignment arrives. Reading without an assignment would be a different guarantee, where
-   * every replica publishes every stranded row.
-   *
-   * @param after the last id of the page before, so a page is never read twice
-   */
-  async #read(after?: string): Promise<Row[]> {
-    const lanes = this.#atom.slots(LANES)
-
-    if (lanes === null || lanes.length === 0) return []
-
-    return this.#storage!
-      .outbox!.pending(lanes, Date.now(), this.#batch, after)
-      .catch((error) => {
-        console.warn('Outbox read failed', { error })
-
-        return []
-      })
-  }
-
-  /**
-   * One batched write for many events, which is why the ids are held in memory rather than
-   * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
-   * marked is simply published again, which is within the contract.
-   *
-   */
-  async #mark(): Promise<void> {
-    if (this.#published.size === 0) return
-
-    const ids = [...this.#published]
-
-    try {
-      await this.#storage!.outbox!.settle(ids)
-
-      for (const id of ids) this.#published.delete(id)
-    } catch (error) {
-      console.warn('Outbox marking failed', { count: ids.length, error })
-    }
-  }
-
-  /**
-   * A lane this replica currently owns, so that in steady state it settles its own rows
-   * before it ever reads them. Any lane at all when it owns none: the row still has to be
-   * written, and whoever ends up owning that lane will pump it.
-   *
-   */
-  #lane(): number {
-    const owned = this.#atom.slots(LANES)
-
-    return owned === null || owned.length === 0
-      ? Math.floor(Math.random() * LANES)
-      : owned[Math.floor(Math.random() * owned.length)]
-  }
-}
-
-function number(
-  variable: string,
-  declared: number | undefined,
-  fallback: number
-): number {
-  if (declared !== undefined) return declared
-
-  const value = Number(environment.get(variable))
-
-  return Number.isNaN(value) || value <= 0 ? fallback : value
-}
-
-async function delay(ms: number): Promise<void> {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms).unref()
-  })
-}
-
-/**
- * Constant, never configuration: rows carry their lane, so lowering this would leave rows in
- * lanes nobody reads any more. It is also the ceiling on replicas of one component, and a
- * power of two so that the common replica counts divide evenly.
- */
-export const LANES = 128
-
-/** one cycle reads, publishes and marks; in steady state it finds nothing to read */
-const INTERVAL = 5000
-
-/**
- * `gap = interval * K`. Not a steady-state necessity — a replica writes into a lane it owns
- * and marks what it published — but a guard for when a lane changes hands between the write
- * and the settle. Two cycles of separation, plus one of margin.
- */
-const K = 3
-
-/** how many rows one read brings back; the pump reads on while a page comes back full */
-const BATCH = 200
-
-const DRAIN = 10_000
-const INFLIGHT = 1000
-const PUBLISHED = 10_000

package/source/query/criteria.ts

@@ -1,90 +0,0 @@
-import { parse } from '@rsql/parser'
-import { QuerySyntaxException } from '../exceptions.js'
-import type { Node } from '../types/storages.js'
-
-/** What a component declares about the properties a criteria may select on. */
-export type Properties = Record<string, { type: string }>
-
-export function criteria(expression: string, properties?: Properties): Node {
-  let ast: Node
-
-  try {
-    ast = parse(expression) as unknown as Node
-  } catch (e) {
-    throw new QuerySyntaxException((e as Error).message)
-  }
-
-  if (properties !== undefined) read(ast, properties)
-
-  return ast
-}
-
-function read(node: Node, properties: Properties): void {
-  if (
-    node.type === 'COMPARISON' &&
-    node.left?.type === 'SELECTOR' &&
-    node.right?.type === 'VALUE'
-  ) {
-    const selector = node.left.selector as string
-    const property = properties[selector]
-
-    if (property === undefined) {
-      throw new QuerySyntaxException(`Criteria selector '${selector}' is not defined`)
-    }
-
-    const cast = CAST[property.type]
-
-    // `=in=` and `=out=` carry a list, and casting that as one value gives whatever
-    // a comma-separated string reads as
-    if (cast !== undefined)
-      node.right.value = Array.isArray(node.right.value)
-        ? node.right.value.map((value: string) => cast(value, selector))
-        : cast(node.right.value as string, selector)
-  } else {
-    if (node.left !== undefined) read(node.left, properties)
-    if (node.right !== undefined) read(node.right, properties)
-  }
-}
-
-/**
- * A criteria arrives as text, so a value is read as what the property it selects on holds. What
- * cannot be read as that is refused rather than passed on: `parseInt` answers `NaN` for a word
- * and `12` for `12kg`, and a storage asked to match either finds nothing and says nothing —
- * a mistyped filter reads as an empty result.
- *
- * A string is left as it came, because every text is one: what a criteria may compare a string
- * to is not the type's to say.
- */
-const CAST: Record<string, (value: string, selector: string) => unknown> = {
-  number: (value, selector) => finite(value, selector, 'a number'),
-
-  integer: (value, selector) => {
-    const number = finite(value, selector, 'an integer')
-
-    if (!Number.isInteger(number)) refuse(selector, 'an integer', value)
-
-    return number
-  },
-
-  boolean: (value, selector) => {
-    if (value === 'true') return true
-    if (value === 'false') return false
-
-    return refuse(selector, 'a boolean', value)
-  }
-}
-
-function finite(value: string, selector: string, expected: string): number {
-  // `Number` reads a blank string as zero, and a criteria that says nothing says nothing
-  const number = value.trim() === '' ? Number.NaN : Number(value)
-
-  if (!Number.isFinite(number)) refuse(selector, expected, value)
-
-  return number
-}
-
-function refuse(selector: string, expected: string, value: string): never {
-  throw new QuerySyntaxException(
-    `Criteria selector '${selector}' takes ${expected}, and '${value}' is not one`
-  )
-}

package/source/query/options.ts

@@ -1,35 +0,0 @@
-import { QuerySyntaxException } from '../exceptions.js'
-import type { Options } from '../types/storages.js'
-import type { Properties } from './criteria.js'
-
-export function options(given: Record<string, any>, properties: Properties): Options {
-  if (given.sort !== undefined) given.sort = sort(given.sort, properties)
-
-  if (given.projection !== undefined) projection(given.projection, properties)
-
-  return given as Options
-}
-
-function sort(sort: string[], properties: Properties): Array<[string, string]> {
-  const result: Array<[string, string]> = []
-
-  for (const sorting of sort) {
-    const [property, direction] = sorting.split(':')
-
-    if (properties[property] === undefined)
-      throw new QuerySyntaxException(`Sort property '${property}' is not defined`)
-
-    result.push([property, direction ?? 'asc'])
-  }
-
-  return result
-}
-
-function projection(projection: string[], properties: Properties): void {
-  for (const property of projection)
-    if (properties[property] === undefined)
-      throw new QuerySyntaxException(`Projection property '${property}' is not defined`)
-
-  for (const property of ['VERSION', 'CREATED', 'UPDATED', 'DELETED'])
-    if (!projection.includes(property)) projection.push(property)
-}

package/source/query.ts

@@ -1,67 +0,0 @@
-import { empty } from '@toa.io/generic'
-import * as criteria from './query/criteria.js'
-import * as options from './query/options.js'
-import type { Properties } from './query/criteria.js'
-import type { Node, Options, Query as Parsed } from './types/storages.js'
-import type { Query as Requested } from './types/request.js'
-
-const parse = { ...criteria, ...options }
-
-/** Translates the query a request carries into the one a storage is given. */
-export class Query {
-  readonly #properties: Properties
-
-  /** parsed criteria by their expression */
-  readonly #asts = new Map<string, Node>()
-
-  public constructor(properties: Properties) {
-    this.#properties = properties
-  }
-
-  public parse(query: Requested): Parsed {
-    const result: Parsed = {}
-    const { id, ids, version, criteria, search, ...rest } = query
-
-    const options = this.#options(rest)
-
-    if (id !== undefined) result.id = id
-    if (ids !== undefined) result.ids = ids
-    if (version !== undefined) result.version = version
-    if (criteria !== undefined) result.criteria = this.#criteria(criteria)
-    if (search !== undefined) result.search = search
-    if (options !== undefined) result.options = options
-
-    return result
-  }
-
-  #options(given: Record<string, any>): Options | undefined {
-    if (empty(given)) return undefined
-
-    return parse.options(given, this.#properties)
-  }
-
-  /**
-   * Lexing an RSQL expression is the most expensive thing this class does, and the same
-   * expression arrives over and over: a route builds it from its own declaration and the
-   * request parameters. The tree is only read downstream — a storage builds a fresh query
-   * from it — so it is kept rather than parsed again.
-   *
-   * Expressions come from the client, hence the bound. An invalid one throws before it
-   * reaches the cache.
-   */
-  #criteria(criteria: string): Node {
-    const known = this.#asts.get(criteria)
-
-    if (known !== undefined) return known
-
-    const ast = parse.criteria(criteria, this.#properties)
-
-    if (this.#asts.size >= LIMIT) this.#asts.clear()
-
-    this.#asts.set(criteria, ast)
-
-    return ast
-  }
-}
-
-const LIMIT = 1024

package/source/receiver.ts

@@ -1,120 +0,0 @@
-import { console, decode, run, type SpanOptions } from 'openspan'
-import { add } from '@toa.io/generic'
-import { Connector } from './connector.js'
-import type { Component } from './component.js'
-import type { Receiver as Bridge } from './types/bridges.js'
-import type { Message } from './types/message.js'
-import type { Request, Source } from './types/request.js'
-
-export interface Definition {
-  conditioned?: boolean
-  adaptive?: boolean
-  operation: string
-  label?: string
-  destination?: string
-  arguments?: unknown[]
-  origin?: Source
-}
-
-export class Receiver extends Connector {
-  readonly #conditioned: boolean | undefined
-  readonly #adaptive: boolean | undefined
-  readonly #endpoint: string
-  readonly #label: string
-  readonly #destination: string
-  readonly #arguments: unknown[] | undefined
-  readonly #origin: Source | undefined
-  readonly #local: Component
-  readonly #bridge: Bridge | undefined
-  readonly #delivery: SpanOptions
-  readonly #processing: SpanOptions
-
-  public constructor(definition: Definition, local: Component, bridge?: Bridge) {
-    super()
-
-    const { conditioned, adaptive, operation } = definition
-
-    this.#conditioned = conditioned
-    this.#adaptive = adaptive
-    this.#endpoint = operation
-    this.#label = definition.label ?? operation
-    this.#destination = definition.destination ?? this.#label
-    this.#arguments = definition.arguments
-    this.#origin = definition.origin
-
-    this.#local = local
-    this.#bridge = bridge
-
-    this.depends(local)
-    if (bridge !== undefined) this.depends(bridge)
-
-    /*
-     * The delivery span is created on behalf of the messaging destination, so that
-     * each consumer forms its own complete producer/consumer pair, and service graphs
-     * display fan-out correctly: producer -> destination -> each consumer
-     * (Tempo pairs spans one-to-one, thus multiple consumers can not pair
-     * with a single producer span, see grafana/tempo#5408)
-     */
-    this.#delivery = {
-      name: `${this.#label} deliver`,
-      kind: 'producer',
-      service: this.#destination,
-      attributes: { 'messaging.destination.name': this.#destination }
-    }
-
-    this.#processing = {
-      name: `${this.#label} process`,
-      kind: 'consumer',
-      service: local.locator.id,
-      attributes: { 'messaging.destination.name': this.#destination }
-    }
-  }
-
-  /** @hot */
-  public async receive(message: Message): Promise<void> {
-    const { payload, telemetry, ...extensions } = message
-
-    if (this.#conditioned === true && (await this.#bridge?.condition(payload)) === false)
-      return
-
-    const request = await this.#request(payload)
-
-    add(request, extensions)
-
-    // set after `add`, so that a message field can not spoof the origin
-    if (this.#origin !== undefined) request.source = this.#origin
-
-    // continue the trace from the producer span
-    const remote = telemetry === undefined ? null : decode(telemetry)
-    const task = async (): Promise<void> => {
-      await this.#process(request)
-    }
-
-    if (remote === null) await task()
-    else await run(remote, task)
-  }
-
-  async #process(request: Request): Promise<void> {
-    return console.span(this.#delivery, async () =>
-      console.span(this.#processing, async () => {
-        try {
-          await this.#local.invoke(this.#endpoint, request)
-        } catch (error) {
-          console.error('Receiver error', {
-            component: this.#local.locator.id,
-            endpoint: this.#endpoint,
-            error
-          })
-
-          throw error
-        }
-      })
-    )
-  }
-
-  async #request(payload: object): Promise<Request> {
-    return this.#adaptive === true
-      ? await this.#bridge!.request(payload, ...(this.#arguments ?? []))
-      : { input: payload }
-  }
-}

package/source/reflection.ts

@@ -1,20 +0,0 @@
-import { Connector } from './connector.js'
-
-/** What a reflection reads, once, as it connects. */
-export type Source<T = any> = () => Promise<T>
-
-export class Reflection<T = any> extends Connector {
-  public value: T | undefined
-
-  readonly #source: Source<T>
-
-  public constructor(source: Source<T>) {
-    super()
-
-    this.#source = source
-  }
-
-  protected override async open(): Promise<void> {
-    this.value = await this.#source()
-  }
-}

package/source/remote.ts

@@ -1,20 +0,0 @@
-import assert from 'node:assert'
-import { Component } from './component.js'
-import type { Invocable } from './component.js'
-
-/** A call knows what its endpoint declares, which is what an explanation is read from. */
-export interface Explicable extends Invocable {
-  explain(): any
-}
-
-export class Remote extends Component<Explicable> {
-  protected override kind = 'client' as const
-
-  public explain(endpoint: string): any {
-    if (!(endpoint in this.operations))
-      // `assert.fail`, not `assert.ok`: the message is built only when it is needed
-      assert.fail(`Endpoint '${endpoint}' is not provided by '${this.locator.id}'`)
-
-    return this.operations[endpoint].explain()
-  }
-}