@toa.io/core 1.0.0-alpha.292 → 1.0.0-alpha.299

package/source/operation.ts

@@ -1,139 +0,0 @@
-import { Readable } from 'node:stream'
-import { Connector } from './connector.js'
-import { SystemException, RequestContractException } from './exceptions.js'
-import { environment } from '@toa.io/generic'
-import type { Cascade } from './cascade.js'
-import type { State } from './state.js'
-import type { Query as Translator } from './query.js'
-import type { Contract } from './contract/contract.js'
-import type { Entity } from './entities/entity.js'
-import type { EntitySet } from './entities/set.js'
-import type { Changeset } from './entities/changeset.js'
-import type { scope as Scope } from './types/operations.js'
-import type { Query, Request } from './types/request.js'
-
-/** What an operation acquires for the algorithm to run against. */
-export type Scoped = Entity | EntitySet | Changeset | Readable | null
-
-/** What one invocation carries from step to step. */
-export interface Store {
-  request: Request
-  scope?: Scoped
-  state?: any
-  reply?: any
-  /** set by a transition, which is the only operation that runs its steps again */
-  retry?: () => Promise<any>
-}
-
-export interface Contracts {
-  request: Contract
-  reply: Contract
-}
-
-export interface Definition {
-  scope: Scope
-  concurrency?: string
-}
-
-export class Operation extends Connector {
-  public scope: State
-
-  /**
-   * Whether what this operation acquires may be modified and committed. Only a
-   * transition commits, and only a commit needs the pre-image an entity keeps
-   * to diff the new state against.
-   *
-   * @protected
-   */
-  protected mutable: boolean = false
-
-  readonly #cascade: Cascade
-  readonly #contracts: Contracts
-  readonly #query: Translator
-  readonly #scope: Scope
-
-  // eslint-disable-next-line max-params
-  public constructor(
-    cascade: Cascade,
-    scope: State,
-    contracts: Contracts,
-    query: Translator,
-    definition: Definition
-  ) {
-    super()
-
-    this.scope = scope
-
-    this.#cascade = cascade
-    this.#contracts = contracts
-    this.#query = query
-    this.#scope = definition.scope
-
-    this.depends(cascade)
-  }
-
-  public async invoke(request: Request): Promise<any> {
-    try {
-      if (request.authentic !== true) this.#contracts.request.fit(request)
-
-      // the request carries the query onward in its parsed form: what a storage is given,
-      // not what the caller sent
-      if ('query' in request)
-        request.query = this.#query.parse(request.query as Query) as any
-
-      // validate entity
-      if ('entity' in request) this.scope.fit(request.entity)
-
-      const store = { request }
-
-      return await this.process(store)
-    } catch (e) {
-      const exception = e instanceof Error ? new SystemException(e) : e
-
-      return { exception }
-    }
-  }
-
-  protected async process(store: Store): Promise<any> {
-    await this.acquire(store)
-    await this.run(store)
-    await this.commit(store)
-
-    return store.reply
-  }
-
-  protected async acquire(store: Store): Promise<void> {
-    if (this.#scope === 'none') return
-
-    const scope = await this.query(store.request.query)
-    const raw = scope === null || scope instanceof Readable
-
-    store.scope = scope
-    store.state = raw ? scope : (scope as Entity).get()
-  }
-
-  protected async run(store: Store): Promise<void> {
-    const { request, state } = store
-    const reply = await this.#cascade.run(request.input, state)
-
-    // validate reply only on local environments
-    if (environment.get('TOA_ENV') === 'local' && !(reply instanceof Readable))
-      this.#contracts.reply.fit(reply)
-
-    store.reply = reply
-  }
-
-  protected async commit(_store: Store): Promise<void> {}
-
-  protected async query(query?: Query): Promise<Scoped> {
-    if (query === undefined)
-      throw new RequestContractException('Request query is required')
-
-    const acquire = this.scope[this.#scope] as (
-      query: Query,
-      mutable?: boolean
-    ) => Promise<Scoped>
-
-    return acquire.call(this.scope, query, this.mutable)
-  }
-}

package/source/outbox/index.ts

@@ -1 +0,0 @@
-export { Outbox, LANES } from './outbox.js'

package/source/outbox/outbox.ts

@@ -1,341 +0,0 @@
-import { console } from 'openspan'
-import { Connector } from '../connector.js'
-import { newid } from '../entities/newid.js'
-import { environment } from '@toa.io/generic'
-import type { Emission } from '../emission.js'
-import type { Atom } from '../types/atomicity.js'
-import type { Storage } from '../types/storages.js'
-import type { Row } from '../types/outbox.js'
-import type { Event } from '../types/state.js'
-
-export interface Options {
-  interval?: number
-  batch?: number
-  gap?: number
-}
-
-/**
- * Owns the intent to publish. A row is built before the write so that the storage can commit
- * it in the same transaction as the entity; publication then happens off the operation's path,
- * and anything that fails to publish is recovered from the row.
- *
- * The mechanism is a safety net: in a healthy system the row is written, published within
- * milliseconds by the same process, and marked published on that process's next tick.
- *
- * A storage that cannot commit a row atomically has no outbox, and this degrades to the
- * inline emission it replaces.
- */
-export class Outbox extends Connector {
-  readonly #emission: Emission
-  readonly #storage: Storage | undefined
-  readonly #atom: Atom
-
-  readonly #gap: number
-  readonly #interval: number
-  readonly #batch: number
-  readonly #defer: boolean
-
-  /** ids this process has published, held until a cycle marks them */
-  readonly #published = new Set<string>()
-
-  /** in-flight publications, awaited (with a bound) on close */
-  readonly #inflight = new Set<Promise<void>>()
-
-  /** rows this replica is publishing right now, so a cycle does not pick them up again */
-  readonly #publishing = new Set<string>()
-
-  #timer: NodeJS.Timeout | undefined
-  #off: (() => void) | undefined
-  #pumping = false
-  #closing = false
-
-  // eslint-disable-next-line max-params
-  public constructor(
-    emission: Emission,
-    storage: Storage | undefined,
-    atom: Atom,
-    options: Options = {}
-  ) {
-    super()
-
-    this.#emission = emission
-    this.#storage = storage
-    this.#atom = atom
-
-    this.#interval = number('TOA_OUTBOX_INTERVAL', options.interval, INTERVAL)
-    this.#batch = number('TOA_OUTBOX_BATCH', options.batch, BATCH)
-    this.#gap = options.gap ?? this.#interval * K
-    this.#defer = environment.get('TOA_OUTBOX_DEFER') === '1'
-
-    this.depends(emission)
-    this.depends(atom)
-
-    if (storage !== undefined) this.depends(storage)
-  }
-
-  /** whether the storage can commit a row atomically with the entity */
-  public get durable(): boolean {
-    return this.#storage?.outbox !== undefined
-  }
-
-  /**
-   * An assignment's images are the write's own, so it hands over an event with neither, and
-   * the storage fills them in.
-   */
-  public row(event: Partial<Event>): Row {
-    return {
-      id: newid(),
-      lane: this.#lane(),
-      published: false,
-      pending: Date.now() + this.#gap,
-      event: event as Event
-    }
-  }
-
-  /**
-   * Hands a committed row over. Awaited by the caller only on the legacy path — with an
-   * outbox this returns at once and the broker leaves the operation's path.
-   */
-  public publish(row: Row): Promise<void> | void {
-    // without a durable outbox this is the inline path, and the caller awaits the emission
-    if (!this.durable) return this.#emission.emit(row.event)
-
-    /*
-     * A publication started while the pump is closing would outlive the emitters it needs,
-     * and `comq` waits on a connection that is going rather than failing. The row is already
-     * durable, so leaving it is exactly what it is for.
-     */
-    if (
-      this.#closing ||
-      this.#defer ||
-      this.#inflight.size >= INFLIGHT ||
-      this.#published.size >= PUBLISHED
-    )
-      return
-
-    void this.#publish(row)
-  }
-
-  protected override async open(): Promise<void> {
-    if (!this.durable) return
-
-    if (this.#defer)
-      console.warn(
-        'Outbox immediate publication is deferred; events are published by the pump only'
-      )
-
-    this.#timer = setInterval(() => {
-      this.#tick()
-    }, this.#interval)
-    this.#timer.unref()
-
-    /*
-     * A lane changing hands is exactly when rows stranded in it become this replica's to
-     * publish, and the cycle would not notice for up to an interval. Being told costs a cycle
-     * that finds nothing in the usual case, where the claim arrives once and never changes.
-     */
-    this.#off = this.#atom.onassigned(() => {
-      this.#tick()
-    })
-  }
-
-  protected override async close(): Promise<void> {
-    this.#closing = true
-
-    this.#off?.()
-
-    if (this.#timer !== undefined) clearInterval(this.#timer)
-
-    await this.#drain()
-    await this.#mark()
-  }
-
-  /**
-   * Publishes one row and swallows the failure: the row stays unpublished and comes back on a
-   * later cycle, which is the whole point of having written it.
-   *
-   * There is no timeout here on purpose. A publication is a confirmed write to a durable
-   * exchange, and `comq` waits for the broker to come back rather than failing — abandoning
-   * it would not stop it, it would only mean the row is published twice once it lands. What
-   * bounds this instead is the in-flight cap and the drain on close.
-   *
-   */
-  async #publish(row: Row): Promise<void> {
-    this.#publishing.add(row.id)
-
-    const publishing = this.#emission.emit(row.event)
-
-    this.#inflight.add(publishing)
-
-    try {
-      await publishing
-
-      this.#published.add(row.id)
-    } catch (error) {
-      console.warn('Event publication failed', { row: row.id, error })
-    } finally {
-      this.#inflight.delete(publishing)
-      this.#publishing.delete(row.id)
-    }
-  }
-
-  /**
-   * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
-   * unbounded drain outlives any grace period.
-   *
-   */
-  async #drain(): Promise<void> {
-    if (this.#inflight.size === 0) return
-
-    await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)])
-  }
-
-  /**
-   * Reads what is due, publishes it, and marks everything this process has sent — what it just
-   * published and what the immediate path published since the last cycle. One cycle at a time.
-   *
-   */
-  #tick(): void {
-    if (this.#pumping) return
-
-    this.#pumping = true
-
-    void this.#pump().finally(() => (this.#pumping = false))
-  }
-
-  async #pump(): Promise<void> {
-    let page: Row[]
-    let after: string | undefined
-
-    do {
-      page = await this.#read(after)
-
-      if (page.length === 0) break
-
-      // so a page is never read twice
-      after = page[page.length - 1]?.id
-
-      /*
-       * A row is unpublished in the database until a cycle marks it, so a page includes what
-       * this replica is sending right now and what a failed marking left behind. Only this
-       * process knows either.
-       */
-      const rows = page.filter(
-        (row) => !this.#published.has(row.id) && !this.#publishing.has(row.id)
-      )
-
-      if (rows.length > 0) {
-        console.info('Outbox recovering unpublished events', { count: rows.length })
-
-        // every row is given its chance; what the broker refused stays unpublished and comes
-        // back on a later cycle
-        await Promise.allSettled(rows.map(async (row) => this.#publish(row)))
-      }
-
-      // a full page is a page that may have been cut short
-    } while (page.length === this.#batch)
-
-    await this.#mark()
-  }
-
-  /**
-   * One page of what is due. In a healthy system the first one is empty, every cycle — a row is
-   * due only if the process that wrote it failed to publish or died before marking it.
-   *
-   * Reading is suspended, not stopped, while this replica does not know which lanes are its
-   * own: the cycle keeps running and keeps marking, and reading resumes as soon as an
-   * assignment arrives. Reading without an assignment would be a different guarantee, where
-   * every replica publishes every stranded row.
-   *
-   * @param after the last id of the page before, so a page is never read twice
-   */
-  async #read(after?: string): Promise<Row[]> {
-    const lanes = this.#atom.slots(LANES)
-
-    if (lanes === null || lanes.length === 0) return []
-
-    return this.#storage!
-      .outbox!.pending(lanes, Date.now(), this.#batch, after)
-      .catch((error) => {
-        console.warn('Outbox read failed', { error })
-
-        return []
-      })
-  }
-
-  /**
-   * One batched write for many events, which is why the ids are held in memory rather than
-   * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
-   * marked is simply published again, which is within the contract.
-   *
-   */
-  async #mark(): Promise<void> {
-    if (this.#published.size === 0) return
-
-    const ids = [...this.#published]
-
-    try {
-      await this.#storage!.outbox!.settle(ids)
-
-      for (const id of ids) this.#published.delete(id)
-    } catch (error) {
-      console.warn('Outbox marking failed', { count: ids.length, error })
-    }
-  }
-
-  /**
-   * A lane this replica currently owns, so that in steady state it settles its own rows
-   * before it ever reads them. Any lane at all when it owns none: the row still has to be
-   * written, and whoever ends up owning that lane will pump it.
-   *
-   */
-  #lane(): number {
-    const owned = this.#atom.slots(LANES)
-
-    return owned === null || owned.length === 0
-      ? Math.floor(Math.random() * LANES)
-      : owned[Math.floor(Math.random() * owned.length)]
-  }
-}
-
-function number(
-  variable: string,
-  declared: number | undefined,
-  fallback: number
-): number {
-  if (declared !== undefined) return declared
-
-  const value = Number(environment.get(variable))
-
-  return Number.isNaN(value) || value <= 0 ? fallback : value
-}
-
-async function delay(ms: number): Promise<void> {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms).unref()
-  })
-}
-
-/**
- * Constant, never configuration: rows carry their lane, so lowering this would leave rows in
- * lanes nobody reads any more. It is also the ceiling on replicas of one component, and a
- * power of two so that the common replica counts divide evenly.
- */
-export const LANES = 128
-
-/** one cycle reads, publishes and marks; in steady state it finds nothing to read */
-const INTERVAL = 5000
-
-/**
- * `gap = interval * K`. Not a steady-state necessity — a replica writes into a lane it owns
- * and marks what it published — but a guard for when a lane changes hands between the write
- * and the settle. Two cycles of separation, plus one of margin.
- */
-const K = 3
-
-/** how many rows one read brings back; the pump reads on while a page comes back full */
-const BATCH = 200
-
-const DRAIN = 10_000
-const INFLIGHT = 1000
-const PUBLISHED = 10_000

package/source/query/criteria.ts

@@ -1,90 +0,0 @@
-import { parse } from '@rsql/parser'
-import { QuerySyntaxException } from '../exceptions.js'
-import type { Node } from '../types/storages.js'
-
-/** What a component declares about the properties a criteria may select on. */
-export type Properties = Record<string, { type: string }>
-
-export function criteria(expression: string, properties?: Properties): Node {
-  let ast: Node
-
-  try {
-    ast = parse(expression) as unknown as Node
-  } catch (e) {
-    throw new QuerySyntaxException((e as Error).message)
-  }
-
-  if (properties !== undefined) read(ast, properties)
-
-  return ast
-}
-
-function read(node: Node, properties: Properties): void {
-  if (
-    node.type === 'COMPARISON' &&
-    node.left?.type === 'SELECTOR' &&
-    node.right?.type === 'VALUE'
-  ) {
-    const selector = node.left.selector as string
-    const property = properties[selector]
-
-    if (property === undefined) {
-      throw new QuerySyntaxException(`Criteria selector '${selector}' is not defined`)
-    }
-
-    const cast = CAST[property.type]
-
-    // `=in=` and `=out=` carry a list, and casting that as one value gives whatever
-    // a comma-separated string reads as
-    if (cast !== undefined)
-      node.right.value = Array.isArray(node.right.value)
-        ? node.right.value.map((value: string) => cast(value, selector))
-        : cast(node.right.value as string, selector)
-  } else {
-    if (node.left !== undefined) read(node.left, properties)
-    if (node.right !== undefined) read(node.right, properties)
-  }
-}
-
-/**
- * A criteria arrives as text, so a value is read as what the property it selects on holds. What
- * cannot be read as that is refused rather than passed on: `parseInt` answers `NaN` for a word
- * and `12` for `12kg`, and a storage asked to match either finds nothing and says nothing —
- * a mistyped filter reads as an empty result.
- *
- * A string is left as it came, because every text is one: what a criteria may compare a string
- * to is not the type's to say.
- */
-const CAST: Record<string, (value: string, selector: string) => unknown> = {
-  number: (value, selector) => finite(value, selector, 'a number'),
-
-  integer: (value, selector) => {
-    const number = finite(value, selector, 'an integer')
-
-    if (!Number.isInteger(number)) refuse(selector, 'an integer', value)
-
-    return number
-  },
-
-  boolean: (value, selector) => {
-    if (value === 'true') return true
-    if (value === 'false') return false
-
-    return refuse(selector, 'a boolean', value)
-  }
-}
-
-function finite(value: string, selector: string, expected: string): number {
-  // `Number` reads a blank string as zero, and a criteria that says nothing says nothing
-  const number = value.trim() === '' ? Number.NaN : Number(value)
-
-  if (!Number.isFinite(number)) refuse(selector, expected, value)
-
-  return number
-}
-
-function refuse(selector: string, expected: string, value: string): never {
-  throw new QuerySyntaxException(
-    `Criteria selector '${selector}' takes ${expected}, and '${value}' is not one`
-  )
-}

package/source/query/options.ts

@@ -1,35 +0,0 @@
-import { QuerySyntaxException } from '../exceptions.js'
-import type { Options } from '../types/storages.js'
-import type { Properties } from './criteria.js'
-
-export function options(given: Record<string, any>, properties: Properties): Options {
-  if (given.sort !== undefined) given.sort = sort(given.sort, properties)
-
-  if (given.projection !== undefined) projection(given.projection, properties)
-
-  return given as Options
-}
-
-function sort(sort: string[], properties: Properties): Array<[string, string]> {
-  const result: Array<[string, string]> = []
-
-  for (const sorting of sort) {
-    const [property, direction] = sorting.split(':')
-
-    if (properties[property] === undefined)
-      throw new QuerySyntaxException(`Sort property '${property}' is not defined`)
-
-    result.push([property, direction ?? 'asc'])
-  }
-
-  return result
-}
-
-function projection(projection: string[], properties: Properties): void {
-  for (const property of projection)
-    if (properties[property] === undefined)
-      throw new QuerySyntaxException(`Projection property '${property}' is not defined`)
-
-  for (const property of ['VERSION', 'CREATED', 'UPDATED', 'DELETED'])
-    if (!projection.includes(property)) projection.push(property)
-}

package/source/query.ts

@@ -1,67 +0,0 @@
-import { empty } from '@toa.io/generic'
-import * as criteria from './query/criteria.js'
-import * as options from './query/options.js'
-import type { Properties } from './query/criteria.js'
-import type { Node, Options, Query as Parsed } from './types/storages.js'
-import type { Query as Requested } from './types/request.js'
-
-const parse = { ...criteria, ...options }
-
-/** Translates the query a request carries into the one a storage is given. */
-export class Query {
-  readonly #properties: Properties
-
-  /** parsed criteria by their expression */
-  readonly #asts = new Map<string, Node>()
-
-  public constructor(properties: Properties) {
-    this.#properties = properties
-  }
-
-  public parse(query: Requested): Parsed {
-    const result: Parsed = {}
-    const { id, ids, version, criteria, search, ...rest } = query
-
-    const options = this.#options(rest)
-
-    if (id !== undefined) result.id = id
-    if (ids !== undefined) result.ids = ids
-    if (version !== undefined) result.version = version
-    if (criteria !== undefined) result.criteria = this.#criteria(criteria)
-    if (search !== undefined) result.search = search
-    if (options !== undefined) result.options = options
-
-    return result
-  }
-
-  #options(given: Record<string, any>): Options | undefined {
-    if (empty(given)) return undefined
-
-    return parse.options(given, this.#properties)
-  }
-
-  /**
-   * Lexing an RSQL expression is the most expensive thing this class does, and the same
-   * expression arrives over and over: a route builds it from its own declaration and the
-   * request parameters. The tree is only read downstream — a storage builds a fresh query
-   * from it — so it is kept rather than parsed again.
-   *
-   * Expressions come from the client, hence the bound. An invalid one throws before it
-   * reaches the cache.
-   */
-  #criteria(criteria: string): Node {
-    const known = this.#asts.get(criteria)
-
-    if (known !== undefined) return known
-
-    const ast = parse.criteria(criteria, this.#properties)
-
-    if (this.#asts.size >= LIMIT) this.#asts.clear()
-
-    this.#asts.set(criteria, ast)
-
-    return ast
-  }
-}
-
-const LIMIT = 1024