@toa.io/core 1.0.0-alpha.28 → 1.0.0-alpha.282

package/src/exceptions.js

@@ -1,10 +1,7 @@
-'use strict'
+import { swap } from '@toa.io/generic'
 
-const { swap } = require('@toa.io/generic')
-
-const codes = {
+export const codes = {
   System: 0,
-  NotImplemented: 10,
 
   Contract: 200,
   RequestSyntax: 201,
@@ -12,6 +9,7 @@ const codes = {
   RequestConflict: 203,
   ResponseContract: 211,
   EntityContract: 212,
+  EntityGuard: 213,
   QuerySyntax: 221,
 
   State: 300,
@@ -28,17 +26,20 @@ const codes = {
 /**
  * @implements {toa.core.Exception}
  */
-class Exception {
+export class Exception {
   code
   message
 
-  constructor (code, message) {
+  constructor (code, message, cause) {
     this.code = code
     this.message = message
+
+    if (cause !== undefined)
+      this.cause = cause
   }
 }
 
-class SystemException extends Exception {
+export class SystemException extends Exception {
   stack
 
   constructor (error) {
@@ -48,53 +49,66 @@ class SystemException extends Exception {
   }
 }
 
-class ContractException extends Exception {
-  keyword
-  property
-  schema
-  path
-
-  constructor (code, error) {
-    super(code || codes.Contract, error.message)
+export class ContractException extends Exception {
+  constructor (code, error, cause) {
+    super(code || codes.Contract, typeof error === 'string' ? error : error?.message, cause)
 
-    this.keyword = error.keyword
-    this.property = error.property
-    this.schema = error.schema
-    this.path = error.path
+    if (typeof error === 'object' && error !== null)
+      for (const k of ['keyword', 'property', 'schema', 'path', 'params'])
+        if (k in error)
+          this[k] = error[k]
   }
 }
 
-class RequestContractException extends ContractException {
-  constructor (error) { super(codes.RequestContract, error) }
+export class RequestContractException extends ContractException {
+  constructor (error, cause) { super(codes.RequestContract, error, cause) }
 }
 
-class ResponseContractException extends ContractException {
-  constructor (error) { super(codes.ResponseContract, error) }
+export class ResponseContractException extends ContractException {
+  constructor (error, cause) { super(codes.ResponseContract, error, cause) }
 }
 
-class EntityContractException extends ContractException {
-  constructor (error) { super(codes.EntityContract, error) }
+export class EntityContractException extends ContractException {
+  constructor (error, cause) { super(codes.EntityContract, error, cause) }
+}
+
+export class EntityGuardException extends ContractException {
+  constructor (name, cause) { super(codes.EntityGuard, name, cause) }
 }
 
 // #region exports
-exports.Exception = Exception
-exports.SystemException = SystemException
-exports.RequestContractException = RequestContractException
-exports.ResponseContractException = ResponseContractException
-exports.EntityContractException = EntityContractException
 
-for (const [name, code] of Object.entries(codes)) {
+
+
+
+
+
+
+// a module's exports are static, so the ones that follow a code are named rather
+// than generated onto the namespace
+function derive (name) {
   const classname = name + 'Exception'
 
-  if (exports[classname] === undefined) {
-    exports[classname] = class extends Exception {
-      constructor (message) {
-        super(code, message ?? classname)
-      }
+  return class extends Exception {
+    constructor (message, cause) {
+      super(codes[name], message ? `${classname}: ${message}` : classname, cause)
     }
+
+    static name = classname
   }
 }
 
-exports.codes = codes
-exports.names = swap(codes)
+export const RequestSyntaxException = derive('RequestSyntax')
+export const RequestConflictException = derive('RequestConflict')
+export const QuerySyntaxException = derive('QuerySyntax')
+export const StateException = derive('State')
+export const StateNotFoundException = derive('StateNotFound')
+export const StatePreconditionException = derive('StatePrecondition')
+export const StateConcurrencyException = derive('StateConcurrency')
+export const StateInitializationException = derive('StateInitialization')
+export const DuplicateException = derive('Duplicate')
+export const CommunicationException = derive('Communication')
+export const TransmissionException = derive('Transmission')
+
+export const names = swap(codes)
 // #endregion

package/src/exposition.js

@@ -1,8 +1,6 @@
-'use strict'
+import { Connector } from './connector.js'
 
-const { Connector } = require('./connector')
-
-class Exposition extends Connector {
+export class Exposition extends Connector {
   locator
 
   #exposition
@@ -20,8 +18,7 @@ class Exposition extends Connector {
 }
 
 const expose = (manifest) => {
-  const { namespace, name, operations, events, entity } = manifest
-  return { namespace, name, operations, events, entity }
-}
+  const { namespace, name, entity, operations, events } = manifest
 
-exports.Exposition = Exposition
+  return { namespace, name, entity, operations, events }
+}

package/src/guard.js

@@ -0,0 +1,13 @@
+export class Guard {
+  name
+  #bridge
+
+  constructor (name, bridge) {
+    this.name = name
+    this.#bridge = bridge
+  }
+
+  fit (state, origin) {
+    return this.#bridge.fit(state, origin)
+  }
+}

package/src/index.js

@@ -1,47 +1,28 @@
-const { Assignment } = require('./assignment')
-const { Call } = require('./call')
-const { Cascade } = require('./cascade')
-const { Composition } = require('./composition')
-const { Connector } = require('./connector')
-const { Context } = require('./context')
-const { Discovery } = require('./discovery')
-const { Emission } = require('./emission')
-const { Event } = require('./event')
-const { Exposition } = require('./exposition')
-const { Locator } = require('./locator')
-const { Observation } = require('./observation')
-const { Operation } = require('./operation')
-const { Query } = require('./query')
-const { Receiver } = require('./receiver')
-const { Reflection } = require('./reflection')
-const { Remote } = require('./remote')
-const { Component } = require('./component')
-const { State } = require('./state')
-const { Transition } = require('./transition')
-const { Transmission } = require('./transmission')
-
-exports.entities = require('./entities')
-exports.exceptions = require('./exceptions')
-exports.contract = require('./contract')
-
-exports.Assignment = Assignment
-exports.Call = Call
-exports.Cascade = Cascade
-exports.Component = Component
-exports.Composition = Composition
-exports.Connector = Connector
-exports.Context = Context
-exports.Discovery = Discovery
-exports.Emission = Emission
-exports.Event = Event
-exports.Exposition = Exposition
-exports.Locator = Locator
-exports.Observation = Observation
-exports.Operation = Operation
-exports.Query = Query
-exports.Receiver = Receiver
-exports.Reflection = Reflection
-exports.Remote = Remote
-exports.State = State
-exports.Transition = Transition
-exports.Transmission = Transmission
+export { Assignment } from './assignment.js'
+export { Call } from './call.js'
+export { Cascade } from './cascade.js'
+export { Component } from './component.js'
+export { Composition } from './composition.js'
+export { Connector } from './connector.js'
+export { Context } from './context.js'
+export { Discovery } from './discovery.js'
+export { Effect } from './effect.js'
+export { Emission } from './emission.js'
+export { Event } from './event.js'
+export { Exposition } from './exposition.js'
+export { Locator } from './locator.js'
+export { Observation } from './observation.js'
+export { Outbox } from './outbox/index.js'
+export { Operation } from './operation.js'
+export { Query } from './query.js'
+export { Receiver } from './receiver.js'
+export { Reflection } from './reflection.js'
+export { Remote } from './remote.js'
+export { State } from './state.js'
+export { Transition } from './transition.js'
+export { Transmission } from './transmission.js'
+export { Unmanaged } from './unmanaged.js'
+export { Guard } from './guard.js'
+export * as entities from './entities/index.js'
+export * as exceptions from './exceptions.js'
+export * as contract from './contract/index.js'

package/src/locator.js

@@ -1,11 +1,9 @@
-'use strict'
-
-const { concat } = require('@toa.io/generic')
+import { concat } from '@toa.io/generic'
 
 /**
  * @implements {toa.core.Locator}
  */
-class Locator {
+export class Locator {
   name
   namespace
 
@@ -50,5 +48,3 @@ class Locator {
 }
 
 const DOT = '.'
-
-exports.Locator = Locator

package/src/observation.js

@@ -1,20 +1,8 @@
-'use strict'
-
-const { Operation } = require('./operation')
-
-class Observation extends Operation {
-  async acquire (store) {
-    const scope = await this.query(store.request.query)
-    const state = scope === null ? null : scope.get()
-
-    store.scope = scope
-    store.state = state
-  }
+import { Operation } from './operation.js'
 
+export class Observation extends Operation {
   async run (store) {
-    if (store.scope === null) store.reply = null
+    if (store.scope === null || (store.scope?.deleted === true && store.request.query?.options?.deleted !== true)) store.reply = null
     else await super.run(store)
   }
 }
-
-exports.Observation = Observation

package/src/operation.js

@@ -1,11 +1,19 @@
-'use strict'
+import { Connector } from './connector.js'
+import { SystemException, RequestContractException } from './exceptions.js'
+import { Readable } from 'node:stream'
 
-const { Connector } = require('./connector')
-const { SystemException } = require('./exceptions')
-
-class Operation extends Connector {
+export class Operation extends Connector {
   scope
 
+  /**
+   * Whether what this operation acquires may be modified and committed. Only a
+   * transition commits, and only a commit needs the pre-image an entity keeps
+   * to diff the new state against.
+   *
+   * @protected
+   */
+  mutable = false
+
   #cascade
   #contracts
   #query
@@ -26,8 +34,15 @@ class Operation extends Connector {
 
   async invoke (request) {
     try {
-      if (request.authentic !== true) this.#contracts.request.fit(request)
-      if ('query' in request) request.query = this.#query.parse(request.query)
+      if (request.authentic !== true)
+        this.#contracts.request.fit(request)
+
+      if ('query' in request)
+        request.query = this.#query.parse(request.query)
+
+      // validate entity
+      if ('entity' in request)
+        this.scope.fit(request.entity)
 
       const store = { request }
 
@@ -47,14 +62,24 @@ class Operation extends Connector {
     return store.reply
   }
 
-  async acquire () {}
+  async acquire (store) {
+    if (this.#scope === 'none')
+      return
+
+    const scope = await this.query(store.request.query)
+    const raw = scope === null || scope instanceof Readable
+
+    store.scope = scope
+    store.state = raw ? scope : scope.get()
+  }
 
   async run (store) {
     const { request, state } = store
-    // noinspection UnnecessaryLocalVariableJS
-    const reply = await this.#cascade.run(request.input, state) || {}
+    const reply = await this.#cascade.run(request.input, state)
 
-    // this.#contracts.reply.fit(reply)
+    // validate reply only on local environments
+    if (process.env.TOA_ENV === 'local' && !(reply instanceof Readable))
+      this.#contracts.reply.fit(reply)
 
     store.reply = reply
   }
@@ -62,8 +87,9 @@ class Operation extends Connector {
   async commit () {}
 
   async query (query) {
-    return this.scope[this.#scope](query)
+    if (query === undefined)
+      throw new RequestContractException('Request query is required')
+
+    return this.scope[this.#scope](query, this.mutable)
   }
 }
-
-exports.Operation = Operation

package/src/outbox/index.js

@@ -0,0 +1 @@
+export { Outbox, LANES } from './outbox.js'

package/src/outbox/outbox.js

@@ -0,0 +1,296 @@
+import { console } from 'openspan'
+import { Connector } from '../connector.js'
+import { newid } from '../entities/newid.js'
+
+/**
+ * Owns the intent to publish. A row is built before the write so that the storage can commit
+ * it in the same transaction as the entity; publication then happens off the operation's path,
+ * and anything that fails to publish is recovered from the row.
+ *
+ * The mechanism is a safety net: in a healthy system the row is written, published within
+ * milliseconds by the same process, and marked published on that process's next tick.
+ *
+ * A storage that cannot commit a row atomically has no outbox, and this degrades to the
+ * inline emission it replaces.
+ */
+export class Outbox extends Connector {
+  #emission
+  #storage
+  #atom
+
+  #gap
+  #interval
+  #batch
+  #defer
+
+  /** ids this process has published, held until a cycle marks them */
+  #published = new Set()
+
+  /** in-flight publications, awaited (with a bound) on close */
+  #inflight = new Set()
+
+  /** rows this replica is publishing right now, so a cycle does not pick them up again */
+  #publishing = new Set()
+
+  #timer
+  #pumping = false
+  #closing = false
+
+  constructor (emission, storage, atom, options = {}) {
+    super()
+
+    this.#emission = emission
+    this.#storage = storage
+    this.#atom = atom
+
+    this.#interval = number('TOA_OUTBOX_INTERVAL', options.interval, INTERVAL)
+    this.#batch = number('TOA_OUTBOX_BATCH', options.batch, BATCH)
+    this.#gap = options.gap ?? this.#interval * K
+    this.#defer = process.env.TOA_OUTBOX_DEFER === '1'
+
+    this.depends(emission)
+    this.depends(atom)
+
+    if (storage !== undefined) this.depends(storage)
+  }
+
+  /** whether the storage can commit a row atomically with the entity */
+  get durable () {
+    return this.#storage?.outbox !== undefined
+  }
+
+  /**
+   * @param event {toa.core.transition.Event}
+   * @returns {object}
+   */
+  row (event) {
+    return {
+      id: newid(),
+      lane: this.#lane(),
+      published: false,
+      pending: Date.now() + this.#gap,
+      event
+    }
+  }
+
+  /**
+   * Hands a committed row over. Awaited by the caller only on the legacy path — with an
+   * outbox this returns at once and the broker leaves the operation's path.
+   */
+  publish (row) {
+    if (!this.durable)
+      return this.#emission.emit(row.event)
+
+    /*
+     * A publication started while the pump is closing would outlive the emitters it needs,
+     * and `comq` waits on a connection that is going rather than failing. The row is already
+     * durable, so leaving it is exactly what it is for.
+     */
+    if (this.#closing || this.#defer ||
+      this.#inflight.size >= INFLIGHT || this.#published.size >= PUBLISHED)
+      return
+
+    void this.#publish(row)
+  }
+
+  async open () {
+    if (!this.durable) return
+
+    if (this.#defer)
+      console.warn('Outbox immediate publication is deferred; events are published by the pump only')
+
+    this.#timer = setInterval(() => this.#tick(), this.#interval)
+    this.#timer.unref()
+  }
+
+  async close () {
+    this.#closing = true
+
+    if (this.#timer !== undefined) clearInterval(this.#timer)
+
+    await this.#drain()
+    await this.#mark()
+  }
+
+  /**
+   * Publishes one row and swallows the failure: the row stays unpublished and comes back on a
+   * later cycle, which is the whole point of having written it.
+   *
+   * There is no timeout here on purpose. A publication is a confirmed write to a durable
+   * exchange, and `comq` waits for the broker to come back rather than failing — abandoning
+   * it would not stop it, it would only mean the row is published twice once it lands. What
+   * bounds this instead is the in-flight cap and the drain on close.
+   *
+   * @private
+   */
+  async #publish (row) {
+    this.#publishing.add(row.id)
+
+    const publishing = this.#emission.emit(row.event)
+
+    this.#inflight.add(publishing)
+
+    try {
+      await publishing
+
+      this.#published.add(row.id)
+    } catch (error) {
+      console.warn('Event publication failed', { row: row.id, error })
+    } finally {
+      this.#inflight.delete(publishing)
+      this.#publishing.delete(row.id)
+    }
+  }
+
+  /**
+   * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
+   * unbounded drain outlives any grace period.
+   *
+   * @private
+   */
+  async #drain () {
+    if (this.#inflight.size === 0) return
+
+    await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)])
+  }
+
+  /**
+   * Reads what is due, publishes it, and marks everything this process has sent — what it just
+   * published and what the immediate path published since the last cycle. One cycle at a time.
+   *
+   * @private
+   */
+  #tick () {
+    if (this.#pumping) return
+
+    this.#pumping = true
+
+    void this.#pump().finally(() => (this.#pumping = false))
+  }
+
+  /** @private */
+  async #pump () {
+    let page
+
+    do {
+      page = await this.#read(page?.[page.length - 1]?.id)
+
+      if (page.length === 0) break
+
+      /*
+       * A row is unpublished in the database until a cycle marks it, so a page includes what
+       * this replica is sending right now and what a failed marking left behind. Only this
+       * process knows either.
+       */
+      const rows = page.filter((row) =>
+        !this.#published.has(row.id) && !this.#publishing.has(row.id))
+
+      if (rows.length > 0) {
+        console.info('Outbox recovering unpublished events', { count: rows.length })
+
+        // every row is given its chance; what the broker refused stays unpublished and comes
+        // back on a later cycle
+        await Promise.allSettled(rows.map((row) => this.#publish(row)))
+      }
+
+      // a full page is a page that may have been cut short
+    } while (page.length === this.#batch)
+
+    await this.#mark()
+  }
+
+  /**
+   * One page of what is due. In a healthy system the first one is empty, every cycle — a row is
+   * due only if the process that wrote it failed to publish or died before marking it.
+   *
+   * Reading is suspended, not stopped, while this replica does not know which lanes are its
+   * own: the cycle keeps running and keeps marking, and reading resumes as soon as an
+   * assignment arrives. Reading without an assignment would be a different guarantee, where
+   * every replica publishes every stranded row.
+   *
+   * @private
+   * @param {string} [after] the last id of the page before, so a page is never read twice
+   */
+  async #read (after) {
+    const lanes = this.#atom.slots(LANES)
+
+    if (lanes === null || lanes.length === 0) return []
+
+    return this.#storage.outbox.pending(lanes, Date.now(), this.#batch, after)
+      .catch((error) => {
+        console.warn('Outbox read failed', { error })
+
+        return []
+      })
+  }
+
+  /**
+   * One batched write for many events, which is why the ids are held in memory rather than
+   * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
+   * marked is simply published again, which is within the contract.
+   *
+   * @private
+   */
+  async #mark () {
+    if (this.#published.size === 0) return
+
+    const ids = [...this.#published]
+
+    try {
+      await this.#storage.outbox.settle(ids)
+
+      for (const id of ids) this.#published.delete(id)
+    } catch (error) {
+      console.warn('Outbox marking failed', { count: ids.length, error })
+    }
+  }
+
+   /**
+   * A lane this replica currently owns, so that in steady state it settles its own rows
+   * before it ever reads them. Any lane at all when it owns none: the row still has to be
+   * written, and whoever ends up owning that lane will pump it.
+   *
+   * @private
+   */
+  #lane () {
+    const owned = this.#atom.slots(LANES)
+
+    return owned === null || owned.length === 0
+      ? Math.floor(Math.random() * LANES)
+      : owned[Math.floor(Math.random() * owned.length)]
+  }
+}
+
+function number (variable, declared, fallback) {
+  if (declared !== undefined) return declared
+
+  const value = Number(process.env[variable])
+
+  return Number.isNaN(value) || value <= 0 ? fallback : value
+}
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref())
+
+/**
+ * Constant, never configuration: rows carry their lane, so lowering this would leave rows in
+ * lanes nobody reads any more. It is also the ceiling on replicas of one component, and a
+ * power of two so that the common replica counts divide evenly.
+ */
+export const LANES = 128
+
+/** one cycle reads, publishes and marks; in steady state it finds nothing to read */
+const INTERVAL = 5000
+
+/**
+ * `gap = interval * K`. Not a steady-state necessity — a replica writes into a lane it owns
+ * and marks what it published — but a guard for when a lane changes hands between the write
+ * and the settle. Two cycles of separation, plus one of margin.
+ */
+const K = 3
+
+/** how many rows one read brings back; the pump reads on while a page comes back full */
+const BATCH = 200
+
+const DRAIN = 10_000
+const INFLIGHT = 1000
+const PUBLISHED = 10_000