@toa.io/core 1.0.0-alpha.27 → 1.0.0-alpha.272

package/src/outbox/outbox.js

@@ -0,0 +1,301 @@
+'use strict'
+
+const { console } = require('openspan')
+const { Connector } = require('../connector')
+const { newid } = require('../entities/newid')
+
+/**
+ * Owns the intent to publish. A row is built before the write so that the storage can commit
+ * it in the same transaction as the entity; publication then happens off the operation's path,
+ * and anything that fails to publish is recovered from the row.
+ *
+ * The mechanism is a safety net: in a healthy system the row is written, published within
+ * milliseconds by the same process, and marked published on that process's next tick.
+ *
+ * A storage that cannot commit a row atomically has no outbox, and this degrades to the
+ * inline emission it replaces.
+ */
+class Outbox extends Connector {
+  #emission
+  #storage
+  #atom
+
+  #gap
+  #interval
+  #batch
+  #defer
+
+  /** ids this process has published, held until a cycle marks them */
+  #published = new Set()
+
+  /** in-flight publications, awaited (with a bound) on close */
+  #inflight = new Set()
+
+  /** rows this replica is publishing right now, so a cycle does not pick them up again */
+  #publishing = new Set()
+
+  #timer
+  #pumping = false
+  #closing = false
+
+  constructor (emission, storage, atom, options = {}) {
+    super()
+
+    this.#emission = emission
+    this.#storage = storage
+    this.#atom = atom
+
+    this.#interval = number('TOA_OUTBOX_INTERVAL', options.interval, INTERVAL)
+    this.#batch = number('TOA_OUTBOX_BATCH', options.batch, BATCH)
+    this.#gap = options.gap ?? this.#interval * K
+    this.#defer = process.env.TOA_OUTBOX_DEFER === '1'
+
+    this.depends(emission)
+    this.depends(atom)
+
+    if (storage !== undefined) this.depends(storage)
+  }
+
+  /** whether the storage can commit a row atomically with the entity */
+  get durable () {
+    return this.#storage?.outbox !== undefined
+  }
+
+  /**
+   * @param event {toa.core.transition.Event}
+   * @returns {object}
+   */
+  row (event) {
+    return {
+      id: newid(),
+      lane: this.#lane(),
+      published: false,
+      pending: Date.now() + this.#gap,
+      event
+    }
+  }
+
+  /**
+   * Hands a committed row over. Awaited by the caller only on the legacy path — with an
+   * outbox this returns at once and the broker leaves the operation's path.
+   */
+  publish (row) {
+    if (!this.durable)
+      return this.#emission.emit(row.event)
+
+    /*
+     * A publication started while the pump is closing would outlive the emitters it needs,
+     * and `comq` waits on a connection that is going rather than failing. The row is already
+     * durable, so leaving it is exactly what it is for.
+     */
+    if (this.#closing || this.#defer ||
+      this.#inflight.size >= INFLIGHT || this.#published.size >= PUBLISHED)
+      return
+
+    void this.#publish(row)
+  }
+
+  async open () {
+    if (!this.durable) return
+
+    if (this.#defer)
+      console.warn('Outbox immediate publication is deferred; events are published by the pump only')
+
+    this.#timer = setInterval(() => this.#tick(), this.#interval)
+    this.#timer.unref()
+  }
+
+  async close () {
+    this.#closing = true
+
+    if (this.#timer !== undefined) clearInterval(this.#timer)
+
+    await this.#drain()
+    await this.#mark()
+  }
+
+  /**
+   * Publishes one row and swallows the failure: the row stays unpublished and comes back on a
+   * later cycle, which is the whole point of having written it.
+   *
+   * There is no timeout here on purpose. A publication is a confirmed write to a durable
+   * exchange, and `comq` waits for the broker to come back rather than failing — abandoning
+   * it would not stop it, it would only mean the row is published twice once it lands. What
+   * bounds this instead is the in-flight cap and the drain on close.
+   *
+   * @private
+   */
+  async #publish (row) {
+    this.#publishing.add(row.id)
+
+    const publishing = this.#emission.emit(row.event)
+
+    this.#inflight.add(publishing)
+
+    try {
+      await publishing
+
+      this.#published.add(row.id)
+    } catch (error) {
+      console.warn('Event publication failed', { row: row.id, error })
+    } finally {
+      this.#inflight.delete(publishing)
+      this.#publishing.delete(row.id)
+    }
+  }
+
+  /**
+   * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
+   * unbounded drain outlives any grace period.
+   *
+   * @private
+   */
+  async #drain () {
+    if (this.#inflight.size === 0) return
+
+    await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)])
+  }
+
+  /**
+   * Reads what is due, publishes it, and marks everything this process has sent — what it just
+   * published and what the immediate path published since the last cycle. One cycle at a time.
+   *
+   * @private
+   */
+  #tick () {
+    if (this.#pumping) return
+
+    this.#pumping = true
+
+    void this.#pump().finally(() => (this.#pumping = false))
+  }
+
+  /** @private */
+  async #pump () {
+    let page
+
+    do {
+      page = await this.#read(page?.[page.length - 1]?.id)
+
+      if (page.length === 0) break
+
+      /*
+       * A row is unpublished in the database until a cycle marks it, so a page includes what
+       * this replica is sending right now and what a failed marking left behind. Only this
+       * process knows either.
+       */
+      const rows = page.filter((row) =>
+        !this.#published.has(row.id) && !this.#publishing.has(row.id))
+
+      if (rows.length > 0) {
+        console.info('Outbox recovering unpublished events', { count: rows.length })
+
+        // every row is given its chance; what the broker refused stays unpublished and comes
+        // back on a later cycle
+        await Promise.allSettled(rows.map((row) => this.#publish(row)))
+      }
+
+      // a full page is a page that may have been cut short
+    } while (page.length === this.#batch)
+
+    await this.#mark()
+  }
+
+  /**
+   * One page of what is due. In a healthy system the first one is empty, every cycle — a row is
+   * due only if the process that wrote it failed to publish or died before marking it.
+   *
+   * Reading is suspended, not stopped, while this replica does not know which lanes are its
+   * own: the cycle keeps running and keeps marking, and reading resumes as soon as an
+   * assignment arrives. Reading without an assignment would be a different guarantee, where
+   * every replica publishes every stranded row.
+   *
+   * @private
+   * @param {string} [after] the last id of the page before, so a page is never read twice
+   */
+  async #read (after) {
+    const lanes = this.#atom.slots(LANES)
+
+    if (lanes === null || lanes.length === 0) return []
+
+    return this.#storage.outbox.pending(lanes, Date.now(), this.#batch, after)
+      .catch((error) => {
+        console.warn('Outbox read failed', { error })
+
+        return []
+      })
+  }
+
+  /**
+   * One batched write for many events, which is why the ids are held in memory rather than
+   * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
+   * marked is simply published again, which is within the contract.
+   *
+   * @private
+   */
+  async #mark () {
+    if (this.#published.size === 0) return
+
+    const ids = [...this.#published]
+
+    try {
+      await this.#storage.outbox.settle(ids)
+
+      for (const id of ids) this.#published.delete(id)
+    } catch (error) {
+      console.warn('Outbox marking failed', { count: ids.length, error })
+    }
+  }
+
+   /**
+   * A lane this replica currently owns, so that in steady state it settles its own rows
+   * before it ever reads them. Any lane at all when it owns none: the row still has to be
+   * written, and whoever ends up owning that lane will pump it.
+   *
+   * @private
+   */
+  #lane () {
+    const owned = this.#atom.slots(LANES)
+
+    return owned === null || owned.length === 0
+      ? Math.floor(Math.random() * LANES)
+      : owned[Math.floor(Math.random() * owned.length)]
+  }
+}
+
+function number (variable, declared, fallback) {
+  if (declared !== undefined) return declared
+
+  const value = Number(process.env[variable])
+
+  return Number.isNaN(value) || value <= 0 ? fallback : value
+}
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref())
+
+/**
+ * Constant, never configuration: rows carry their lane, so lowering this would leave rows in
+ * lanes nobody reads any more. It is also the ceiling on replicas of one component, and a
+ * power of two so that the common replica counts divide evenly.
+ */
+const LANES = 128
+
+/** one cycle reads, publishes and marks; in steady state it finds nothing to read */
+const INTERVAL = 5000
+
+/**
+ * `gap = interval * K`. Not a steady-state necessity — a replica writes into a lane it owns
+ * and marks what it published — but a guard for when a lane changes hands between the write
+ * and the settle. Two cycles of separation, plus one of margin.
+ */
+const K = 3
+
+/** how many rows one read brings back; the pump reads on while a page comes back full */
+const BATCH = 200
+
+const DRAIN = 10_000
+const INFLIGHT = 1000
+const PUBLISHED = 10_000
+
+exports.Outbox = Outbox
+exports.LANES = LANES

package/src/query/options.js

@@ -33,8 +33,9 @@ const projection = (projection, properties) => {
     }
   }
 
-  if (projection.includes('_version') === false)
-    projection.push('_version')
+  for (const property of ['_version', '_created', '_updated', '_deleted'])
+    if (!projection.includes(property))
+      projection.push(property)
 }
 
 exports.options = options

package/src/query.js

@@ -7,6 +7,9 @@ class Query {
   #properties
   #system
 
+  /** @type {Map<string, object>} parsed criteria by their expression */
+  #asts = new Map()
+
   constructor (properties) {
     this.#properties = properties
     this.#system = Object.keys(properties).filter((key) => properties[key].system === true)
@@ -19,13 +22,15 @@ class Query {
   parse (query) {
     /** @type {toa.core.storages.Query} */
     const result = {}
-    const { id, version, criteria, ...rest } = query
+    const { id, ids, version, criteria, search, ...rest } = query
 
     const options = this.#options(rest)
 
     if (id !== undefined) result.id = id
+    if (ids !== undefined) result.ids = ids
     if (version !== undefined) result.version = version
-    if (criteria !== undefined) result.criteria = parse.criteria(criteria, this.#properties)
+    if (criteria !== undefined) result.criteria = this.#criteria(criteria)
+    if (search !== undefined) result.search = search
     if (options !== undefined) result.options = options
 
     return result
@@ -36,6 +41,31 @@ class Query {
 
     return parse.options(options, this.#properties, this.#system)
   }
+
+  /**
+   * Lexing an RSQL expression is the most expensive thing this class does, and the same
+   * expression arrives over and over: a route builds it from its own declaration and the
+   * request parameters. The tree is only read downstream — a storage builds a fresh query
+   * from it — so it is kept rather than parsed again.
+   *
+   * Expressions come from the client, hence the bound. An invalid one throws before it
+   * reaches the cache.
+   */
+  #criteria (criteria) {
+    const known = this.#asts.get(criteria)
+
+    if (known !== undefined) return known
+
+    const ast = parse.criteria(criteria, this.#properties)
+
+    if (this.#asts.size >= LIMIT) this.#asts.clear()
+
+    this.#asts.set(criteria, ast)
+
+    return ast
+  }
 }
 
+const LIMIT = 1024
+
 exports.Query = Query

package/src/receiver.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { console, decode, run } = require('openspan')
 const { add } = require('@toa.io/generic')
 const { Connector } = require('./connector')
 
@@ -16,18 +17,28 @@ class Receiver extends Connector {
   /** @type {string} */
   #endpoint
 
-  /** @type {toa.core.Component} */
+  /** @type {string} */
+  #label
+
+  /** @type {string} */
+  #destination
+
+  /** @type {unknown[]} */
+  #arguments
+
+  /** @type {toa.core.Source} */
+  #origin
+
   #local
 
-  /** @type {toa.core.bridges.Receiver} */
   #bridge
 
-  /**
-   *
-   * @param {toa.norm.component.Receiver} definition
-   * @param {toa.core.Component} local
-   * @param {toa.core.bridges.Receiver} bridge
-   */
+  /** @type {object} */
+  #delivery
+
+  /** @type {object} */
+  #processing
+
   constructor (definition, local, bridge) {
     super()
 
@@ -36,17 +47,42 @@ class Receiver extends Connector {
     this.#conditioned = conditioned
     this.#adaptive = adaptive
     this.#endpoint = operation
+    this.#label = definition.label ?? operation
+    this.#destination = definition.destination ?? this.#label
+    this.#arguments = definition.arguments
+    this.#origin = definition.origin
 
     this.#local = local
     this.#bridge = bridge
 
     this.depends(local)
     if (bridge !== undefined) this.depends(bridge)
+
+    /*
+     * The delivery span is created on behalf of the messaging destination, so that
+     * each consumer forms its own complete producer/consumer pair, and service graphs
+     * display fan-out correctly: producer -> destination -> each consumer
+     * (Tempo pairs spans one-to-one, thus multiple consumers can not pair
+     * with a single producer span, see grafana/tempo#5408)
+     */
+    this.#delivery = {
+      name: `${this.#label} deliver`,
+      kind: 'producer',
+      service: this.#destination,
+      attributes: { 'messaging.destination.name': this.#destination }
+    }
+
+    this.#processing = {
+      name: `${this.#label} process`,
+      kind: 'consumer',
+      service: local.locator.id,
+      attributes: { 'messaging.destination.name': this.#destination }
+    }
   }
 
   /** @hot */
   async receive (message) {
-    const { payload, ...extensions } = message
+    const { payload, telemetry, ...extensions } = message
 
     if (this.#conditioned && await this.#bridge.condition(payload) === false) return
 
@@ -54,11 +90,37 @@ class Receiver extends Connector {
 
     add(request, extensions)
 
-    await this.#local.invoke(this.#endpoint, request)
+    // set after `add`, so that a message field can not spoof the origin
+    if (this.#origin !== undefined) request.source = this.#origin
+
+    // continue the trace from the producer span
+    const remote = telemetry === undefined ? null : decode(telemetry)
+    const task = () => this.#process(request)
+
+    if (remote === null)
+      await task()
+    else
+      await run(remote, task)
+  }
+
+  async #process (request) {
+    return console.span(this.#delivery, async () => console.span(this.#processing, async () => {
+      try {
+        await this.#local.invoke(this.#endpoint, request)
+      } catch (error) {
+        console.error('Receiver error', {
+          component: this.#local.locator.id,
+          endpoint: this.#endpoint,
+          error
+        })
+
+        throw error
+      }
+    }))
   }
 
   async #request (payload) {
-    return this.#adaptive ? await this.#bridge.request(payload) : { input: payload }
+    return this.#adaptive ? await this.#bridge.request(payload, ...(this.#arguments ?? [])) : { input: payload }
   }
 }
 

package/src/remote.js

@@ -1,16 +1,17 @@
 'use strict'
 
-const { console } = require('@toa.io/console')
-
+const assert = require('node:assert')
 const { Component } = require('./component')
 
 class Remote extends Component {
-  async open () {
-    console.info(`Remote '${this.locator.id}' connected`)
-  }
+  kind = 'client'
+
+  explain (endpoint) {
+    if (!(endpoint in this.operations))
+      // `assert.fail`, not `assert.ok`: the message is built only when it is needed
+      assert.fail(`Endpoint '${endpoint}' is not provided by '${this.locator.id}'`)
 
-  async dispose () {
-    console.info(`Remote '${this.locator.id}' disconnected`)
+    return this.operations[endpoint].explain()
   }
 }