@toa.io/atomicity 1.0.0-alpha.271

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@toa.io/atomicity",
+  "version": "1.0.0-alpha.271",
+  "description": "Toa Atomicity Connector",
+  "author": "temich <tema.gurtovoy@gmail.com>",
+  "homepage": "https://github.com/toa-io/toa#readme",
+  "main": "src/index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toa-io/toa.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toa-io/toa/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@sesamecare-oss/redlock": "1.4.0",
+    "@toa.io/core": "1.0.0-alpha.270",
+    "ioredis": "6.0.0",
+    "n-and-i": "0.3.0",
+    "openspan": "1.0.0-alpha.270"
+  },
+  "scripts": {
+    "test": "echo \"Error: run tests from root\" && exit 1"
+  }
+}

package/readme.md

@@ -0,0 +1,90 @@
+# Toa Atomicity
+
+Decisions several replicas have to make together, arbitrated by Redis.
+
+Every component has one as `context.atom`, shared by the replicas of that component. Nothing is
+declared in the manifest.
+
+## Partitioning
+
+An exclusive claim on one of a fixed number of slots: while a replica holds a slot, no other
+replica of the group holds it.
+
+```javascript
+const slots = context.atom.slots(128) // [0, 2, 4, …] — or null, owning nothing
+```
+
+Returns `null` while this replica owns nothing — at startup, during a rollout, or while Redis is unreachable.
+
+Uses [n-and-i](https://github.com/temich/nandi).
+
+## Metering
+
+Debt the group has run up under each key, in milliseconds. A call adds its own deltas and answers
+what the group has reached.
+
+```javascript
+const [debt] = await context.atom.meter(['sam'], [1000])
+```
+
+Debt drains a millisecond a millisecond, on Redis' clock. Keys belong to the group.
+
+## Locking
+
+`routine` runs while no other replica of the group holds `keys`. Acquiring waits for as long as it
+takes, and a lease is extended for as long as the routine runs.
+
+```javascript
+await context.atom.lock('the ledger', async () => { … })
+```
+
+Uses [redlock](https://github.com/sesamecare/redlock). The key is written to every address and a
+majority holding it is the lock, so a minority can be lost or failed over without invalidating one.
+Against a single address there is no majority to lose: a restart that drops the key, or a failover
+to a replica that has not received it, can leave two holders. Entity writes do not rest on either —
+they have `_version`.
+
+The routine is given an `AbortSignal` and a context. Extension can fail while the routine runs, and
+the signal is how it says so:
+
+```javascript
+await context.atom.lock('the ledger', async (signal) => {
+  await something()
+
+  if (signal.aborted) throw signal.error
+})
+```
+
+## Outside a component
+
+```javascript
+const { Factory } = require('@toa.io/atomicity')
+
+const atom = new Factory().atom('mail.sender')
+
+await atom.connect()
+```
+
+## Configuration
+
+Requires Redis.
+
+```yaml
+# context.toa.yaml
+atomicity:
+  redis: redis://redis.example.com    # one address, or an odd number of them
+  interval: 5000                      # how often a replica registers, milliseconds
+```
+
+Several addresses are **independent servers**, not the nodes of a cluster. The lock is taken on a
+quorum of them; partitioning and metering use the first, because each of them counts on a single
+key. An even number is refused: `floor(n / 2) + 1` means four tolerate one loss exactly as three do,
+and two tolerate none at all.
+
+`redis://localhost` under `TOA_DEV=1`.
+
+One set of clients per process, shared by every atom in it, and an unreachable Redis does not fail
+a start.
+
+A key names what it is for and whose it is: `slots:{group}:…`, `meter:<group>:<key>`,
+`lock:<group>:<key>`.

package/src/atom.js

@@ -0,0 +1,176 @@
+'use strict'
+
+const { console } = require('openspan')
+const { Connector } = require('@toa.io/core')
+
+/**
+ * What one group of replicas decides together, in one place: which of them owns what, what they
+ * have spent between them, and which of them holds a name while it works.
+ *
+ * @implements {toa.core.atomicity.Atom}
+ */
+class Atom extends Connector {
+  #connection
+  #name
+  #interval
+
+  /** @type {{ i: number, n: number } | null} */
+  #assignment = null
+
+  /** @type {AbortController} */
+  #abort
+
+  /** @type {Promise<void>} */
+  #discovering
+
+  /** the loop's own logger, carrying the group every line belongs to */
+  #console
+
+  constructor (connection, name, interval) {
+    super()
+
+    this.#connection = connection
+    this.#name = name
+    this.#interval = interval ?? override() ?? INTERVAL
+    this.#console = console.fork({ group: name })
+
+    this.depends(connection)
+  }
+
+  /**
+   * An exclusive claim on slots of `0..total`: while this replica holds one, no other replica
+   * of the group does.
+   *
+   * The arithmetic is n-and-i. Every replica registers in a Redis counter once per interval
+   * and receives a `{ i, n }` pair once two consecutive intervals have agreed on it, so a
+   * replica that has just joined, stalled or restarted claims nothing until the group has
+   * settled.
+   *
+   * @param total {number}
+   * @returns {number[] | null} null while this replica owns nothing
+   */
+  slots (total) {
+    const assignment = this.#assignment
+
+    if (assignment === null) return null
+
+    const { i, n } = assignment
+    const owned = []
+
+    for (let slot = i; slot < total; slot += n) owned.push(slot)
+
+    return owned
+  }
+
+  /**
+   * Debt the group has run up under each key, in milliseconds: every call adds its own deltas
+   * and reads back where the group stands, so a replica reports only what it has spent and
+   * still throttles on what all of them have.
+   *
+   * @param {string[]} keys
+   * @param {number[]} deltas
+   * @returns {Promise<number[]>}
+   */
+  async meter (keys, deltas) {
+    const meter = this.#connection.meter
+
+    if (meter === undefined)
+      throw new Error('Metering requires atomicity. Set TOA_ATOMICITY_REDIS.')
+
+    return meter.meter(this.#keys(METER, keys), deltas)
+  }
+
+  /**
+   * Runs `routine` holding `keys`, and while it holds them no other replica of the group does.
+   * Waits for as long as it takes to acquire them.
+   *
+   * @param {string | string[]} keys
+   * @param {() => Promise<any>} routine
+   * @returns {Promise<any>}
+   */
+  async lock (keys, routine) {
+    const redlock = this.#connection.redlock
+
+    if (redlock === undefined)
+      throw new Error('Locking requires atomicity. Set TOA_ATOMICITY_REDIS.')
+
+    return redlock.using(this.#keys(LOCK, keys), LEASE, routine)
+  }
+
+  /**
+   * A key names what it is for and whose it is, in that order, so that a name used for a lock
+   * and for a meter is two keys, and two groups using the same name do not meet either.
+   *
+   * @private
+   */
+  #keys (kind, keys) {
+    if (typeof keys === 'string') keys = [keys]
+
+    return keys.map((key) => `${kind}${this.#name}:${key}`)
+  }
+
+  async open () {
+    const redis = this.#connection.redis
+
+    // without a Redis nothing can be owned exclusively, so nothing is claimed at all
+    if (redis === undefined) return
+
+    this.#abort = new AbortController()
+    this.#discovering = this.#discover(redis)
+  }
+
+  async close () {
+    this.#assignment = null
+
+    this.#abort?.abort()
+
+    await this.#discovering
+  }
+
+  /**
+   * Nothing is caught here on purpose. A registration that fails is handled inside n-and-i,
+   * which hands out the idle pair rather than raising, and an aborted signal ends the loop the
+   * way `break` does — so the only thing that can reach this is a fault in the call itself,
+   * and a fault deserves to be seen rather than turned into standing down.
+   *
+   * @private
+   */
+  async #discover (redis) {
+    const { discover } = await import('n-and-i')
+
+    const loop = discover({
+      redis,
+      prefix: SLOTS,
+      name: this.#name,
+      interval: this.#interval,
+      signal: this.#abort.signal,
+      console: this.#console
+    })
+
+    // the loop yields when ownership changes, which is exactly when work has to be handed over
+    for await (const { i, n } of loop)
+      this.#assignment = i === null ? null : { i, n }
+  }
+}
+
+function override () {
+  const value = Number(process.env.TOA_ATOMICITY_INTERVAL)
+
+  return Number.isNaN(value) || value <= 0 ? undefined : value
+}
+
+const INTERVAL = 5000
+
+/** how long a lock is held before it has to be extended, in milliseconds */
+const LEASE = 5000
+
+/*
+ * What a key is for, ahead of whose it is. The three live in one Redis and are written by
+ * three different things, so nothing but this keeps a group's lock on a name apart from its
+ * meter on the same name.
+ */
+const SLOTS = 'slots:'
+const METER = 'meter:'
+const LOCK = 'lock:'
+
+exports.Atom = Atom

package/src/connection.js

@@ -0,0 +1,118 @@
+'use strict'
+
+const { Redlock } = require('@sesamecare-oss/redlock')
+const { Redis } = require('ioredis')
+const { console } = require('openspan')
+const { Connector } = require('@toa.io/core')
+const { Meter } = require('./meter')
+
+/**
+ * One set of clients per process, shared by every atom in it. Each keeps its own keys, so a
+ * composition of five components opens one connection between them however much they decide.
+ */
+class Connection extends Connector {
+  /**
+   * The one the registry and the meter use. Both count on a single key — replicas per interval,
+   * debt per name — and a key spread over several servers would be several answers.
+   *
+   * @type {import('ioredis').Redis}
+   */
+  redis
+
+  /** @type {Meter} */
+  meter
+
+  /** @type {Redlock} */
+  redlock
+
+  /** @type {import('ioredis').Redis[]} */
+  #clients = []
+
+  async open () {
+    const urls = resolve()
+
+    if (urls.length === 0) {
+      console.warn('Atomicity is not configured, so nothing is owned and nothing is metered. ' +
+        'Set TOA_ATOMICITY_REDIS.')
+
+      return
+    }
+
+    /*
+     * Refused rather than degraded, because an even number can never be what it looks like: a
+     * lock is taken on `floor(n / 2) + 1` of the addresses, so four tolerate one loss exactly
+     * as three do, and two tolerate none at all — one fewer than one address does.
+     */
+    if (urls.length % 2 === 0)
+      throw new Error(`Atomicity takes an odd number of addresses, ${urls.length} given.`)
+
+    this.#clients = urls.map((url) => new Redis(url, OPTIONS))
+
+    /*
+     * Connecting is not awaited, and a failure to connect is not an error here. Coordination
+     * that cannot be reached must read exactly as coordination that was never configured —
+     * nothing is owned, and whoever asked stands down — and a process that could not start
+     * because of it would be the opposite of that. The client retries on its own, so a Redis
+     * that comes up later is picked up without anything being restarted.
+     */
+    for (const client of this.#clients)
+      // ioredis leaves `message` empty on a refused connection, where the code is the whole story
+      client.on('error', (error) =>
+        console.warn('Atomicity is unreachable, so nothing is owned',
+          { host: client.options.host, error: error.code ?? error.message }))
+
+    this.redis = this.#clients[0]
+
+    // one script per process, whatever the groups sharing this client meter under
+    this.meter = new Meter(this.redis)
+
+    /*
+     * Every address, because this is the one decision that can be taken on a quorum: the same
+     * key is written to all of them, and a majority holding it survives losing a minority. The
+     * registry and the meter cannot be held that way — their key is one key — so the addresses
+     * past the first buy an uninterrupted lock and nothing else.
+     */
+    this.redlock = new Redlock(this.#clients, { retryCount: -1 })
+
+    console.info('Atomicity connecting to redis', { nodes: this.#clients.length })
+  }
+
+  async close () {
+    for (const client of this.#clients) client.disconnect()
+
+    // a closed connection holds nothing: it is opened again with whatever is configured then,
+    // and a client that has been disconnected would fail every command put to it
+    this.#clients = []
+    this.redis = undefined
+    this.meter = undefined
+    this.redlock = undefined
+  }
+}
+
+const OPTIONS = { enableReadyCheck: true }
+
+function resolve () {
+  const value = process.env[VARIABLE]
+
+  // an empty value is atomicity turned off, where an absent one in development is the
+  // local Redis — as everything else in development resolves
+  if (value === undefined) return process.env.TOA_DEV === '1' ? [DEV] : []
+
+  return value.split(' ').filter((url) => url !== '')
+}
+
+let instance
+
+/** the connection every atom of the process shares */
+const connection = () => (instance ??= new Connection())
+
+/** @internal for tests */
+const reset = () => (instance = undefined)
+
+const VARIABLE = 'TOA_ATOMICITY_REDIS'
+const DEV = 'redis://localhost'
+
+exports.Connection = Connection
+exports.connection = connection
+exports.reset = reset
+exports.VARIABLE = VARIABLE

package/src/factory.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const { Atom } = require('./atom')
+const { connection } = require('./connection')
+
+class Factory {
+  /**
+   * What the replicas of one group decide together. They find each other by `group` and by
+   * nothing else, so what shares a name shares a decision.
+   *
+   * @param {string} group
+   * @param {object} [options]
+   */
+  atom (group, options = {}) {
+    return atom(group, options)
+  }
+}
+
+/** @type {Map<string, Atom>} */
+const atoms = new Map()
+
+/**
+ * One atom per group per process, however many ask for it. A second would register a second
+ * time in a group that has one replica, and every replica of that group would then be told it
+ * owns half of what it does.
+ */
+function atom (group, options) {
+  let atom = atoms.get(group)
+
+  if (atom === undefined) {
+    atom = new Atom(connection(), group, options.interval)
+
+    atoms.set(group, atom)
+  }
+
+  return atom
+}
+
+exports.Factory = Factory

package/src/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const { Factory } = require('./factory')
+const { VARIABLE } = require('./connection')
+
+exports.Factory = Factory
+exports.VARIABLE = VARIABLE

package/src/meter.js

@@ -0,0 +1,83 @@
+'use strict'
+
+/**
+ * Debt shared with every other process metering under the same key.
+ *
+ * A key holds one number: the milliseconds of debt everyone metering it has run up
+ * between them. Debt drains at a millisecond a millisecond and grows by what each
+ * process reports, so it is additive — which is what lets a process report only its
+ * own increments, on its own schedule, and still read back where the group stands.
+ *
+ * The clock is Redis' own, so the processes reporting need not agree on the time, and
+ * what comes back is a duration rather than a moment, so neither does the caller.
+ *
+ * A whole batch is metered by one script, because the callers are rate limiters and
+ * the number of keys they watch is the number of clients they are watching for.
+ */
+class Meter {
+  #redis
+
+  constructor (redis) {
+    this.#redis = redis
+
+    redis.defineCommand(COMMAND, { lua: SCRIPT })
+  }
+
+  /**
+   * Adds each delta to its key, and answers what the group has reached.
+   *
+   * @param {string[]} keys
+   * @param {number[]} deltas
+   * @returns {Promise<number[]>}
+   */
+  async meter (keys, deltas) {
+    if (keys.length <= CHUNK) return this.#call(keys, deltas)
+
+    // arguments are spread into the call, so a batch is split rather than risking the
+    // engine's limit on how many there may be
+    const debts = []
+
+    for (let i = 0; i < keys.length; i += CHUNK)
+      debts.push(...await this.#call(keys.slice(i, i + CHUNK), deltas.slice(i, i + CHUNK)))
+
+    return debts
+  }
+
+  /** @private */
+  async #call (keys, deltas) {
+    // `defineCommand` extends the client at runtime
+    return this.#redis[COMMAND](keys.length, ...keys, ...deltas)
+  }
+}
+
+const COMMAND = 'meter'
+
+/** How long a key outlives the debt on it, in milliseconds. */
+const GRACE = 10_000
+
+/** Keys per call, bounding how many arguments are spread into one. */
+const CHUNK = 2000
+
+const SCRIPT = `
+local time = redis.call('TIME')
+local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+local debts = {}
+
+for i = 1, #KEYS do
+  local key = KEYS[i]
+  local state = redis.call('HMGET', key, 'd', 'a')
+  local debt = tonumber(state[1]) or 0
+  local at = tonumber(state[2]) or now
+
+  debt = math.max(0, debt - (now - at)) + tonumber(ARGV[i])
+
+  redis.call('HSET', key, 'd', debt, 'a', now)
+  redis.call('PEXPIRE', key, debt + ${GRACE})
+
+  debts[i] = debt
+end
+
+return debts
+`
+
+exports.Meter = Meter