@toa.io/extensions.cadence 1.0.0-alpha.284

package/source/Dispatcher.test.ts

@@ -0,0 +1,271 @@
+import { it, beforeEach, afterEach, mock } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { Dispatcher } from './Dispatcher.js'
+import { LANES } from './const.js'
+import type { Local } from './Local.js'
+import type { atomicity } from '@toa.io/core'
+
+type Invoke = (endpoint: string, request: any) => Promise<unknown>
+type Slots = (total: number) => number[] | null
+type Listener = () => void
+
+const INTERVAL = 100
+
+let metronome: Local & { invoke: ReturnType<typeof mock.fn<Invoke>> }
+let target: Local & { invoke: ReturnType<typeof mock.fn<Invoke>> }
+let atom: atomicity.Atom & { slots: ReturnType<typeof mock.fn<Slots>> }
+let listeners: Listener[]
+let rows: Row[]
+
+interface Row {
+  id: string
+  lane: number
+  due: number
+  endpoint: string
+  request: object | null
+}
+
+const row = (id: string, due: number, lane = 0): Row =>
+  ({ id, lane, due, endpoint: 'tea.pots.transit', request: { input: { id } } })
+
+const connector = (properties: object): any =>
+  ({ ...properties, link: mock.fn(), connect: async () => {}, disconnect: async () => {} })
+
+const advance = async (ms: number): Promise<void> => {
+  mock.timers.tick(ms)
+
+  for (let i = 0; i < 30; i++) await Promise.resolve()
+}
+
+/** the query one scan made */
+const reads = (): any[] => metronome.invoke.mock.calls
+  .filter((call) => call.arguments[0] === 'enumerate')
+  .map((call) => call.arguments[1].query)
+
+/** what has been settled, flattened */
+const settled = (): string[] => metronome.invoke.mock.calls
+  .filter((call) => call.arguments[0] === 'settle')
+  .flatMap((call) => call.arguments[1].query.ids as string[])
+
+const range = (total: number): number[] => Array.from({ length: total }, (_, i) => i)
+
+const create = (): Dispatcher =>
+  new Dispatcher(metronome, () => target, atom)
+
+
+beforeEach(() => {
+  process.env.TOA_CADENCE_DISCRETENESS = String(INTERVAL)
+  mock.timers.enable({ apis: ['setTimeout', 'setInterval', 'Date'], now: 0 })
+
+  rows = []
+  listeners = []
+
+  metronome = connector({
+    invoke: mock.fn<Invoke>(async (endpoint: string) => endpoint === 'enumerate' ? rows : null)
+  })
+
+  target = connector({ invoke: mock.fn<Invoke>(async () => null) })
+
+  atom = connector({
+    slots: mock.fn<Slots>((total) => range(total)),
+    onassigned: (listener: Listener) => {
+      listeners.push(listener)
+      listener()
+
+      return () => { listeners = listeners.filter((one) => one !== listener) }
+    }
+  })
+})
+
+afterEach(() => {
+  mock.timers.reset()
+  delete process.env.TOA_CADENCE_DISCRETENESS
+})
+
+it('should read what is due within the interval, in the lanes it owns', async () => {
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(INTERVAL)
+
+  const [first] = reads()
+
+  // the first scan is the one the subscription runs as it is added, at time zero
+  assert.strictEqual(first.criteria,
+    `lane=in=(${range(LANES).join(',')});due<${INTERVAL};expires>0`)
+  assert.deepStrictEqual(first.sort, ['due:asc'])
+})
+
+it('should read nothing while it owns nothing', async () => {
+  atom.slots.mock.mockImplementation(() => null)
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(INTERVAL * 3)
+
+  assert.strictEqual(reads().length, 0)
+})
+
+it('should call at the due time and not before', async () => {
+  rows = [row('a', 60)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(1)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'armed, not called')
+
+  await advance(60)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1)
+
+  const [endpoint, request] = target.invoke.mock.calls[0].arguments
+
+  assert.strictEqual(endpoint, 'transit')
+  assert.deepStrictEqual(request, { input: { id: 'a' }, task: true })
+})
+
+it('should call one whose time has already passed at once', async () => {
+  rows = [row('a', -10_000)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(0)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1,
+    'a scan reads what is overdue along with what is coming')
+})
+
+it('should arm a row once, however often a scan reads it', async () => {
+  rows = [row('a', 10_000)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+
+  // one interval at a time: a scan is one at a time, and a single jump would find the first
+  // still running and skip the rest
+  for (let cycle = 0; cycle < 4; cycle++) await advance(INTERVAL)
+
+  assert.ok(reads().length >= 3, 'it kept scanning')
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'and armed it only once')
+})
+
+it('should not arm a row again while its call is in flight', async () => {
+  let release: () => void = () => {}
+
+  rows = [row('a', 10)]
+  target.invoke.mock.mockImplementation(async () =>
+    await new Promise<void>((resolve) => { release = resolve }))
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(10) // the call goes out, and does not come back
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1)
+
+  // a scan while it is in flight reads the row again, and settles what it thinks was called
+  await advance(INTERVAL)
+
+  // far enough for anything that scan armed to have fired
+  await advance(INTERVAL)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1, 'and does not send it a second time')
+
+  release()
+  await advance(0)
+})
+
+it('should settle what it has called', async () => {
+  rows = [row('a', 10)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(10)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1)
+
+  rows = []
+  await advance(INTERVAL)
+
+  assert.deepStrictEqual(settled(), ['a'])
+})
+
+it('should settle a row whose call failed, and keep dispatching', async () => {
+  rows = [row('a', 10), row('b', 20)]
+  target.invoke.mock.mockImplementationOnce(async () => { throw new Error('nope') })
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(20)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 2, 'the one after it is still called')
+
+  rows = []
+  await advance(INTERVAL)
+
+  assert.deepStrictEqual(settled().sort(), ['a', 'b'], 'and the one that failed is settled')
+})
+
+it('should drop timers for lanes it no longer owns', async () => {
+  rows = [row('a', 10_000, 7)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(INTERVAL)
+
+  rows = []
+  atom.slots.mock.mockImplementation(() => [0])
+
+  for (const listener of listeners) listener()
+
+  await advance(20_000)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'lane 7 is somebody else\'s now')
+})
+
+it('should drop everything it holds when it owns nothing', async () => {
+  rows = [row('a', 10_000, 7)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(INTERVAL)
+
+  rows = []
+  atom.slots.mock.mockImplementation(() => null)
+
+  for (const listener of listeners) listener()
+
+  await advance(20_000)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0,
+    'whoever holds lane 7 now is the one to make that call')
+})
+
+it('should scan at once when what it owns changes', async () => {
+  atom.slots.mock.mockImplementation(() => null)
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(0)
+
+  assert.strictEqual(reads().length, 0)
+
+  atom.slots.mock.mockImplementation((total) => range(total))
+
+  for (const listener of listeners) listener()
+
+  await advance(0)
+
+  assert.strictEqual(reads().length, 1, 'without waiting out an interval')
+})

package/source/Dispatcher.ts

@@ -0,0 +1,221 @@
+import { console } from 'openspan'
+import { Connector, Locator } from '@toa.io/core'
+import { BATCH, DISCRETENESS, LANES, number } from './const.js'
+import type { Local } from './Local.js'
+import type { atomicity } from '@toa.io/core'
+
+/**
+ * Makes the calls that were put off.
+ *
+ * Two tiers, which is what a store polled rarely and a call due to the second need between
+ * them: the rows are the record, and one scan primes the next interval of them into timers.
+ * A crash loses the timers and nothing else — the next scan reads what is overdue along with
+ * what is upcoming, so a dispatcher that died mid-interval costs a delay and not a call.
+ *
+ * Partitioned by lane exactly as the outbox is, so every replica scans and dispatches its own
+ * share of every interval rather than taking turns at whole ones.
+ */
+export class Dispatcher extends Connector {
+  private readonly metronome: Local
+  private readonly resolve: (locator: Locator) => Local
+  private readonly atom: atomicity.Atom
+
+  /** rows in this replica's hands — a timer set for them, or a call of theirs on its way */
+  private readonly armed = new Map<string, Armed>()
+
+  /** ids called and not yet removed, held until a scan settles them */
+  private called = new Set<string>()
+
+  /** what it has called through, so that each is torn down with it */
+  private readonly targets = new Set<Local>()
+
+  /** milliseconds between passes, and how far ahead each one reaches */
+  private readonly discreteness: number
+
+  private timer?: NodeJS.Timeout
+  private off?: () => void
+  private scanning = false
+  private closing = false
+
+  public constructor (metronome: Local, resolve: (locator: Locator) => Local,
+    atom: atomicity.Atom) {
+    super()
+
+    this.metronome = metronome
+    this.resolve = resolve
+    this.atom = atom
+    this.discreteness = number('TOA_CADENCE_DISCRETENESS', DISCRETENESS * 1000)
+
+    this.depends(metronome)
+    this.depends(atom)
+  }
+
+  protected override async open (): Promise<void> {
+    this.timer = setInterval(() => { this.tick() }, this.discreteness)
+    this.timer.unref()
+
+    // a lane changing hands is when its rows become this replica's, or stop being
+    this.off = this.atom.onassigned(() => { this.adopt() })
+  }
+
+  protected override async close (): Promise<void> {
+    this.closing = true
+
+    this.off?.()
+    clearInterval(this.timer)
+
+    for (const { timer } of this.armed.values()) clearTimeout(timer)
+
+    this.armed.clear()
+
+    await this.settle()
+  }
+
+  /** One scan at a time: a slow one must not have another started underneath it. */
+  private tick (): void {
+    if (this.scanning || this.closing) return
+
+    this.scanning = true
+
+    void this.scan()
+      .catch((error: unknown) => { console.error('Delayed calls scan failed', { error }) })
+      .finally(() => { this.scanning = false })
+  }
+
+  /**
+   * What this replica owns has changed. Timers for lanes it no longer holds are dropped —
+   * whoever holds them now will arm them — and a scan runs at once rather than at the next
+   * interval, so lanes just acquired are not left waiting.
+   */
+  private adopt (): void {
+    // nothing owned is nothing held: what was armed under a claim this replica no longer has
+    // belongs to whoever has it now, and firing it anyway is the call twice
+    this.disarm(this.atom.slots(LANES) ?? [])
+
+    this.tick()
+  }
+
+  /** What is armed for a lane this replica no longer holds: whoever holds it now will arm it. */
+  private disarm (lanes: number[]): void {
+    for (const [id, { timer, lane }] of this.armed) {
+      if (lanes.includes(lane)) continue
+
+      clearTimeout(timer)
+      this.armed.delete(id)
+    }
+  }
+
+  private async scan (): Promise<void> {
+    const lanes = this.atom.slots(LANES)
+
+    if (lanes === null || lanes.length === 0) return
+
+    await this.settle()
+
+    const now = Date.now()
+    const until = now + this.discreteness
+
+    const rows = await this.metronome.invoke('enumerate', {
+      query: {
+        // a call nothing was running to make in time has expired, and is simply not read
+        criteria: `lane=in=(${lanes.join(',')});due<${until};expires>${now}`,
+        sort: ['due:asc'],
+        limit: BATCH
+      }
+    }) as Row[]
+
+    for (const row of rows) this.arm(row)
+  }
+
+  /**
+   * A row already armed is left alone: a scan reads what it read before, because a row stays
+   * until its call has been made.
+   */
+  private arm (row: Row): void {
+    if (this.armed.has(row.id) || this.called.has(row.id)) return
+
+    const timer = setTimeout(() => { void this.dispatch(row) }, Math.max(row.due - Date.now(), 0))
+
+    timer.unref()
+    this.armed.set(row.id, { timer, lane: row.lane })
+  }
+
+  /**
+   * The call goes out before the row is removed, never the other way round: a crash in between
+   * makes the call twice, and the other order would lose it. It travels as a task, so the
+   * broker holds it if the target is not there to take it.
+   */
+  private async dispatch (row: Row): Promise<void> {
+    if (this.closing) {
+      this.armed.delete(row.id)
+
+      return
+    }
+
+    const [namespace, name, endpoint] = row.endpoint.split('.')
+    const local = this.target(new Locator(name, namespace))
+
+    /*
+     * Attempted before it is made, not after it comes back. One attempt is what a dispatcher
+     * has to give: what it can tell about a failure is nothing, and a request the target's
+     * schema no longer fits never becomes one it does.
+     */
+    this.called.add(row.id)
+
+    /*
+     * The call travels as a task, so what raises here is this side of it: a stored request the
+     * target's contract no longer fits, an endpoint it no longer has, a broker that refused the
+     * enqueue. What the operation itself does with it happens in the target's own process and
+     * never comes back.
+     *
+     * Skipped, as a pulse skips: reported, and the row settled on the attempt either way.
+     */
+    await local.invoke(endpoint, { input: null, ...row.request, task: true })
+      .catch((error: unknown) => {
+        console.error('Delayed call failed', { endpoint: row.endpoint, id: row.id, error })
+      })
+      // held for as long as the call is, so `armed` answers what this replica has in hand
+      // rather than what it has a timer for
+      .finally(() => { this.armed.delete(row.id) })
+  }
+
+  /**
+   * A remote is resolved once per target and held: it opens a connection, and one nothing
+   * depends on would outlive the dispatcher that made it.
+   */
+  private target (locator: Locator): Local {
+    const local = this.resolve(locator)
+
+    if (!this.targets.has(local)) {
+      this.targets.add(local)
+      this.depends(local)
+    }
+
+    return local
+  }
+
+  /** One write for many rows, which is why the ids are held rather than settled one by one. */
+  private async settle (): Promise<void> {
+    if (this.called.size === 0) return
+
+    const ids = [...this.called]
+
+    await this.metronome.invoke('settle', { query: { ids }, input: null })
+
+    // held until the write comes back, so that one which does not leaves them to a later scan
+    for (const id of ids) this.called.delete(id)
+  }
+}
+
+interface Armed {
+  timer: NodeJS.Timeout
+  lane: number
+}
+
+interface Row {
+  id: string
+  lane: number
+  due: number
+  endpoint: string
+  request?: object
+}

package/source/Factory.ts

@@ -0,0 +1,73 @@
+import { Connector, Locator } from '@toa.io/core'
+import { Composition } from './Composition.js'
+import { COMPONENT, NAMESPACE } from './const.js'
+import { Aspect } from './Aspect.js'
+import { Dispatcher } from './Dispatcher.js'
+import { Local } from './Local.js'
+import { Pulse } from './Pulse.js'
+import type { Declaration } from './types.js'
+import type { extensions } from '@toa.io/core'
+
+export class Factory implements extensions.Factory {
+  private readonly host: Host
+
+  /** one per component, however much of this extension calls through it */
+  private readonly locals: Record<string, Local> = {}
+
+  public constructor (host: Host) {
+    this.host = host
+  }
+
+  /**
+   * What runs beside a component. Its own component gets the dispatcher; everyone else gets a
+   * timer per operation they declared a cadence for.
+   *
+   * A tenant is created before the components of its composition are, which is why nothing here
+   * reaches for one.
+   */
+  public tenant (locator: Locator, declaration: object): Connector {
+    if (mine(locator))
+      return new Dispatcher(this.metronome(), (target) => this.local(target),
+        this.host.atom(locator.id))
+
+    const tenant = new Connector()
+    const pulses = declaration as Declaration | null
+
+    // `cadence: ~` is a component that only delays calls
+    if (pulses === null) return tenant
+
+    const local = this.local(locator)
+    const atom = this.host.atom(locator.id)
+
+    for (const [endpoint, { cycle, intervals }] of Object.entries(pulses))
+      tenant.depends(new Pulse({ locator, endpoint, cycle, intervals }, local, atom))
+
+    return tenant
+  }
+
+  /** `context.delay` — for every component but the one that keeps the calls. */
+  public aspect (locator: Locator): extensions.Aspect | extensions.Aspect[] {
+    if (mine(locator)) return []
+
+    return new Aspect(this.metronome())
+  }
+
+  public service (): Connector {
+    return new Composition(this.host)
+  }
+
+  private metronome (): Local {
+    return this.local(new Locator(COMPONENT, NAMESPACE))
+  }
+
+  private local (locator: Locator): Local {
+    return (this.locals[locator.id] ??= new Local(this.host, locator))
+  }
+}
+
+/** Whether this is a component the extension ships rather than one of the application's. */
+function mine (locator: Locator): boolean {
+  return locator.namespace === NAMESPACE
+}
+
+export type Host = extensions.Host

package/source/Local.ts

@@ -0,0 +1,39 @@
+import { Connector } from '@toa.io/core'
+import type { Host } from './Factory.js'
+import type { Locator, Remote, Reply, Request } from '@toa.io/core'
+
+/**
+ * The component a tenant belongs to, as something to call.
+ *
+ * Resolved on the first call and not before: a tenant is created ahead of the components of its
+ * composition, so at that point there is nothing to look up yet. A pulse calls at a boundary,
+ * which is long after everything is up.
+ */
+export class Local extends Connector {
+  private readonly host: Host
+  private readonly locator: Locator
+  private remote?: Promise<Remote>
+
+  public constructor (host: Host, locator: Locator) {
+    super()
+
+    this.host = host
+    this.locator = locator
+  }
+
+  public async invoke (endpoint: string, request: Request): Promise<Reply> {
+    this.remote ??= this.locate()
+
+    return await (await this.remote).invoke(endpoint, request)
+  }
+
+  private async locate (): Promise<Remote> {
+    const remote = await this.host.remote(this.locator)
+
+    this.depends(remote)
+
+    await remote.connect()
+
+    return remote
+  }
+}