@toa.io/extensions.cadence 1.0.0-alpha.284 → 1.0.0-alpha.287

package/source/Dispatcher.test.ts

@@ -1,10 +1,11 @@
 import { it, beforeEach, afterEach, mock } from 'node:test'
 import assert from 'node:assert/strict'
+import { console } from 'openspan'
 
 import { Dispatcher } from './Dispatcher.js'
-import { LANES } from './const.js'
+import { BATCH, LANES } from './const.js'
 import type { Local } from './Local.js'
-import type { atomicity } from '@toa.io/core'
+import type { atomicity } from '@toa.io/core/types'
 
 type Invoke = (endpoint: string, request: any) => Promise<unknown>
 type Slots = (total: number) => number[] | null
@@ -17,20 +18,33 @@ let target: Local & { invoke: ReturnType<typeof mock.fn<Invoke>> }
 let atom: atomicity.Atom & { slots: ReturnType<typeof mock.fn<Slots>> }
 let listeners: Listener[]
 let rows: Row[]
+let warn: ReturnType<typeof mock.method<typeof console, 'warn'>>
 
 interface Row {
   id: string
   lane: number
   due: number
+  expires: number
   endpoint: string
   request: object | null
 }
 
-const row = (id: string, due: number, lane = 0): Row =>
-  ({ id, lane, due, endpoint: 'tea.pots.transit', request: { input: { id } } })
+/** `expires` defaults to no bound at all, which is what `overdue: null` writes */
+const row = (id: string, due: number, lane = 0, expires = Number.MAX_SAFE_INTEGER): Row => ({
+  id,
+  lane,
+  due,
+  expires,
+  endpoint: 'tea.pots.transit',
+  request: { input: { id } }
+})
 
-const connector = (properties: object): any =>
-  ({ ...properties, link: mock.fn(), connect: async () => {}, disconnect: async () => {} })
+const connector = (properties: object): any => ({
+  ...properties,
+  link: mock.fn(),
+  connect: async () => {},
+  disconnect: async () => {}
+})
 
 const advance = async (ms: number): Promise<void> => {
   mock.timers.tick(ms)
@@ -39,20 +53,20 @@ const advance = async (ms: number): Promise<void> => {
 }
 
 /** the query one scan made */
-const reads = (): any[] => metronome.invoke.mock.calls
-  .filter((call) => call.arguments[0] === 'enumerate')
-  .map((call) => call.arguments[1].query)
+const reads = (): any[] =>
+  metronome.invoke.mock.calls
+    .filter((call) => call.arguments[0] === 'enumerate')
+    .map((call) => call.arguments[1].query)
 
 /** what has been settled, flattened */
-const settled = (): string[] => metronome.invoke.mock.calls
-  .filter((call) => call.arguments[0] === 'settle')
-  .flatMap((call) => call.arguments[1].query.ids as string[])
+const settled = (): string[] =>
+  metronome.invoke.mock.calls
+    .filter((call) => call.arguments[0] === 'settle')
+    .flatMap((call) => call.arguments[1].query.ids as string[])
 
 const range = (total: number): number[] => Array.from({ length: total }, (_, i) => i)
 
-const create = (): Dispatcher =>
-  new Dispatcher(metronome, () => target, atom)
-
+const create = (): Dispatcher => new Dispatcher(metronome, () => target, atom)
 
 beforeEach(() => {
   process.env.TOA_CADENCE_DISCRETENESS = String(INTERVAL)
@@ -60,9 +74,12 @@ beforeEach(() => {
 
   rows = []
   listeners = []
+  warn = mock.method(console, 'warn', () => undefined)
 
   metronome = connector({
-    invoke: mock.fn<Invoke>(async (endpoint: string) => endpoint === 'enumerate' ? rows : null)
+    invoke: mock.fn<Invoke>(async (endpoint: string) =>
+      endpoint === 'enumerate' ? rows : null
+    )
   })
 
   target = connector({ invoke: mock.fn<Invoke>(async () => null) })
@@ -73,17 +90,20 @@ beforeEach(() => {
       listeners.push(listener)
       listener()
 
-      return () => { listeners = listeners.filter((one) => one !== listener) }
+      return () => {
+        listeners = listeners.filter((one) => one !== listener)
+      }
     }
   })
 })
 
 afterEach(() => {
   mock.timers.reset()
+  warn.mock.restore()
   delete process.env.TOA_CADENCE_DISCRETENESS
 })
 
-it('should read what is due within the interval, in the lanes it owns', async () => {
+it('should read two intervals ahead, in the lanes it owns', async () => {
   const dispatcher = create()
 
   await dispatcher.connect()
@@ -91,12 +111,122 @@ it('should read what is due within the interval, in the lanes it owns', async ()
 
   const [first] = reads()
 
-  // the first scan is the one the subscription runs as it is added, at time zero
-  assert.strictEqual(first.criteria,
-    `lane=in=(${range(LANES).join(',')});due<${INTERVAL};expires>0`)
+  // the first scan is the one the subscription runs as it is added, at time zero. Two
+  // intervals, so that consecutive passes overlap; and no bound on `expires`, because an
+  // expired row is settled rather than left where nothing will ever read it again
+  assert.strictEqual(
+    first.criteria,
+    `lane=in=(${range(LANES).join(',')});due<${2 * INTERVAL}`
+  )
   assert.deepStrictEqual(first.sort, ['due:asc'])
 })
 
+it('should reach past the next pass, so a row on its horizon is armed before it is due', async () => {
+  rows = [row('a', INTERVAL + 10)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+
+  // gone from the store before the next pass, so nothing but the scan at time zero can have
+  // armed it — which is what reaching two intervals rather than one is for
+  rows = []
+
+  await advance(INTERVAL)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'armed, not yet due')
+
+  await advance(10)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1, 'and called at its due time')
+})
+
+it('should settle an expired row without calling it', async () => {
+  rows = [row('a', -10_000, 0, -5_000)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(0)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'past the bound its caller gave it')
+
+  rows = []
+  await advance(INTERVAL)
+
+  assert.deepStrictEqual(
+    settled(),
+    ['a'],
+    'and settled, or nothing would ever read it again'
+  )
+})
+
+it('should settle what it has called even while it owns nothing', async () => {
+  rows = [row('a', 10)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(10)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1)
+
+  // the lane is somebody else's now, and a row left live is one they call again
+  rows = []
+  atom.slots.mock.mockImplementation(() => null)
+
+  await advance(INTERVAL)
+
+  assert.deepStrictEqual(settled(), ['a'])
+})
+
+it('should say so when a scan fills its batch', async () => {
+  rows = Array.from({ length: BATCH }, (_, i) => row(String(i), 10_000))
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(0)
+
+  const warned = warn.mock.calls.map((call) => call.arguments[0] as string)
+
+  assert.ok(
+    warned.some((message) => message.includes('filled its batch')),
+    'the horizon is truncated and the rest arrive late'
+  )
+})
+
+it('should say so when a pass is skipped because the one before it is still running', async () => {
+  let release: () => void = () => {}
+
+  metronome.invoke.mock.mockImplementation(async (endpoint: string) => {
+    if (endpoint !== 'enumerate') return null
+
+    await new Promise<void>((resolve) => {
+      release = resolve
+    })
+
+    return []
+  })
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(0) // the scan at time zero, which does not come back
+
+  await advance(INTERVAL)
+
+  const warned = warn.mock.calls.map((call) => call.arguments[0] as string)
+
+  assert.ok(
+    warned.some((message) => message.includes('has not returned')),
+    'a dispatcher that stops dispatching says so'
+  )
+
+  release()
+  await advance(0)
+})
+
 it('should read nothing while it owns nothing', async () => {
   atom.slots.mock.mockImplementation(() => null)
 
@@ -136,8 +266,11 @@ it('should call one whose time has already passed at once', async () => {
   await dispatcher.connect()
   await advance(0)
 
-  assert.strictEqual(target.invoke.mock.callCount(), 1,
-    'a scan reads what is overdue along with what is coming')
+  assert.strictEqual(
+    target.invoke.mock.callCount(),
+    1,
+    'a scan reads what is overdue along with what is coming'
+  )
 })
 
 it('should arm a row once, however often a scan reads it', async () => {
@@ -159,8 +292,12 @@ it('should not arm a row again while its call is in flight', async () => {
   let release: () => void = () => {}
 
   rows = [row('a', 10)]
-  target.invoke.mock.mockImplementation(async () =>
-    await new Promise<void>((resolve) => { release = resolve }))
+  target.invoke.mock.mockImplementation(
+    async () =>
+      await new Promise<void>((resolve) => {
+        release = resolve
+      })
+  )
 
   const dispatcher = create()
 
@@ -175,7 +312,11 @@ it('should not arm a row again while its call is in flight', async () => {
   // far enough for anything that scan armed to have fired
   await advance(INTERVAL)
 
-  assert.strictEqual(target.invoke.mock.callCount(), 1, 'and does not send it a second time')
+  assert.strictEqual(
+    target.invoke.mock.callCount(),
+    1,
+    'and does not send it a second time'
+  )
 
   release()
   await advance(0)
@@ -199,19 +340,29 @@ it('should settle what it has called', async () => {
 
 it('should settle a row whose call failed, and keep dispatching', async () => {
   rows = [row('a', 10), row('b', 20)]
-  target.invoke.mock.mockImplementationOnce(async () => { throw new Error('nope') })
+  target.invoke.mock.mockImplementationOnce(async () => {
+    throw new Error('nope')
+  })
 
   const dispatcher = create()
 
   await dispatcher.connect()
   await advance(20)
 
-  assert.strictEqual(target.invoke.mock.callCount(), 2, 'the one after it is still called')
+  assert.strictEqual(
+    target.invoke.mock.callCount(),
+    2,
+    'the one after it is still called'
+  )
 
   rows = []
   await advance(INTERVAL)
 
-  assert.deepStrictEqual(settled().sort(), ['a', 'b'], 'and the one that failed is settled')
+  assert.deepStrictEqual(
+    settled().sort(),
+    ['a', 'b'],
+    'and the one that failed is settled'
+  )
 })
 
 it('should drop timers for lanes it no longer owns', async () => {
@@ -229,7 +380,7 @@ it('should drop timers for lanes it no longer owns', async () => {
 
   await advance(20_000)
 
-  assert.strictEqual(target.invoke.mock.callCount(), 0, 'lane 7 is somebody else\'s now')
+  assert.strictEqual(target.invoke.mock.callCount(), 0, "lane 7 is somebody else's now")
 })
 
 it('should drop everything it holds when it owns nothing', async () => {
@@ -247,8 +398,11 @@ it('should drop everything it holds when it owns nothing', async () => {
 
   await advance(20_000)
 
-  assert.strictEqual(target.invoke.mock.callCount(), 0,
-    'whoever holds lane 7 now is the one to make that call')
+  assert.strictEqual(
+    target.invoke.mock.callCount(),
+    0,
+    'whoever holds lane 7 now is the one to make that call'
+  )
 })
 
 it('should scan at once when what it owns changes', async () => {
@@ -269,3 +423,57 @@ it('should scan at once when what it owns changes', async () => {
 
   assert.strictEqual(reads().length, 1, 'without waiting out an interval')
 })
+
+it('should give up a row a scan stops reading, so a cancellation still lands', async () => {
+  rows = [row('a', INTERVAL + 50)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(1)
+
+  // cancelled: the row is tombstoned, so the next scan does not read it
+  rows = []
+
+  await advance(INTERVAL) // the scan that finds it gone
+  await advance(INTERVAL) // and past when it would have been called
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'the timer was given up with the row')
+})
+
+it('should keep a row the scan still reads', async () => {
+  rows = [row('a', INTERVAL + 50)]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(INTERVAL)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 0, 'not yet due')
+
+  await advance(50)
+
+  assert.strictEqual(target.invoke.mock.callCount(), 1)
+})
+
+it('should not give up a row when the batch was filled', async () => {
+  const held = row('held', INTERVAL + 50)
+
+  rows = [held, ...Array.from({ length: BATCH - 1 }, (_, i) => row('b' + i, 10_000))]
+
+  const dispatcher = create()
+
+  await dispatcher.connect()
+  await advance(1)
+
+  // a full batch is a read that was cut short, so absence from the next one is not a
+  // cancellation — these are rows it never reached, not rows nobody owes
+  rows = Array.from({ length: BATCH }, (_, i) => row('c' + i, 20_000))
+
+  await advance(INTERVAL)
+  await advance(50)
+
+  const called = target.invoke.mock.calls.map((call) => call.arguments[1].input.id)
+
+  assert.deepStrictEqual(called, ['held'], 'the row it armed is still called')
+})

package/source/Dispatcher.ts

@@ -2,15 +2,16 @@ import { console } from 'openspan'
 import { Connector, Locator } from '@toa.io/core'
 import { BATCH, DISCRETENESS, LANES, number } from './const.js'
 import type { Local } from './Local.js'
-import type { atomicity } from '@toa.io/core'
+import type { atomicity } from '@toa.io/core/types'
 
 /**
  * Makes the calls that were put off.
  *
  * Two tiers, which is what a store polled rarely and a call due to the second need between
- * them: the rows are the record, and one scan primes the next interval of them into timers.
- * A crash loses the timers and nothing else — the next scan reads what is overdue along with
- * what is upcoming, so a dispatcher that died mid-interval costs a delay and not a call.
+ * them: the rows are the record, and one scan primes the next two intervals of them into
+ * timers. A crash loses the timers and nothing else — the next scan reads what is overdue
+ * along with what is upcoming, so a dispatcher that died mid-interval costs a delay and not a
+ * call.
  *
  * Partitioned by lane exactly as the outbox is, so every replica scans and dispatches its own
  * share of every interval rather than taking turns at whole ones.
@@ -37,8 +38,14 @@ export class Dispatcher extends Connector {
   private scanning = false
   private closing = false
 
-  public constructor (metronome: Local, resolve: (locator: Locator) => Local,
-    atom: atomicity.Atom) {
+  /** when the scan that is running started, so a tick that finds it can say how long */
+  private started = 0
+
+  public constructor(
+    metronome: Local,
+    resolve: (locator: Locator) => Local,
+    atom: atomicity.Atom
+  ) {
     super()
 
     this.metronome = metronome
@@ -50,15 +57,19 @@ export class Dispatcher extends Connector {
     this.depends(atom)
   }
 
-  protected override async open (): Promise<void> {
-    this.timer = setInterval(() => { this.tick() }, this.discreteness)
+  protected override async open(): Promise<void> {
+    this.timer = setInterval(() => {
+      this.tick()
+    }, this.discreteness)
     this.timer.unref()
 
     // a lane changing hands is when its rows become this replica's, or stop being
-    this.off = this.atom.onassigned(() => { this.adopt() })
+    this.off = this.atom.onassigned(() => {
+      this.adopt()
+    })
   }
 
-  protected override async close (): Promise<void> {
+  protected override async close(): Promise<void> {
     this.closing = true
 
     this.off?.()
@@ -72,14 +83,37 @@ export class Dispatcher extends Connector {
   }
 
   /** One scan at a time: a slow one must not have another started underneath it. */
-  private tick (): void {
-    if (this.scanning || this.closing) return
+  private tick(): void {
+    if (this.closing) return
+
+    /*
+     * Skipped, and said so. The guard itself is right — a second scan on top of one that has
+     * not returned only makes it worse — but a pass that outlives its interval takes every
+     * pass after it with it, and nothing is dispatched at all while that lasts. Silently, it
+     * is a dispatcher that stops dispatching in a deployment that ran for months.
+     */
+    if (this.scanning) {
+      const running = Date.now() - this.started
+      const details = { running, discreteness: this.discreteness }
+      const message = 'Delayed calls scan has not returned, so this pass is skipped'
+
+      // past a few intervals it is not a slow pass but a stuck one
+      if (running > STALLED * this.discreteness) console.error(message, details)
+      else console.warn(message, details)
+
+      return
+    }
 
     this.scanning = true
+    this.started = Date.now()
 
     void this.scan()
-      .catch((error: unknown) => { console.error('Delayed calls scan failed', { error }) })
-      .finally(() => { this.scanning = false })
+      .catch((error: unknown) => {
+        console.error('Delayed calls scan failed', { error })
+      })
+      .finally(() => {
+        this.scanning = false
+      })
   }
 
   /**
@@ -87,7 +121,7 @@ export class Dispatcher extends Connector {
    * whoever holds them now will arm them — and a scan runs at once rather than at the next
    * interval, so lanes just acquired are not left waiting.
    */
-  private adopt (): void {
+  private adopt(): void {
     // nothing owned is nothing held: what was armed under a claim this replica no longer has
     // belongs to whoever has it now, and firing it anyway is the call twice
     this.disarm(this.atom.slots(LANES) ?? [])
@@ -96,7 +130,7 @@ export class Dispatcher extends Connector {
   }
 
   /** What is armed for a lane this replica no longer holds: whoever holds it now will arm it. */
-  private disarm (lanes: number[]): void {
+  private disarm(lanes: number[]): void {
     for (const [id, { timer, lane }] of this.armed) {
       if (lanes.includes(lane)) continue
 
@@ -105,36 +139,101 @@ export class Dispatcher extends Connector {
     }
   }
 
-  private async scan (): Promise<void> {
+  private async scan(): Promise<void> {
+    // ahead of the claim, because settling needs none: the rows are ones this replica has
+    // already called, and one left live is one the next owner of its lane calls again
+    await this.settle()
+
     const lanes = this.atom.slots(LANES)
 
     if (lanes === null || lanes.length === 0) return
 
-    await this.settle()
-
     const now = Date.now()
-    const until = now + this.discreteness
 
-    const rows = await this.metronome.invoke('enumerate', {
+    /*
+     * Two intervals, so that consecutive passes overlap by one whole one. Reaching exactly as
+     * far as the next pass leaves a sliver beyond the horizon that the next pass reads already
+     * overdue — the width of the drift between the two, which a row with a tight `overdue`
+     * does not survive. The overlap absorbs that, a settle that ran long, a tick skipped
+     * outright, and the clock skew between the replica that wrote `due` and this one.
+     *
+     * Reaching further costs nothing a cancellation would pay for, because what is armed is
+     * given up again the moment a scan stops reading it — see `forget`.
+     */
+    const until = now + 2 * this.discreteness
+
+    const rows = (await this.metronome.invoke('enumerate', {
       query: {
-        // a call nothing was running to make in time has expired, and is simply not read
-        criteria: `lane=in=(${lanes.join(',')});due<${until};expires>${now}`,
+        // expiry is not a criterion: a row nobody reads is a row nobody settles, and it would
+        // stay live for good. It is read, and settled without the call being made
+        criteria: `lane=in=(${lanes.join(',')});due<${until}`,
         sort: ['due:asc'],
         limit: BATCH
       }
-    }) as Row[]
+    })) as Row[]
+
+    // the horizon is truncated to the last row of a full batch, and everything past it arrives
+    // on a later pass already overdue. Where this is logged, the batch is too small for the rate
+    const truncated = rows.length === BATCH
+
+    if (truncated)
+      console.warn('Delayed calls scan filled its batch', { limit: BATCH, lanes: lanes.length })
 
     for (const row of rows) this.arm(row)
+
+    if (!truncated) this.forget(new Set(rows.map((row) => row.id)))
+  }
+
+  /**
+   * Gives up what the store no longer answers with. A scan reads everything still owed in the
+   * lanes this replica holds, so a row it armed before and does not read now is one nothing
+   * owes any more — cancelled, most of the time — and a timer left for it makes the call
+   * regardless.
+   *
+   * This is what lets a scan reach further ahead than the next one without a cancellation
+   * becoming any less reliable: a row is held only for as long as the store keeps saying it is
+   * owed, so what bounds cancelling is the period between passes and not how far one reaches.
+   *
+   * Two things are not evidence of a cancellation. A batch filled to its limit is a read that
+   * was cut short, and says nothing about what it did not reach. And a call already on its way
+   * is settled by its own path rather than by what a scan reads.
+   */
+  private forget(read: Set<string>): void {
+    for (const [id, { timer }] of this.armed) {
+      if (read.has(id) || this.called.has(id)) continue
+
+      clearTimeout(timer)
+      this.armed.delete(id)
+    }
   }
 
   /**
    * A row already armed is left alone: a scan reads what it read before, because a row stays
    * until its call has been made.
    */
-  private arm (row: Row): void {
+  private arm(row: Row): void {
     if (this.armed.has(row.id) || this.called.has(row.id)) return
 
-    const timer = setTimeout(() => { void this.dispatch(row) }, Math.max(row.due - Date.now(), 0))
+    const now = Date.now()
+
+    /*
+     * Past the bound its caller gave it, so the call is not made — and the row is settled all
+     * the same, silently, because an expired call is not an event anything sees. Left unread
+     * it would never be settled either: a row nothing will ever act on, sitting in the
+     * collection and in front of every scan, for good.
+     */
+    if (row.expires <= now) {
+      this.called.add(row.id)
+
+      return
+    }
+
+    const timer = setTimeout(
+      () => {
+        void this.dispatch(row)
+      },
+      Math.max(row.due - now, 0)
+    )
 
     timer.unref()
     this.armed.set(row.id, { timer, lane: row.lane })
@@ -145,7 +244,7 @@ export class Dispatcher extends Connector {
    * makes the call twice, and the other order would lose it. It travels as a task, so the
    * broker holds it if the target is not there to take it.
    */
-  private async dispatch (row: Row): Promise<void> {
+  private async dispatch(row: Row): Promise<void> {
     if (this.closing) {
       this.armed.delete(row.id)
 
@@ -170,20 +269,27 @@ export class Dispatcher extends Connector {
      *
      * Skipped, as a pulse skips: reported, and the row settled on the attempt either way.
      */
-    await local.invoke(endpoint, { input: null, ...row.request, task: true })
+    await local
+      .invoke(endpoint, { input: null, ...row.request, task: true })
       .catch((error: unknown) => {
-        console.error('Delayed call failed', { endpoint: row.endpoint, id: row.id, error })
+        console.error('Delayed call failed', {
+          endpoint: row.endpoint,
+          id: row.id,
+          error
+        })
       })
       // held for as long as the call is, so `armed` answers what this replica has in hand
       // rather than what it has a timer for
-      .finally(() => { this.armed.delete(row.id) })
+      .finally(() => {
+        this.armed.delete(row.id)
+      })
   }
 
   /**
    * A remote is resolved once per target and held: it opens a connection, and one nothing
    * depends on would outlive the dispatcher that made it.
    */
-  private target (locator: Locator): Local {
+  private target(locator: Locator): Local {
     const local = this.resolve(locator)
 
     if (!this.targets.has(local)) {
@@ -195,7 +301,7 @@ export class Dispatcher extends Connector {
   }
 
   /** One write for many rows, which is why the ids are held rather than settled one by one. */
-  private async settle (): Promise<void> {
+  private async settle(): Promise<void> {
     if (this.called.size === 0) return
 
     const ids = [...this.called]
@@ -216,6 +322,13 @@ interface Row {
   id: string
   lane: number
   due: number
+
+  /** past this the call is not made; the row is settled without it */
+  expires: number
+
   endpoint: string
   request?: object
 }
+
+/** intervals a scan may run for before it is a stuck pass rather than a slow one */
+const STALLED = 5