@superblocksteam/shared 0.9607.3 → 0.9608.1

package/src/socket/heartbeat.test.ts

@@ -0,0 +1,538 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { SocketHeartbeat } from './heartbeat.js';
+
+const INTERVAL_MS = 100;
+const TIMEOUT_MS = 50;
+
+/** A beat whose answer the test decides, so the deadline is the only thing racing it. */
+function controllableBeat() {
+  const pending: { reject: (error: unknown) => void; resolve: () => void }[] = [];
+  const beat = vi.fn(
+    () =>
+      new Promise<void>((resolve, reject) => {
+        pending.push({ reject, resolve });
+      })
+  );
+  return {
+    answerLast: () => pending[pending.length - 1]?.resolve(),
+    beat,
+    refuseLast: (error: unknown) => pending[pending.length - 1]?.reject(error),
+    pending
+  };
+}
+
+function heartbeat(overrides: Partial<ConstructorParameters<typeof SocketHeartbeat>[0]> = {}) {
+  const onLost = vi.fn();
+  const onOutcome = vi.fn();
+  const { answerLast, beat, refuseLast } = controllableBeat();
+  const subject = new SocketHeartbeat({
+    beat,
+    intervalMs: INTERVAL_MS,
+    onLost,
+    onOutcome,
+    timeoutMs: TIMEOUT_MS,
+    ...overrides
+  });
+  return { answerLast, beat, onLost, onOutcome, refuseLast, subject };
+}
+
+/** Runs one full interval, then the beat deadline, letting each settle. */
+async function elapse(ms: number) {
+  await vi.advanceTimersByTimeAsync(ms);
+}
+
+const outcomes = (onOutcome: ReturnType<typeof vi.fn>): string[] => onOutcome.mock.calls.map((call) => call[0] as string);
+
+describe('SocketHeartbeat', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.clearAllTimers();
+    vi.useRealTimers();
+  });
+
+  // The answer has to pace the beats, not the interval. Otherwise a channel that
+  // stopped answering collects a new pending request every interval, none of
+  // which can ever be reconciled.
+  it('keeps a single beat in flight however many intervals pass', async () => {
+    const { beat, subject } = heartbeat({ timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS * 5);
+
+    expect(beat).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  // A request channel's own budget is the whole caller-facing operation, so a
+  // beat that borrows it cannot report a miss inside any window a caller waits in.
+  it('holds a beat to its own deadline rather than the request channel budget', async () => {
+    const { onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['grace']);
+    subject.stop();
+  });
+
+  // A refusal is a definite answer about the peer - it arrived, so the channel
+  // works - and counting it as a lapse would hide the very thing it reports.
+  it('records a refusal apart from a silence rather than excusing it as a lapse', async () => {
+    const { onOutcome, refuseLast, subject } = heartbeat();
+    subject.start();
+    subject.recordReceive();
+
+    await elapse(INTERVAL_MS);
+    refuseLast(new Error('peer is shutting down'));
+    await elapse(1);
+
+    expect(outcomes(onOutcome)).toEqual(['refused']);
+    expect(subject.account).toMatchObject({
+      failures: 0,
+      lapses: 0,
+      lastError: 'peer is shutting down',
+      refusals: 1,
+      unanswered: 0
+    });
+    subject.stop();
+  });
+
+  // A refusal is an answer: the peer is reachable and the request channel works,
+  // so whatever is wrong is not the connection. Spending the grace meant for a
+  // starved pong on one would tear a working socket down over an
+  // application-level problem.
+  it('does not let a refusal spend the grace meant for a silence', async () => {
+    const { onLost, onOutcome, refuseLast, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    refuseLast(new Error('peer is shutting down'));
+    await elapse(1);
+
+    // One unanswered beat after the refusal is still only the first silence.
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['refused', 'grace']);
+    expect(onLost).not.toHaveBeenCalled();
+    subject.stop();
+  });
+
+  // ...but an answer to one beat says nothing about the beats that went
+  // missing, so a refusal must not clear a run of silences either. Otherwise a
+  // channel alternating between timing out and refusing never gives up.
+  it('does not let a refusal clear a run of silences', async () => {
+    const { onLost, onOutcome, refuseLast, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    refuseLast(new Error('peer is shutting down'));
+    await elapse(1);
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['grace', 'refused', 'miss']);
+    expect(onLost).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  // Inbound traffic proves the connection is alive, so an unanswered beat on a
+  // socket that is still delivering data indicts the request channel, not the
+  // connection. Closing on that would kill a healthy-but-busy socket.
+  it('lapses an unanswered beat while the socket is provably receiving', async () => {
+    const { onLost, onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    subject.recordReceive();
+    await elapse(TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['lapse']);
+    expect(subject.account).toMatchObject({ failures: 0, lapses: 1, unanswered: 1 });
+    expect(onLost).not.toHaveBeenCalled();
+    subject.stop();
+  });
+
+  // Proof of life has to undo a suspicion the same way an answer does, or a
+  // single miss during a large transfer stays on the books until a pong lands.
+  it('resets the miss count when inbound traffic proves the connection alive', async () => {
+    const { onLost, subject } = heartbeat();
+    subject.start();
+
+    // First beat: nothing inbound, so a genuine miss held back by grace.
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+    expect(subject.account).toMatchObject({ failures: 1 });
+
+    // Second beat: unanswered, but the socket is provably carrying traffic.
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    subject.recordReceive();
+    await elapse(TIMEOUT_MS);
+
+    expect(subject.account).toMatchObject({ failures: 0, lapses: 1, unanswered: 2 });
+    expect(onLost).not.toHaveBeenCalled();
+    subject.stop();
+  });
+
+  // Inbound data proves the receive path, not that the request channel answers.
+  // Without a bound, a wedged channel hides behind continuous pushes forever.
+  it('stops excusing a silent request channel once the silences are exhausted', async () => {
+    const maxConsecutiveUnanswered = 3;
+    const { onLost, onOutcome, subject } = heartbeat({ maxConsecutiveUnanswered });
+    subject.start();
+
+    // Traffic arrives during every beat's own wait, so each one is excused -
+    // until the allowance runs out and the next silence is believed anyway.
+    for (let beats = 0; beats < maxConsecutiveUnanswered; beats += 1) {
+      await elapse(INTERVAL_MS - (beats === 0 ? 0 : TIMEOUT_MS));
+      subject.recordReceive();
+      await elapse(TIMEOUT_MS);
+    }
+
+    expect(outcomes(onOutcome)).toEqual(['lapse', 'lapse', 'miss']);
+    expect(onLost).toHaveBeenCalledTimes(1);
+    // Both counts ride along on the close: how long the silence had run, and
+    // how much of it the socket was demonstrably carrying traffic through.
+    expect(onLost.mock.calls[0]?.[0]).toMatchObject({
+      failures: 1,
+      lapses: maxConsecutiveUnanswered - 1,
+      unanswered: maxConsecutiveUnanswered
+    });
+    subject.stop();
+  });
+
+  // The bound is on silence, not on the excuse. Two unanswered beats with
+  // nothing inbound must report the same run length as two that were excused,
+  // or telemetry cannot tell "died after two misses" from "wedged behind
+  // continuous pushes until the cap".
+  it('counts every unanswered beat toward the bound, excused or not', async () => {
+    const { onLost, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS * 2 + TIMEOUT_MS);
+
+    expect(onLost).toHaveBeenCalledTimes(1);
+    expect(onLost.mock.calls[0]?.[0]).toMatchObject({ failures: 2, lapses: 0, unanswered: 2 });
+    subject.stop();
+  });
+
+  // The bound is what stops a wedged request channel hiding behind inbound
+  // pushes forever, and a refusal answers one beat - not the beats that already
+  // went missing. Clearing the run on a refusal hands that channel an unlimited
+  // deferral: every silence is excused by traffic, so `unanswered` is the only
+  // thing left that can ever end it.
+  it('does not let a refusal defer the bound on a silent request channel', async () => {
+    const maxConsecutiveUnanswered = 3;
+    const { onLost, onOutcome, refuseLast, subject } = heartbeat({ maxConsecutiveUnanswered });
+    subject.start();
+
+    // Traffic arrives during the beat's own wait, so this silence is excused.
+    await elapse(INTERVAL_MS);
+    subject.recordReceive();
+    await elapse(TIMEOUT_MS);
+
+    // A refusal interleaves: recorded as the answer it is, but it must not wipe
+    // the run of silence already on the books.
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    refuseLast(new Error('peer is shutting down'));
+    await elapse(1);
+
+    await elapse(INTERVAL_MS - 1);
+    subject.recordReceive();
+    await elapse(TIMEOUT_MS);
+
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    subject.recordReceive();
+    await elapse(TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['lapse', 'refused', 'lapse', 'miss']);
+    expect(onLost).toHaveBeenCalledTimes(1);
+    expect(onLost.mock.calls[0]?.[0]).toMatchObject({ refusals: 1, unanswered: maxConsecutiveUnanswered });
+    subject.stop();
+  });
+
+  // `unanswered` is what the caller reports as the length of the silence that
+  // killed the connection, and it is documented as bounded by the cap. Beats
+  // keep going out after the alarm has been raised - the caller still has to
+  // tear the socket down - so without saturating it the reported run climbs
+  // past the cap it is promised to respect.
+  it('never reports a longer silence than the bound allows', async () => {
+    const maxConsecutiveUnanswered = 3;
+    const { onOutcome, subject } = heartbeat({ maxConsecutiveUnanswered });
+    subject.start();
+
+    await elapse(INTERVAL_MS * 5 + TIMEOUT_MS);
+
+    expect(subject.account.unanswered).toBe(maxConsecutiveUnanswered);
+    const lastAccount = onOutcome.mock.calls[onOutcome.mock.calls.length - 1]?.[2] as { unanswered: number };
+    expect(lastAccount.unanswered).toBe(maxConsecutiveUnanswered);
+    subject.stop();
+  });
+
+  // The alarm is edge-triggered so one run of misses causes one teardown, but
+  // that assumes the teardown happened. When the caller cannot act - the socket
+  // lookup throws, the close rejects - latching would mean nothing ever closes
+  // the connection and nothing ever says so again.
+  it('keeps the alarm armed when the caller could not act on the loss', async () => {
+    const onLost = vi.fn(() => {
+      throw new Error('the socket was already gone');
+    });
+    const { subject } = heartbeat({ maxConsecutiveMisses: 1, onLost });
+    subject.start();
+
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+    expect(onLost).toHaveBeenCalledTimes(1);
+
+    await elapse(INTERVAL_MS);
+    expect(onLost).toHaveBeenCalledTimes(2);
+    subject.stop();
+  });
+
+  // One missed beat is survivable: a bulk response is a single websocket message
+  // and no receive event fires until its last byte lands, so a transfer only has
+  // to finish inside one extra interval for the next answer to get through.
+  it('gives the first genuine miss grace and loses the beat on the second', async () => {
+    const { onLost, onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+    expect(onLost).not.toHaveBeenCalled();
+
+    await elapse(INTERVAL_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['grace', 'miss']);
+    expect(onLost).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  it('resets both counters when a beat is answered', async () => {
+    const { answerLast, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS + TIMEOUT_MS);
+    expect(subject.account).toMatchObject({ failures: 1 });
+
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    answerLast();
+    await elapse(1);
+
+    expect(subject.account).toMatchObject({ failures: 0, lapses: 0, unanswered: 0 });
+    expect(subject.account.lastSuccessAt).toBeDefined();
+    subject.stop();
+  });
+
+  // A deadline armed for a previous connection fires after the restart. Letting
+  // it tick the new run's counters would close a healthy socket on the strength
+  // of a dead one's silence.
+  it('ignores a beat armed before the heartbeat was restarted', async () => {
+    const { onLost, onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    subject.stop();
+    subject.start();
+    await elapse(TIMEOUT_MS);
+
+    expect(onLost).not.toHaveBeenCalled();
+    expect(outcomes(onOutcome)).toEqual([]);
+    expect(subject.account).toMatchObject({ failures: 0, lapses: 0 });
+    subject.stop();
+  });
+
+  // A socket can die before any beat settles, and unanswered beats are the
+  // strongest evidence there is that the peer was gone - so the account cannot
+  // be built from settled outcomes alone.
+  it('counts every beat issued, including one still waiting', async () => {
+    const { subject } = heartbeat({ timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+
+    expect(subject.account).toMatchObject({ failures: 0, sent: 1 });
+    subject.stop();
+  });
+
+  // `sent` says a beat went out; it cannot say whether the last one came back.
+  // The caller reporting a death needs both, because a socket that answered a
+  // beat and then went quiet is otherwise indistinguishable from one answering
+  // right up to the close.
+  it('stamps the outstanding beat and clears the stamp when it settles', async () => {
+    const { answerLast, subject } = heartbeat({ timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    const { pendingSince } = subject.account;
+    expect(pendingSince).toBeTypeOf('number');
+
+    answerLast();
+    await elapse(0);
+
+    expect(subject.account.pendingSince).toBeUndefined();
+    subject.stop();
+  });
+
+  // The account is read after the socket is gone, and stopping is the first
+  // thing a caller does with a dead connection. A stamp cleared on the way out
+  // would erase the one fact that names what the beat was doing when it died -
+  // and no settle is coming, because a retired beat's rejection is dropped.
+  it('keeps the outstanding beat stamped through a stop', async () => {
+    const { subject } = heartbeat({ timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    subject.stop();
+
+    expect(subject.account.pendingSince).toBeTypeOf('number');
+  });
+
+  // A channel that cannot carry a beat yet has no turn to protect either, so a
+  // skipped beat is not a miss - and must not be booked as an answer, which is
+  // what returning early from the beat itself would amount to.
+  it('skips a beat while the channel is not ready to carry one', async () => {
+    const { beat, onOutcome, subject } = heartbeat({ shouldBeat: () => false, timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS * 3);
+
+    expect(beat).not.toHaveBeenCalled();
+    expect(outcomes(onOutcome)).toEqual([]);
+    expect(subject.account).toMatchObject({ failures: 0, sent: 0 });
+    subject.stop();
+  });
+
+  // The alarm is edge-triggered: the counters are an account of what happened,
+  // read when a socket dies, and resetting them to re-arm would erase the very
+  // evidence the death report is built from.
+  it('raises the alarm once rather than on every beat after it', async () => {
+    const { onLost, onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS * 4 + TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual(['grace', 'miss', 'miss', 'miss']);
+    expect(onLost).toHaveBeenCalledTimes(1);
+    expect(subject.account).toMatchObject({ failures: 4 });
+    subject.stop();
+  });
+
+  it('re-arms the alarm once the peer answers again', async () => {
+    const { answerLast, onLost, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS * 2 + TIMEOUT_MS);
+    expect(onLost).toHaveBeenCalledTimes(1);
+
+    await elapse(INTERVAL_MS - TIMEOUT_MS);
+    answerLast();
+    await elapse(1);
+    expect(subject.account).toMatchObject({ failures: 0 });
+
+    await elapse(INTERVAL_MS * 2 + TIMEOUT_MS);
+
+    expect(onLost).toHaveBeenCalledTimes(2);
+    subject.stop();
+  });
+
+  // Two callers drive the editor's beat: its own cadence, and the moment a
+  // user comes back to the keyboard. The second must not have to wait out an
+  // interval sized for idle connections.
+  it('beats on demand outside the interval', async () => {
+    const { beat, onOutcome, subject } = heartbeat({ timeoutMs: 10_000 });
+
+    // Not awaited: the beat is deliberately left in flight, which is what an
+    // on-demand caller sees when the peer is slow.
+    void subject.beatNow();
+    await elapse(1);
+
+    expect(beat).toHaveBeenCalledTimes(1);
+    expect(subject.account).toMatchObject({ sent: 1 });
+    expect(outcomes(onOutcome)).toEqual([]);
+  });
+
+  it('lets an on-demand beat and the interval share the in-flight guard', async () => {
+    const { beat, subject } = heartbeat({ timeoutMs: 10_000 });
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    await subject.beatNow();
+
+    expect(beat).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  // A caller can learn it has a new connection before it is ready to beat on
+  // one. The old socket's armed deadline has to stop counting at that moment,
+  // not whenever the first new beat happens to go out.
+  it('retires an armed beat on reset without touching the interval', async () => {
+    const { onLost, onOutcome, subject } = heartbeat();
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+    subject.reset();
+    await elapse(TIMEOUT_MS);
+
+    expect(outcomes(onOutcome)).toEqual([]);
+    expect(onLost).not.toHaveBeenCalled();
+    expect(subject.account).toMatchObject({ failures: 0, sent: 0, unanswered: 0 });
+
+    // Still beating: reset is not a stop.
+    await elapse(INTERVAL_MS);
+    expect(subject.account).toMatchObject({ sent: 1 });
+    subject.stop();
+  });
+
+  // One heartbeat outlives the connections it beats for, and a superseded
+  // connection's teardown arrives late. If its stop were not bound to the
+  // cadence it started, it would take the live connection's beat down with it -
+  // silently, since nothing re-arms an interval that was never meant to stop.
+  it('ignores a stop belonging to a cadence that has been replaced', async () => {
+    const { beat, subject } = heartbeat({ timeoutMs: 10_000 });
+
+    const stopFirst = subject.start();
+    subject.start();
+    stopFirst();
+
+    await elapse(INTERVAL_MS);
+
+    expect(beat).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  it('stops the cadence it started when nothing has replaced it', async () => {
+    const { beat, subject } = heartbeat({ timeoutMs: 10_000 });
+
+    const stop = subject.start();
+    stop();
+
+    await elapse(INTERVAL_MS * 2);
+
+    expect(beat).not.toHaveBeenCalled();
+  });
+
+  it('replaces a running cadence rather than leaving two beating', async () => {
+    const { beat, subject } = heartbeat({ timeoutMs: 10_000 });
+
+    subject.start();
+    subject.start();
+
+    await elapse(INTERVAL_MS);
+
+    expect(beat).toHaveBeenCalledTimes(1);
+    subject.stop();
+  });
+
+  it('sends nothing before it is started or after it is stopped', async () => {
+    const { beat, subject } = heartbeat();
+
+    await elapse(INTERVAL_MS * 2);
+    expect(beat).not.toHaveBeenCalled();
+
+    subject.start();
+    subject.stop();
+    await elapse(INTERVAL_MS * 2);
+
+    expect(beat).not.toHaveBeenCalled();
+  });
+});