@inetafrica/open-claudia 3.1.27 → 3.2.0

package/test-telegram-poll-recovery.js

@@ -623,6 +623,270 @@ async function testOutagePauseDoesNotFeedTheWedgeExitClock() {
   }
 }
 
+// The fleet reboot loop, reproduced. Everything above stubs the poller; this
+// harness MODELS it, because the bug lived in the interaction the stubs elided:
+// _stopPollingClean destroys the agent, and in production that kills the
+// in-flight getUpdates and re-enters _onPollingHiccup ~1ms later. A stub whose
+// destroy() is a no-op can never show that, which is exactly why v3.1.27's unit
+// test passed against a bug it did not fix (it drove _enterOutagePause, a path
+// the dominant failure never takes). Here destroy() raises the error for real,
+// a poll takes the full 30s hold to complete, and a virtual clock runs the
+// burst past the 10-minute wedge-exit budget.
+function makeVirtualPoller(clock) {
+  const state = { polling: null, pendingPoll: null, onSelfAbort: null, startCount: 0 };
+  const bot = {
+    _polling: null,
+    async stopPolling() {
+      // A plain stop only sets the abort flag — the HTTP request is still open
+      // on the wire. It is _agent.destroy() below that actually kills it, which
+      // is precisely why the resulting error looks like a network fault.
+      if (state.polling) state.polling._abort = true;
+    },
+    async startPolling() {
+      state.startCount += 1;
+      state.polling = { _abort: false, _lastUpdate: 0 }; // fresh object ⇒ _lastUpdate resets to 0
+      bot._polling = state.polling;
+      const mine = state.polling;
+      // A healthy getUpdates completes only after the full 30s long-poll hold.
+      state.pendingPoll = clock.at(clock.now() + 30000, () => {
+        state.pendingPoll = null;
+        if (!mine._abort) mine._lastUpdate = clock.now();
+      });
+    },
+  };
+  const agent = {
+    destroy() {
+      // The real behaviour: tearing the socket down surfaces as a polling error.
+      // No re-dial here — a plain stop set _abort, so the lib's loop does not
+      // reschedule; the heal's own startPolling is what brings polling back.
+      if (state.pendingPoll) {
+        clock.cancel(state.pendingPoll);
+        state.pendingPoll = null;
+        if (state.onSelfAbort) state.onSelfAbort("EFATAL: Error: socket hang up");
+      }
+    },
+  };
+  // A poll that dies on its own (a genuine network fault, _abort still false).
+  // node-telegram-bot-api re-dials within ~300ms, which is why a request is
+  // ALWAYS back in flight by the time a heal runs — and therefore why the heal
+  // always had something to destroy. The live capture shows exactly this:
+  // reqPending was true at every heal:start.
+  state.crash = (msg) => {
+    if (state.pendingPoll) { clock.cancel(state.pendingPoll); state.pendingPoll = null; }
+    if (state.onSelfAbort) state.onSelfAbort(msg);
+    const mine = state.polling;
+    state.pendingPoll = clock.at(clock.now() + 300, () => {
+      state.pendingPoll = clock.at(clock.now() + 30000, () => {
+        state.pendingPoll = null;
+        if (mine && !mine._abort) mine._lastUpdate = clock.now();
+      });
+    });
+  };
+  return { bot, agent, state };
+}
+
+function makeClock(start) {
+  let now = start;
+  let seq = 0;
+  const timers = new Map();
+  return {
+    now: () => now,
+    at(when, fn) { const id = ++seq; timers.set(id, { when, fn }); return id; },
+    cancel(id) { timers.delete(id); },
+    pending: () => timers.size,
+    async advanceTo(target) {
+      for (;;) {
+        let pick = null;
+        for (const [id, t] of timers) {
+          if (t.when > target) continue;
+          if (!pick || t.when < pick.t.when || (t.when === pick.t.when && id < pick.id)) pick = { id, t };
+        }
+        if (!pick) break;
+        timers.delete(pick.id);
+        now = Math.max(now, pick.t.when);
+        await pick.t.fn();
+      }
+      now = target;
+    },
+  };
+}
+
+async function testOneNetworkBlipDoesNotSpiralIntoTheWedgeExitLoop() {
+  const clock = makeClock(1_700_000_000_000);
+  const { bot, agent, state } = makeVirtualPoller(clock);
+
+  const adapter = Object.create(TelegramAdapter.prototype);
+  adapter._healTimer = null;
+  adapter._healthyTimer = null;
+  adapter._healthyBaseline = 0;
+  adapter._healthyReArmMsg = "";
+  adapter._selfAbortUntil = 0;
+  adapter._wedgedSince = 0;
+  adapter._healBackoff = 0;
+  adapter._deferredExitLogAt = 0;
+  adapter._lastHiccupLog = 0;
+  adapter._pausedForOutage = false;
+  adapter._outageProbeTimer = null;
+  adapter._conflictSince = 0;
+  adapter._conflictTimer = null;
+  adapter._pollStartedAt = 0;
+  adapter._anyTurnActive = () => false;
+  adapter._probeTelegramReachable = async () => true; // Telegram was reachable throughout the real burst
+  adapter._agent = agent;
+  adapter.bot = bot;
+  state.onSelfAbort = (msg) => adapter._onPollingHiccup(msg);
+
+  let exited = false;
+  const realExit = process.exit;
+  const realNow = Date.now;
+  const realSetTimeout = global.setTimeout;
+  const realClearTimeout = global.clearTimeout;
+  Date.now = () => clock.now();
+  process.exit = () => { exited = true; throw new Error("__exit__"); };
+  global.setTimeout = (fn, ms) => {
+    if (ms === 500 || ms === 5000) { fn(); return { inline: true }; } // stop-clean internals
+    return clock.at(clock.now() + ms, fn);
+  };
+  global.clearTimeout = (id) => { if (id && !id.inline) clock.cancel(id); };
+
+  try {
+    await bot.startPolling();                       // healthy, mid long-poll hold
+    await clock.advanceTo(clock.now() + 1000);
+    // ONE genuine socket reset — the only external event in the captured burst.
+    state.crash("EFATAL: Error: read ECONNRESET");
+    assert.ok(adapter._wedgedSince > 0, "a real error must open the wedge window");
+
+    // Run well past the 10-minute wedge-exit budget.
+    for (let i = 0; i < 40 && !exited; i += 1) {
+      try { await clock.advanceTo(clock.now() + 30000); }
+      catch (e) { if (e.message !== "__exit__") throw e; }
+    }
+
+    assert.strictEqual(exited, false,
+      "a single ECONNRESET must not end in process.exit — the heal's own teardown was " +
+      "re-triggering the heal, so the wedge clock never cleared and every burst ran to " +
+      "the 10-min backstop (155 self-exits observed locally, 210+ per pod in the fleet)");
+    assert.strictEqual(adapter._wedgedSince, 0,
+      "once a getUpdates completes again the wedge must clear");
+    assert.strictEqual(adapter._healBackoff, 0, "recovery must reset the heal ladder");
+    assert.ok(bot._polling._lastUpdate > 0,
+      "the poll loop must actually complete a getUpdates — pre-fix _lastUpdate stayed 0 forever " +
+      "because each heal aborted the 30s long-poll after 15s");
+    assert.ok(state.startCount <= 4,
+      `recovery must take a couple of restarts, not a storm (took ${state.startCount})`);
+  } finally {
+    Date.now = realNow;
+    process.exit = realExit;
+    global.setTimeout = realSetTimeout;
+    global.clearTimeout = realClearTimeout;
+  }
+}
+
+// Fix 1 in isolation: the "socket hang up" our own _agent.destroy() raises must
+// not schedule a heal. Pre-fix this was the engine of the loop.
+async function testOurOwnTeardownIsNotCountedAsAFault() {
+  const clock = makeClock(1_700_000_000_000);
+  const { bot, agent, state } = makeVirtualPoller(clock);
+
+  const adapter = Object.create(TelegramAdapter.prototype);
+  Object.assign(adapter, {
+    _healTimer: null, _healthyTimer: null, _healthyBaseline: 0, _healthyReArmMsg: "",
+    _selfAbortUntil: 0, _wedgedSince: 0, _healBackoff: 0, _deferredExitLogAt: 0,
+    _lastHiccupLog: 0, _pausedForOutage: false, _outageProbeTimer: null,
+    _conflictSince: 0, _conflictTimer: null, _pollStartedAt: 0,
+    _anyTurnActive: () => false, _probeTelegramReachable: async () => true,
+    _agent: agent, bot,
+  });
+
+  const scheduled = [];
+  const realNow = Date.now;
+  const realSetTimeout = global.setTimeout;
+  const realClearTimeout = global.clearTimeout;
+  Date.now = () => clock.now();
+  global.setTimeout = (fn, ms) => {
+    if (ms === 500 || ms === 5000) { fn(); return { inline: true }; }
+    scheduled.push(ms);
+    return clock.at(clock.now() + ms, fn);
+  };
+  global.clearTimeout = (id) => { if (id && !id.inline) clock.cancel(id); };
+
+  let selfAbortSeen = 0;
+  state.onSelfAbort = (msg) => { selfAbortSeen += 1; adapter._onPollingHiccup(msg); };
+
+  try {
+    await bot.startPolling();
+    await clock.advanceTo(clock.now() + 1000);
+    adapter._wedgedSince = clock.now();      // already in a wedge, as when a heal runs
+    scheduled.length = 0;
+    await adapter._heal();                   // tears the socket down → raises our own error
+
+    assert.strictEqual(selfAbortSeen, 1, "the harness must actually raise the teardown error");
+    assert.deepStrictEqual(scheduled, [35000],
+      "the only timer a heal may arm is its 35s healthy check — pre-fix the teardown error " +
+      "also queued another heal, which tore down the socket again, forever");
+    assert.ok(clock.now() < adapter._selfAbortUntil,
+      "the self-abort window must be open across the restart");
+  } finally {
+    Date.now = realNow;
+    global.setTimeout = realSetTimeout;
+    global.clearTimeout = realClearTimeout;
+  }
+}
+
+// Fix 2: the heal ladder must out-wait one getUpdates. A cap below the poll
+// hold makes recovery unobservable — _lastUpdate can never advance.
+function testHealLadderOutWaitsTheLongPoll() {
+  const adapter = Object.create(TelegramAdapter.prototype);
+  adapter._healBackoff = 0;
+  const delays = [];
+  const realSetTimeout = global.setTimeout;
+  global.setTimeout = (fn, ms) => { delays.push(ms); return { ms }; };
+  try {
+    for (let i = 0; i < 10; i += 1) { adapter._healTimer = null; adapter._scheduleHeal(); }
+  } finally {
+    global.setTimeout = realSetTimeout;
+  }
+  const settled = delays[delays.length - 1];
+  assert.ok(settled > 40000,
+    `the settled heal interval (${settled}ms) must exceed the 40s request timeout that bounds ` +
+    "one getUpdates; the old 15s cap was HALF the 30s long-poll hold, so every attempt was " +
+    "aborted before it could complete and recovery was structurally impossible");
+  assert.ok(delays[0] <= 1000, "early rungs must still retry fast while polling is down");
+}
+
+// Fix 3: re-arming must not push the deadline back, or a heal cadence shorter
+// than the check window starves the only code that clears the wedge.
+function testHealthyCheckCannotBeStarvedByRepeatedReArms() {
+  const adapter = Object.create(TelegramAdapter.prototype);
+  adapter._healthyTimer = null;
+  adapter._healthyBaseline = 0;
+  adapter._healthyReArmMsg = "";
+
+  let armCount = 0;
+  let cleared = 0;
+  const realSetTimeout = global.setTimeout;
+  const realClearTimeout = global.clearTimeout;
+  global.setTimeout = (fn, ms) => { armCount += 1; return { ms, fn }; };
+  global.clearTimeout = () => { cleared += 1; };
+  try {
+    adapter._armHealthyCheck(1000, "first");
+    adapter._armHealthyCheck(2000, "second");
+    adapter._armHealthyCheck(3000, "third");
+  } finally {
+    global.setTimeout = realSetTimeout;
+    global.clearTimeout = realClearTimeout;
+  }
+
+  assert.strictEqual(armCount, 1,
+    "the pending healthy check must keep its original deadline — re-arming it on every heal " +
+    "is why zero healthy checks fired in a 200s+ live capture");
+  assert.strictEqual(cleared, 0, "a pending healthy check must never be cancelled by a re-arm");
+  assert.strictEqual(adapter._healthyBaseline, 3000,
+    "the baseline must track the NEWEST restart, so a poll completed before it cannot be " +
+    "mistaken for proof that the latest restart worked");
+  assert.strictEqual(adapter._healthyReArmMsg, "third", "the re-arm reason must track the newest restart too");
+}
+
 (async () => {
   await testStartDoesNotWaitForFirstLongPollAndArmsStallWatchdog();
   await testHealStopsPlainlyDestroysAgentAndRestartsIdle();
@@ -636,6 +900,10 @@ async function testOutagePauseDoesNotFeedTheWedgeExitClock() {
   await testHealthySignalReArmsWhenNoPollCompleted();
   await testHealthySignalClearsWedgeOnceAPollCompletes();
   await testOutagePauseDoesNotFeedTheWedgeExitClock();
+  await testOneNetworkBlipDoesNotSpiralIntoTheWedgeExitLoop();
+  await testOurOwnTeardownIsNotCountedAsAFault();
+  testHealLadderOutWaitsTheLongPoll();
+  testHealthyCheckCannotBeStarvedByRepeatedReArms();
   console.log("telegram poll recovery OK");
 })().catch((err) => {
   console.error(err.stack || err);