mixdog 0.9.17 → 0.9.19

package/src/runtime/channels/backends/discord-gateway.mjs

@@ -205,20 +205,43 @@ function registerShardListeners(self) {
 
 async function awaitLogin(self) {
   let readyTimeout;
+  // Adaptive ready timeout: grow on consecutive login failures (30s→60s→90s
+  // cap) so an identify delayed by gateway rate-limiting can still complete
+  // its READY handshake instead of tripping a fixed 30s timeout every retry.
+  const readyMs = Math.min(90_000, 30_000 + (self._loginFailures || 0) * 30_000);
   const readyPromise = new Promise((resolve, reject) => {
-    readyTimeout = setTimeout(() => reject(new Error("discord ready timeout (30s)")), 3e4);
+    readyTimeout = setTimeout(
+      () => reject(new Error(`discord ready timeout (${Math.round(readyMs / 1000)}s)`)),
+      readyMs
+    );
     self.client.once(self._discordEvents().ClientReady, () => {
       clearTimeout(readyTimeout);
       resolve();
     });
   });
+  // Guarantee the ready-timeout rejection can never surface as a process-global
+  // unhandledRejection regardless of how the race below settles: attach an
+  // absorbing handler; the real handling still happens via the awaits.
+  readyPromise.catch(() => {});
+  // Observe the ready timer from the start: login() can hang past the 30s
+  // timer (gateway reconnect), and a rejection on an un-awaited readyPromise
+  // becomes a process-global unhandledRejection → fatal crash path
+  // (channels/index.mjs unhandledRejection → _fatalCrash → exit(1)). Racing
+  // login with readyPromise keeps the rejection inside this connect() chain,
+  // which every caller already catches non-fatally.
+  const loginPromise = self.client.login(self.token);
+  // A late login settlement after the timeout raced ahead must not itself
+  // become a second unhandled rejection.
+  loginPromise.catch(() => {});
   try {
-    await self.client.login(self.token);
+    await Promise.race([loginPromise, readyPromise]);
+    await readyPromise;
+    self._loginFailures = 0;
   } catch (err) {
+    self._loginFailures = (self._loginFailures || 0) + 1;
     clearTimeout(readyTimeout);
     throw err;
   }
-  await readyPromise;
 }
 
 export {

package/src/runtime/channels/backends/discord.mjs

@@ -23,6 +23,20 @@ import { normalizeAccess, safeAttName } from "./discord-access.mjs";
 import { MAX_ATTACHMENT_BYTES, downloadSingleAttachment } from "./discord-attachments.mjs";
 import * as gateway from "./discord-gateway.mjs";
 const MAX_CHUNK_LIMIT = MAX_DISCORD_MESSAGE;
+// Per-attempt cap for a single discord.js fetch/send. Without it a wedged
+// socket (network blip, "other side closed") leaves the await pending forever,
+// which upstream leaves the forwarder's `sending` flag stuck and silences all
+// outbound. On timeout we treat the client as dead and reset+reconnect it.
+const SEND_ATTEMPT_TIMEOUT_MS = 30_000;
+// Reconnect backoff: consecutive _resetClient failures back off exponentially
+// with jitter up to a cap so a wedged/rate-limited gateway is not hammered with
+// fixed sub-minute destroy()/login() cycles (that fixed cadence triggered
+// Discord identify rate-limiting → the ready-timeout death loop).
+const RESET_BACKOFF_BASE_MS = 2_000;
+const RESET_BACKOFF_CAP_MS = 60_000;
+// After a timed-out send is reset, cap how long we wait for the original
+// (abandoned) promise to actually settle before giving up with unknown outcome.
+const SETTLE_CAP_MS = 15_000;
 const RECENT_SENT_CAP = 200;
 class DiscordBackend {
   name = "discord";
@@ -41,7 +55,12 @@ class DiscordBackend {
   initialAccess;
   recentSentIds = /* @__PURE__ */ new Set();
   sendCount = 0;
+  _resetFailures = 0;
+  _loginFailures = 0;
   typingIntervals = /* @__PURE__ */ new Map();
+  // In-flight sendMessage() promises, awaited by drainPendingSends() before a
+  // handoff disconnect so a reply is not cut off mid-delivery.
+  _pendingSends = /* @__PURE__ */ new Set();
   constructor(config, stateDir) {
     this.token = config.token;
     this.mainChannelId = config.mainChannelId ?? "";
@@ -74,11 +93,70 @@ class DiscordBackend {
     if (this.client) this.client.destroy();
     this._connectPromise = null;
   }
+  // ── Self-heal helpers ──────────────────────────────────────────────
+  /** Race a discord.js op-promise against a per-attempt timeout so a wedged
+   *  socket can't hang the outbound path. On timeout the rejection carries
+   *  `_timeout` so the caller knows the underlying op is still in flight (and
+   *  must be settled/adopted, not blindly retried → duplicate). */
+  _withTimeout(promise, label) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+      timer = setTimeout(() => {
+        const e = new Error(`${label} timed out after ${SEND_ATTEMPT_TIMEOUT_MS}ms`);
+        e._timeout = true;
+        reject(e);
+      }, SEND_ATTEMPT_TIMEOUT_MS);
+    });
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+  }
+  /** True when an error means the client is dead (aborted, "other side
+   *  closed", or our per-attempt timeout) and must be reset+reconnected. */
+  _shouldResetClient(err) {
+    if (err?.name === "AbortError" || err?.code === "ABORT_ERR") return true;
+    const msg = String(err?.message || err || "").toLowerCase();
+    return msg.includes("aborted") || msg.includes("other side closed") ||
+      msg.includes("socket hang up") || msg.includes("econnreset") || msg.includes("timed out");
+  }
+  /** Destroy the wedged client and rebuild it via connect() so the next
+   *  fetch/send (and the forwarder's queue-head-preserving retry) hits a fresh
+   *  gateway/REST client. Serialized behind a single in-flight reset so
+   *  concurrent callers await the SAME reconnect instead of racing multiple
+   *  destroy()/login() cycles. */
+  async _resetClient() {
+    if (this._resetPromise) return this._resetPromise;
+    this._resetPromise = (async () => {
+      // Back off before rebuilding: on repeated reset failures wait an
+      // exponentially growing (jittered) delay capped at RESET_BACKOFF_CAP_MS
+      // so we never re-identify in a tight loop.
+      const n = this._resetFailures || 0;
+      if (n > 0) {
+        const cap = Math.min(RESET_BACKOFF_CAP_MS, RESET_BACKOFF_BASE_MS * 2 ** (n - 1));
+        const delay = Math.floor(cap / 2 + Math.random() * (cap / 2));
+        await new Promise((r) => setTimeout(r, delay));
+      }
+      try { this.client?.destroy?.(); } catch {}
+      this.client = null;
+      this._connectPromise = null;
+      await this.connect();
+      this._resetFailures = 0;
+    })().catch((err) => {
+      this._resetFailures = (this._resetFailures || 0) + 1;
+      throw err;
+    }).finally(() => { this._resetPromise = null; });
+    return this._resetPromise;
+  }
+  /** Health-check the gateway; reset+reconnect if the ws is not READY. Called
+   *  by the owner re-claim self-heal path. (discord.js Status.Ready === 0) */
+  async healthCheck() {
+    if (this.client && this.client.ws?.status === 0) return;
+    await this._resetClient();
+  }
   resetSendCount() {
     this.sendCount = 0;
   }
   startTyping(channelId) {
     this.stopTyping(channelId);
+    if (!this.client) return;
     const ch = this.client.channels.cache.get(channelId);
     if (ch && "sendTyping" in ch) {
       void ch.sendTyping().catch(() => {
@@ -100,7 +178,25 @@ class DiscordBackend {
     }
   }
   // ── Outbound operations ────────────────────────────────────────────
+  // Public entry: tracks the send so drainPendingSends() can await in-flight
+  // deliveries before an ownership-handoff disconnect. Delegates to the impl.
   async sendMessage(chatId, text, opts) {
+    const p = this._sendMessageImpl(chatId, text, opts);
+    const tracked = Promise.resolve(p).catch(() => {});
+    this._pendingSends.add(tracked);
+    tracked.finally(() => this._pendingSends.delete(tracked));
+    return p;
+  }
+  /** Await in-flight sends (bounded) so a handoff drains outbound before
+   *  disconnect. Bounded by timeoutMs so a wedged send can't stall teardown. */
+  async drainPendingSends(timeoutMs = 4000) {
+    if (this._pendingSends.size === 0) return;
+    await Promise.race([
+      Promise.allSettled([...this._pendingSends]),
+      new Promise((r) => setTimeout(r, timeoutMs)),
+    ]);
+  }
+  async _sendMessageImpl(chatId, text, opts) {
     const ch = await this.fetchAllowedChannel(chatId);
     if (!("send" in ch)) throw new Error("channel is not sendable");
     const files = opts?.files ?? [];
@@ -172,13 +268,38 @@ class DiscordBackend {
         // resumes at this chunk index instead of re-sending 0..i-1.
         let attempt = 0;
         for (;;) {
+          const sendPromise = ch.send(payload);
           try {
-            const sent = await ch.send(payload);
+            const sent = await this._withTimeout(sendPromise, "discord send");
             this.noteSent(sent.id);
             sentIds.push(sent.id);
             break;
           } catch (err) {
             attempt++;
+            if (err?._timeout) {
+              // Watchdog abandoned the send but the socket may still deliver.
+              // Destroy the client to tear down the in-flight REST socket so the
+              // ORIGINAL promise settles, then AWAIT + adopt its real outcome:
+              // delivered → done (no retry → no duplicate); else requeue against
+              // the fresh client via a resume token.
+              try { await this._resetClient(); } catch {}
+              // Cap the settlement wait: destroy() should abort the in-flight
+              // socket, but never hang here if it doesn't. On unknown outcome
+              // throw retryable (duplicate risk accepted over a permanent hang).
+              sendPromise.catch(() => {}); // swallow a late rejection
+              const settled = await Promise.race([
+                sendPromise.then((v) => ({ ok: true, v }), () => ({ ok: false })),
+                new Promise((r) => setTimeout(() => r({ ok: false }), SETTLE_CAP_MS))
+              ]);
+              if (settled.ok) {
+                this.noteSent(settled.v.id);
+                sentIds.push(settled.v.id);
+                break;
+              }
+              const e = err instanceof Error ? err : new Error(String(err));
+              e.resumeToken = { hash: contentHash, nextChunkIdx: i, sentIds: [...sentIds], prefixed: applyPrefix, limit };
+              throw e;
+            }
             const status = err?.status ?? err?.code ?? err?.httpStatus;
             // Classify: PERMANENT = a 4xx client error (unknown channel/message,
             // missing access/permissions, 404 …) that will never succeed on
@@ -192,7 +313,15 @@ class DiscordBackend {
               e.permanent = true;
               throw e;
             }
-            if (attempt >= 3) {
+            // Dead client (abort / "other side closed" / per-attempt timeout):
+            // destroy + reconnect so the forwarder's transient-retry path hits a
+            // fresh client, and throw a resume token now instead of burning the
+            // remaining attempts on a stale channel handle bound to the old client.
+            const dead = this._shouldResetClient(err);
+            if (dead) {
+              try { await this._resetClient(); } catch {}
+            }
+            if (dead || attempt >= 3) {
               // Attach an opaque resume token pointing at the chunk that failed
               // (i). sentIds holds every chunk already delivered (including any
               // seeded from a prior token).
@@ -301,13 +430,13 @@ class DiscordBackend {
     const msg = await ch.messages.fetch(messageId);
     await msg.delete();
   }
-  async downloadAttachment(chatId, messageId) {
+  async downloadAttachment(chatId, messageId, opts = {}) {
     const ch = await this.fetchAllowedChannel(chatId);
     const msg = await ch.messages.fetch(messageId);
     if (msg.attachments.size === 0) return [];
     const results = [];
     for (const att of msg.attachments.values()) {
-      const path = await this.downloadSingleAttachment(att);
+      const path = await this.downloadSingleAttachment(att, opts);
       results.push({
         id: att.id,
         path,
@@ -526,7 +655,10 @@ class DiscordBackend {
   }
   // ── Channel helpers ────────────────────────────────────────────────
   async fetchTextChannel(id) {
-    const ch = await this.client.channels.fetch(id);
+    if (!this.client) await this.connect();
+    const p = this.client.channels.fetch(id);
+    p.catch(() => {}); // if the watchdog wins, don't leak an unhandled rejection
+    const ch = await this._withTimeout(p, "discord channels.fetch");
     if (!ch || !ch.isTextBased()) {
       throw new Error(`channel ${id} not found or not text-based`);
     }
@@ -571,8 +703,8 @@ class DiscordBackend {
       throw new Error(`refusing to send channel state: ${f}`);
     }
   }
-  async downloadSingleAttachment(att) {
-    return downloadSingleAttachment(att, this.inboxDir);
+  async downloadSingleAttachment(att, opts = {}) {
+    return downloadSingleAttachment(att, this.inboxDir, opts);
   }
 }
 export {