@evomap/evolver 1.89.11 → 1.89.12

package/src/proxy/sync/engine.js

@@ -77,12 +77,14 @@ class SyncEngine {
       // the whole tick, schedule the next iteration in `finally` so a
       // surprise throw cannot park the loop.
       let nextDelay = DEFAULT_OUTBOUND_INTERVAL;
+      let hubRetryAfterMs = 0;
       try {
         if (!this._running) return;
         this._outPending = true;
         try {
           const result = await this.outbound.flush();
           if (result.sent > 0) this._lastActivity = Date.now();
+          if (result.retryAfterMs > 0) hubRetryAfterMs = result.retryAfterMs;
           if ((result.sent > 0 || result.dropped > 0) && typeof this.onOutboundFlushed === 'function') {
             try { this.onOutboundFlushed(result); } catch (e) {
               this.logger.warn?.('[sync] onOutboundFlushed callback failed:', e.message);
@@ -96,13 +98,17 @@ class SyncEngine {
           }
         }
         this._outPending = false;
-        try {
-          const pending = this.store.countPending({ direction: 'outbound' });
-          if (pending > 0) nextDelay = 1_000;
-        } catch (err) {
-          // countPending threw (corrupt store, FS hiccup): keep the
-          // default cadence rather than parking the loop.
-          this.logger.error(`[sync] countPending threw (non-fatal): ${err && err.message}`);
+        if (hubRetryAfterMs > 0) {
+          nextDelay = Math.max(1_000, hubRetryAfterMs);
+        } else {
+          try {
+            const pending = this.store.countPending({ direction: 'outbound' });
+            if (pending > 0) nextDelay = 1_000;
+          } catch (err) {
+            // countPending threw (corrupt store, FS hiccup): keep the
+            // default cadence rather than parking the loop.
+            this.logger.error(`[sync] countPending threw (non-fatal): ${err && err.message}`);
+          }
         }
       } catch (err) {
         // Anything that escaped the inner blocks above. Log and let
@@ -123,11 +129,14 @@ class SyncEngine {
       // from inbound.pull / ackDelivered / _isIdle used to escape the
       // setTimeout callback and silently park the inbound loop.
       let nextDelay = DEFAULT_POLL_INTERVAL_ACTIVE;
+      let hubRetryAfterMs = 0;
       try {
         if (!this._running) return;
         try {
           const result = await this.inbound.pull();
-          if (result.received > 0) {
+          if (result.retryAfterMs > 0) {
+            hubRetryAfterMs = result.retryAfterMs;
+          } else if (result.received > 0) {
             this._lastActivity = Date.now();
             if (typeof this.onInboundReceived === 'function') {
               try { this.onInboundReceived(result.received); } catch (e) {
@@ -135,7 +144,10 @@ class SyncEngine {
               }
             }
           }
-          await this.inbound.ackDelivered();
+          if (!hubRetryAfterMs) {
+            const ack = await this.inbound.ackDelivered();
+            if (ack.retryAfterMs > 0) hubRetryAfterMs = ack.retryAfterMs;
+          }
         } catch (err) {
           if (err instanceof AuthError) {
             await this._handleAuthError('inbound');
@@ -143,12 +155,16 @@ class SyncEngine {
             this.logger.error(`[sync] inbound error: ${err.message}`);
           }
         }
-        try {
-          nextDelay = this._isIdle()
-            ? DEFAULT_POLL_INTERVAL_IDLE
-            : DEFAULT_POLL_INTERVAL_ACTIVE;
-        } catch (err) {
-          this.logger.error(`[sync] _isIdle threw (non-fatal): ${err && err.message}`);
+        if (hubRetryAfterMs > 0) {
+          nextDelay = Math.max(1_000, hubRetryAfterMs);
+        } else {
+          try {
+            nextDelay = this._isIdle()
+              ? DEFAULT_POLL_INTERVAL_IDLE
+              : DEFAULT_POLL_INTERVAL_ACTIVE;
+          } catch (err) {
+            this.logger.error(`[sync] _isIdle threw (non-fatal): ${err && err.message}`);
+          }
         }
       } catch (err) {
         this.logger.error(`[sync] inbound tick threw (non-fatal): ${err && err.message}`);

package/src/proxy/sync/inbound.js

@@ -2,7 +2,15 @@
 
 const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
 const { AuthError } = require('../lifecycle/manager');
-const { hubFetch } = require('../../gep/hubFetch');
+const {
+  drainHubResponse,
+  hubFetch,
+  hubUnreachableBackoffMs,
+  isHubUnreachableError,
+  readHubResponseJson,
+  readHubResponseText,
+  throwIfHubUnreachableResponse,
+} = require('../../gep/hubFetch');
 
 const DEFAULT_POLL_INTERVAL_ACTIVE = 10_000;
 const DEFAULT_POLL_INTERVAL_IDLE = 60_000;
@@ -13,9 +21,41 @@ class InboundSync {
     this.hubUrl = hubUrl;
     this.logger = logger || console;
     this.getHeaders = getHeaders;
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _hubUnreachableWaitMs() {
+    return Math.max(0, this._hubUnreachableUntil - Date.now());
+  }
+
+  _recordHubReachable() {
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _recordHubUnreachable(err) {
+    this._hubUnreachableFailures += 1;
+    const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
+    this._hubUnreachableUntil = Date.now() + retryAfterMs;
+    this.logger.warn?.(
+      `[inbound] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
+        `${err && err.message || err}`
+    );
+    return retryAfterMs;
   }
 
   async pull(channel = 'evomap-hub', limit = 50) {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        received: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const cursorKey = `${channel}:inbound_cursor`;
     const cursor = this.store.getCursor(cursorKey);
 
@@ -30,17 +70,20 @@ class InboundSync {
         signal: AbortSignal.timeout(35_000),
       });
 
+      await throwIfHubUnreachableResponse(res, 'inbound pull');
+      this._recordHubReachable();
+
       if (res.status === 403 || res.status === 401) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
       }
 
       if (!res.ok) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new Error(`Hub returned ${res.status}: ${errText}`);
       }
 
-      const data = await res.json();
+      const data = await readHubResponseJson(res);
       const messages = data.messages || [];
 
       if (messages.length > 0) {
@@ -64,12 +107,31 @@ class InboundSync {
       return { received: messages.length, cursor: data.next_cursor || cursor };
     } catch (err) {
       if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        return {
+          received: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+      }
       this.logger.error(`[inbound] pull failed: ${err.message}`);
       return { received: 0, error: err.message };
     }
   }
 
   async ackDelivered(channel = 'evomap-hub') {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        acked: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const delivered = this.store.list({
       type: '%',
       direction: 'inbound',
@@ -97,13 +159,29 @@ class InboundSync {
         body: JSON.stringify({ sender_id: senderId, message_ids: delivered.map(m => m.id) }),
         signal: AbortSignal.timeout(10_000),
       });
-      try {
-        if (res && res.body && typeof res.body.cancel === 'function') {
-          await res.body.cancel().catch(() => {});
-        }
-      } catch (_) { /* never escape the drain helper */ }
+      await throwIfHubUnreachableResponse(res, 'inbound ack');
+      this._recordHubReachable();
+      if (res.status === 403 || res.status === 401) {
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
+        throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
+      }
+      if (!res.ok) {
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
+        throw new Error(`Hub returned ${res.status}: ${errText}`);
+      }
+      await drainHubResponse(res);
       return { acked: delivered.length };
     } catch (err) {
+      if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        return {
+          acked: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+      }
       this.logger.error(`[inbound] ack failed: ${err.message}`);
       return { acked: 0, error: err.message };
     }

package/src/proxy/sync/outbound.js

@@ -3,7 +3,14 @@
 const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
 const { AuthError } = require('../lifecycle/manager');
 const { isProxyTraceUploadPayloadAllowed, resolveTraceMode } = require('../trace/extractor');
-const { hubFetch } = require('../../gep/hubFetch');
+const {
+  hubFetch,
+  hubUnreachableBackoffMs,
+  isHubUnreachableError,
+  readHubResponseJson,
+  readHubResponseText,
+  throwIfHubUnreachableResponse,
+} = require('../../gep/hubFetch');
 
 const MAX_BATCH = 50;
 const MAX_RETRIES = 10;
@@ -14,9 +21,41 @@ class OutboundSync {
     this.hubUrl = hubUrl;
     this.logger = logger || console;
     this.getHeaders = getHeaders;
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _hubUnreachableWaitMs() {
+    return Math.max(0, this._hubUnreachableUntil - Date.now());
+  }
+
+  _recordHubReachable() {
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _recordHubUnreachable(err) {
+    this._hubUnreachableFailures += 1;
+    const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
+    this._hubUnreachableUntil = Date.now() + retryAfterMs;
+    this.logger.warn?.(
+      `[outbound] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
+        `${err && err.message || err}`
+    );
+    return retryAfterMs;
   }
 
   async flush(channel = 'evomap-hub') {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        sent: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const pendingBatch = this.store.pollOutbound({ channel, limit: MAX_BATCH });
     if (pendingBatch.length === 0) return { sent: 0 };
 
@@ -65,17 +104,20 @@ class OutboundSync {
         signal: AbortSignal.timeout(30_000),
       });
 
+      await throwIfHubUnreachableResponse(res, 'outbound flush');
+      this._recordHubReachable();
+
       if (res.status === 403 || res.status === 401) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
       }
 
       if (!res.ok) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new Error(`Hub returned ${res.status}: ${errText}`);
       }
 
-      const data = await res.json();
+      const data = await readHubResponseJson(res);
       const results = data.results || [];
 
       const updates = [];
@@ -112,6 +154,17 @@ class OutboundSync {
       return result;
     } catch (err) {
       if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        const result = {
+          sent: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+        if (dropped > 0) result.dropped = dropped;
+        return result;
+      }
       this.logger.error(`[outbound] flush failed: ${err.message}`);
       for (const m of pending) {
         this.store.incrementRetry(m.id, err.message);