@evomap/evolver 1.89.11 → 1.89.13

package/src/proxy/server/routes.js

@@ -100,6 +100,29 @@ function buildRoutes(store, proxyHandlers, taskMonitor, extensions) {
       return { body: result };
     },
 
+    'POST /conversation/distill': async ({ body }) => {
+      const { distillConversation } = require('../../gep/conversationDistiller');
+      const input = body || {};
+      const distill = distillConversation(input, { persist: input.persist !== false });
+      if (!distill.ok) return { body: distill };
+
+      let publishResult = null;
+      if (input.publish !== false) {
+        publishResult = await proxyHandlers.assetPublish({
+          assets: [distill.gene, distill.capsule].filter(Boolean),
+        });
+      }
+
+      return {
+        body: {
+          ...distill,
+          queued: false,
+          published: Boolean(publishResult),
+          publish_result: publishResult,
+        },
+      };
+    },
+
     'GET /asset/submissions': async ({ query }) => {
       const submissions = store.list({
         type: 'asset_submit',
@@ -378,6 +401,10 @@ function buildRoutes(store, proxyHandlers, taskMonitor, extensions) {
       const inPending = store.countPending({ direction: 'inbound' });
       const nodeId = store.getState('node_id');
       const lastSync = store.getState('last_sync_at');
+      const lastSyncError = store.getState('last_sync_error');
+      const hubAuthStatus = store.getState('hub_auth_status');
+      const reauthBackoffUntil = store.getState('reauth_backoff_until');
+      const helloRateLimitUntil = store.getState('hello_rate_limit_until');
       return {
         body: {
           status: 'running',
@@ -387,6 +414,10 @@ function buildRoutes(store, proxyHandlers, taskMonitor, extensions) {
           outbound_pending: outPending,
           inbound_pending: inPending,
           last_sync_at: lastSync,
+          last_sync_error: lastSyncError || null,
+          hub_auth_status: hubAuthStatus || null,
+          reauth_backoff_until: reauthBackoffUntil || null,
+          hello_rate_limit_until: helloRateLimitUntil || null,
         },
       };
     },

package/src/proxy/server/settings.js

@@ -47,15 +47,19 @@ function writeSettings(data) {
   return merged;
 }
 
-function clearSettings() {
+function clearSettings(opts = {}) {
   try {
     const file = getSettingsFile();
     if (fs.existsSync(file)) {
       const current = readSettings();
+      const proxyPid = current.proxy?.pid;
+      if (!opts.force && proxyPid && proxyPid !== process.pid) return false;
       delete current.proxy;
       fs.writeFileSync(file, JSON.stringify(current, null, 2), 'utf8');
+      return true;
     }
   } catch {}
+  return false;
 }
 
 function isStaleProxy() {
@@ -82,7 +86,7 @@ function isStaleProxy() {
 
 function clearIfStale() {
   if (isStaleProxy()) {
-    clearSettings();
+    clearSettings({ force: true });
     return true;
   }
   return false;

package/src/proxy/sync/engine.js

@@ -77,12 +77,14 @@ class SyncEngine {
       // the whole tick, schedule the next iteration in `finally` so a
       // surprise throw cannot park the loop.
       let nextDelay = DEFAULT_OUTBOUND_INTERVAL;
+      let hubRetryAfterMs = 0;
       try {
         if (!this._running) return;
         this._outPending = true;
         try {
           const result = await this.outbound.flush();
           if (result.sent > 0) this._lastActivity = Date.now();
+          if (result.retryAfterMs > 0) hubRetryAfterMs = result.retryAfterMs;
           if ((result.sent > 0 || result.dropped > 0) && typeof this.onOutboundFlushed === 'function') {
             try { this.onOutboundFlushed(result); } catch (e) {
               this.logger.warn?.('[sync] onOutboundFlushed callback failed:', e.message);
@@ -96,13 +98,17 @@ class SyncEngine {
           }
         }
         this._outPending = false;
-        try {
-          const pending = this.store.countPending({ direction: 'outbound' });
-          if (pending > 0) nextDelay = 1_000;
-        } catch (err) {
-          // countPending threw (corrupt store, FS hiccup): keep the
-          // default cadence rather than parking the loop.
-          this.logger.error(`[sync] countPending threw (non-fatal): ${err && err.message}`);
+        if (hubRetryAfterMs > 0) {
+          nextDelay = Math.max(1_000, hubRetryAfterMs);
+        } else {
+          try {
+            const pending = this.store.countPending({ direction: 'outbound' });
+            if (pending > 0) nextDelay = 1_000;
+          } catch (err) {
+            // countPending threw (corrupt store, FS hiccup): keep the
+            // default cadence rather than parking the loop.
+            this.logger.error(`[sync] countPending threw (non-fatal): ${err && err.message}`);
+          }
         }
       } catch (err) {
         // Anything that escaped the inner blocks above. Log and let
@@ -123,11 +129,14 @@ class SyncEngine {
       // from inbound.pull / ackDelivered / _isIdle used to escape the
       // setTimeout callback and silently park the inbound loop.
       let nextDelay = DEFAULT_POLL_INTERVAL_ACTIVE;
+      let hubRetryAfterMs = 0;
       try {
         if (!this._running) return;
         try {
           const result = await this.inbound.pull();
-          if (result.received > 0) {
+          if (result.retryAfterMs > 0) {
+            hubRetryAfterMs = result.retryAfterMs;
+          } else if (result.received > 0) {
             this._lastActivity = Date.now();
             if (typeof this.onInboundReceived === 'function') {
               try { this.onInboundReceived(result.received); } catch (e) {
@@ -135,7 +144,10 @@ class SyncEngine {
               }
             }
           }
-          await this.inbound.ackDelivered();
+          if (!hubRetryAfterMs) {
+            const ack = await this.inbound.ackDelivered();
+            if (ack.retryAfterMs > 0) hubRetryAfterMs = ack.retryAfterMs;
+          }
         } catch (err) {
           if (err instanceof AuthError) {
             await this._handleAuthError('inbound');
@@ -143,12 +155,16 @@ class SyncEngine {
             this.logger.error(`[sync] inbound error: ${err.message}`);
           }
         }
-        try {
-          nextDelay = this._isIdle()
-            ? DEFAULT_POLL_INTERVAL_IDLE
-            : DEFAULT_POLL_INTERVAL_ACTIVE;
-        } catch (err) {
-          this.logger.error(`[sync] _isIdle threw (non-fatal): ${err && err.message}`);
+        if (hubRetryAfterMs > 0) {
+          nextDelay = Math.max(1_000, hubRetryAfterMs);
+        } else {
+          try {
+            nextDelay = this._isIdle()
+              ? DEFAULT_POLL_INTERVAL_IDLE
+              : DEFAULT_POLL_INTERVAL_ACTIVE;
+          } catch (err) {
+            this.logger.error(`[sync] _isIdle threw (non-fatal): ${err && err.message}`);
+          }
         }
       } catch (err) {
         this.logger.error(`[sync] inbound tick threw (non-fatal): ${err && err.message}`);

package/src/proxy/sync/inbound.js

@@ -2,7 +2,15 @@
 
 const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
 const { AuthError } = require('../lifecycle/manager');
-const { hubFetch } = require('../../gep/hubFetch');
+const {
+  drainHubResponse,
+  hubFetch,
+  hubUnreachableBackoffMs,
+  isHubUnreachableError,
+  readHubResponseJson,
+  readHubResponseText,
+  throwIfHubUnreachableResponse,
+} = require('../../gep/hubFetch');
 
 const DEFAULT_POLL_INTERVAL_ACTIVE = 10_000;
 const DEFAULT_POLL_INTERVAL_IDLE = 60_000;
@@ -13,9 +21,41 @@ class InboundSync {
     this.hubUrl = hubUrl;
     this.logger = logger || console;
     this.getHeaders = getHeaders;
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _hubUnreachableWaitMs() {
+    return Math.max(0, this._hubUnreachableUntil - Date.now());
+  }
+
+  _recordHubReachable() {
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _recordHubUnreachable(err) {
+    this._hubUnreachableFailures += 1;
+    const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
+    this._hubUnreachableUntil = Date.now() + retryAfterMs;
+    this.logger.warn?.(
+      `[inbound] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
+        `${err && err.message || err}`
+    );
+    return retryAfterMs;
   }
 
   async pull(channel = 'evomap-hub', limit = 50) {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        received: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const cursorKey = `${channel}:inbound_cursor`;
     const cursor = this.store.getCursor(cursorKey);
 
@@ -30,17 +70,20 @@ class InboundSync {
         signal: AbortSignal.timeout(35_000),
       });
 
+      await throwIfHubUnreachableResponse(res, 'inbound pull');
+      this._recordHubReachable();
+
       if (res.status === 403 || res.status === 401) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
       }
 
       if (!res.ok) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new Error(`Hub returned ${res.status}: ${errText}`);
       }
 
-      const data = await res.json();
+      const data = await readHubResponseJson(res);
       const messages = data.messages || [];
 
       if (messages.length > 0) {
@@ -64,12 +107,31 @@ class InboundSync {
       return { received: messages.length, cursor: data.next_cursor || cursor };
     } catch (err) {
       if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        return {
+          received: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+      }
       this.logger.error(`[inbound] pull failed: ${err.message}`);
       return { received: 0, error: err.message };
     }
   }
 
   async ackDelivered(channel = 'evomap-hub') {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        acked: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const delivered = this.store.list({
       type: '%',
       direction: 'inbound',
@@ -97,13 +159,29 @@ class InboundSync {
         body: JSON.stringify({ sender_id: senderId, message_ids: delivered.map(m => m.id) }),
         signal: AbortSignal.timeout(10_000),
       });
-      try {
-        if (res && res.body && typeof res.body.cancel === 'function') {
-          await res.body.cancel().catch(() => {});
-        }
-      } catch (_) { /* never escape the drain helper */ }
+      await throwIfHubUnreachableResponse(res, 'inbound ack');
+      this._recordHubReachable();
+      if (res.status === 403 || res.status === 401) {
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
+        throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
+      }
+      if (!res.ok) {
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
+        throw new Error(`Hub returned ${res.status}: ${errText}`);
+      }
+      await drainHubResponse(res);
       return { acked: delivered.length };
     } catch (err) {
+      if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        return {
+          acked: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+      }
       this.logger.error(`[inbound] ack failed: ${err.message}`);
       return { acked: 0, error: err.message };
     }

package/src/proxy/sync/outbound.js

@@ -3,7 +3,14 @@
 const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
 const { AuthError } = require('../lifecycle/manager');
 const { isProxyTraceUploadPayloadAllowed, resolveTraceMode } = require('../trace/extractor');
-const { hubFetch } = require('../../gep/hubFetch');
+const {
+  hubFetch,
+  hubUnreachableBackoffMs,
+  isHubUnreachableError,
+  readHubResponseJson,
+  readHubResponseText,
+  throwIfHubUnreachableResponse,
+} = require('../../gep/hubFetch');
 
 const MAX_BATCH = 50;
 const MAX_RETRIES = 10;
@@ -14,9 +21,41 @@ class OutboundSync {
     this.hubUrl = hubUrl;
     this.logger = logger || console;
     this.getHeaders = getHeaders;
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _hubUnreachableWaitMs() {
+    return Math.max(0, this._hubUnreachableUntil - Date.now());
+  }
+
+  _recordHubReachable() {
+    this._hubUnreachableFailures = 0;
+    this._hubUnreachableUntil = 0;
+  }
+
+  _recordHubUnreachable(err) {
+    this._hubUnreachableFailures += 1;
+    const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
+    this._hubUnreachableUntil = Date.now() + retryAfterMs;
+    this.logger.warn?.(
+      `[outbound] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
+        `${err && err.message || err}`
+    );
+    return retryAfterMs;
   }
 
   async flush(channel = 'evomap-hub') {
+    const waitMs = this._hubUnreachableWaitMs();
+    if (waitMs > 0) {
+      return {
+        sent: 0,
+        error: 'hub_unreachable_backoff',
+        hubUnreachable: true,
+        retryAfterMs: waitMs,
+      };
+    }
+
     const pendingBatch = this.store.pollOutbound({ channel, limit: MAX_BATCH });
     if (pendingBatch.length === 0) return { sent: 0 };
 
@@ -65,17 +104,20 @@ class OutboundSync {
         signal: AbortSignal.timeout(30_000),
       });
 
+      await throwIfHubUnreachableResponse(res, 'outbound flush');
+      this._recordHubReachable();
+
       if (res.status === 403 || res.status === 401) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new AuthError(`Hub ${res.status}: ${errText}`, res.status);
       }
 
       if (!res.ok) {
-        const errText = await res.text().catch(() => 'unknown');
+        const errText = await readHubResponseText(res).catch(() => 'unknown');
         throw new Error(`Hub returned ${res.status}: ${errText}`);
       }
 
-      const data = await res.json();
+      const data = await readHubResponseJson(res);
       const results = data.results || [];
 
       const updates = [];
@@ -112,6 +154,17 @@ class OutboundSync {
       return result;
     } catch (err) {
       if (err instanceof AuthError) throw err;
+      if (isHubUnreachableError(err)) {
+        const retryAfterMs = this._recordHubUnreachable(err);
+        const result = {
+          sent: 0,
+          error: 'hub_unreachable',
+          hubUnreachable: true,
+          retryAfterMs,
+        };
+        if (dropped > 0) result.dropped = dropped;
+        return result;
+      }
       this.logger.error(`[outbound] flush failed: ${err.message}`);
       for (const m of pending) {
         this.store.incrementRetry(m.id, err.message);