nexus-fca 3.1.4 → 3.1.6

package/docs/memory-usage.md

@@ -0,0 +1,50 @@
+# Nexus-FCA Memory Usage Guide
+
+## Built-in guardrails
+
+- **Outbound MQTT buffers** (`src/sendMessageMqtt.js`, `src/listenMqtt.js`)
+  - `_pendingOutbound` tracks only outstanding message IDs and is pruned immediately when an ACK lands.
+  - Health metrics (`lib/health/HealthMetrics.js`) cap ACK samples at 50 entries and keep averages via exponential smoothing, so telemetry never balloons in RAM.
+- **Pending edit queue** (`src/editMessage.js`)
+  - `globalOptions.editSettings.maxPendingEdits` defaults to 200; oldest edits are dropped once the cap is hit, and TTL checks clear stale entries.
+  - Every ACK removes the corresponding entry, and `HealthMetrics` mirrors the size so you can alert if it starts climbing.
+- **Group send queue** (`index.js`)
+  - Each group’s queue is capped (default 100 messages) and sweeps run every 5 minutes to drop idle or over-capacity queues.
+  - Sweeper stats surface through `ctx.health.recordGroupQueuePrune`, so you can confirm that cleanup is happening.
+- **Performance caches** (`lib/performance/PerformanceManager.js`)
+  - Cache maps are bounded by `cacheSize` (default 1000) and enforce TTL, while request time windows keep only the last 100 samples.
+  - `PerformanceOptimizer` trims its request history to 1000 entries and halves the buffer on each cleanup pass.
+- **Database write queue** (`lib/database/EnhancedDatabase.js`)
+  - Writes batch in chunks of 100 and the queue processor re-runs every second. Long outages are the only way to accumulate large queues.
+- **Safety timers** (`lib/safety/FacebookSafety.js`)
+  - All recurring timers are stored and cleared before new ones are scheduled, preventing runaway intervals during reconnect churn.
+
+## Situations that can increase memory
+
+| Area | Why it grows | Mitigation |
+| --- | --- | --- |
+| Database write queue | Target SQLite/SQL server offline → `writeQueue` keeps buffering | Monitor `writeQueue.length` or add alerts around `EnhancedDatabase.processQueue`; if storage is optional, disable DB integration entirely.
+| Multiple PerformanceManager / PerformanceOptimizer instances | Each instance spawns its own metrics intervals and caches | Treat both managers as singletons; share them via dependency injection instead of `new`ing per feature.
+| Elevated group queue caps | Setting `setGroupQueueCapacity` >> 100 multiplies per-thread memory | Keep caps small; rely on `_flushGroupQueue` for bursts instead of raising the ceiling.
+| Pending edit saturation | Frequent edit retries without ACKs hit the 200-item cap | Investigate upstream failures (usually edit rights or MQTT drops). `api.getMemoryMetrics()` will show `pendingEditsDropped` climbing when this happens.
+| Large custom caches | If you override `cacheSize` or TTLs to very large values, the Map will grow | Pick realistic TTLs; if the workload is mostly transient, disable cache (`enableCache: false`).
+
+## Monitoring checklist
+
+1. **Runtime snapshot** – call `api.getMemoryMetrics()` to read pending edit counts, outbound depth, and memory guard actions.
+2. **Health dashboard** – `ctx.health.snapshot()` (or the API wrapper `getHealthMetrics`) exposes ACK latency samples and queue stats.
+3. **Performance events** – `PerformanceManager` emits `metricsUpdate` every 30s; attach a listener and pipe to your logger or Prometheus bridge.
+4. **Node heap checks** – pair the built-in metrics with `process.memoryUsage()` or `--inspect` tooling if you suspect leaks from user code.
+
+## Configuration tips
+
+- Tune `globalOptions.editSettings` if your bot edits aggressively; lower `maxPendingEdits` to 100 and `editTTLms` to 2–3 minutes for tighter control.
+- Use `api.setGroupQueueCapacity(n)` to keep per-thread queues bounded; the sweeper already limits idle queues to 30 minutes, but lower values (10–20) reduce burst memory further.
+- If you don’t need database analytics, avoid initializing `EnhancedDatabase`/`DatabaseManager`; the rest of the stack runs without it.
+- Disable extra instrumentation when running on low-memory hardware:
+  ```js
+  const perfManager = new PerformanceManager({ enableMetrics: false, enableCache: false });
+  ```
+- Always reuse the same `PerformanceOptimizer`/`PerformanceManager` instead of instantiating per request handler, so their intervals remain singular.
+
+Following the defaults keeps Nexus-FCA comfortably under a few hundred megabytes even on small VPS nodes. When memory spikes appear, start by sampling `api.getMemoryMetrics()` and check the table above to see which subsystem is accumulating work. Adjust the related caps or temporarily disable the optional feature until the upstream issue (DB outage, repeated edit failures, etc.) is resolved.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nexus-fca",
-  "version": "3.1.4",
+  "version": "3.1.6",
   "description": "Nexus-FCA 3.1 – THE BEST, SAFEST, MOST STABLE Facebook Messenger API! Email/password + appState login, proxy support (HTTP/HTTPS/SOCKS5), random user agent, proactive cookie refresh, MQTT stability, session protection, and TypeScript support.",
   "main": "index.js",
   "repository": {

package/src/listenMqtt.js

@@ -93,19 +93,34 @@ function fetchSeqID(defaultFuncs, api, ctx, callback) {
 
 // Adaptive backoff state (per-process singleton like) - tie to ctx
 function getBackoffState(ctx){
+  const envBase = parseInt(process.env.NEXUS_MQTT_BACKOFF_BASE, 10) || 1000;
+  const envMax = parseInt(process.env.NEXUS_MQTT_BACKOFF_MAX, 10) || (5 * 60 * 1000);
+  const envFactor = parseFloat(process.env.NEXUS_MQTT_BACKOFF_FACTOR) || 2;
+  const backoffOverrides = (ctx.globalOptions && ctx.globalOptions.backoff) || {};
+  const resolved = {
+    base: Number.isFinite(backoffOverrides.baseMs) ? backoffOverrides.baseMs : envBase,
+    max: Number.isFinite(backoffOverrides.maxMs) ? backoffOverrides.maxMs : envMax,
+    factor: Number.isFinite(backoffOverrides.factor) ? backoffOverrides.factor : envFactor,
+    jitter: typeof backoffOverrides.jitter === 'number' ? backoffOverrides.jitter : 0.25,
+    resetAfterMs: Number.isFinite(backoffOverrides.resetAfterMs) ? backoffOverrides.resetAfterMs : (3 * 60 * 1000)
+  };
   if(!ctx._adaptiveReconnect){
-    const envBase = parseInt(process.env.NEXUS_MQTT_BACKOFF_BASE, 10) || 1000;
-    const envMax = parseInt(process.env.NEXUS_MQTT_BACKOFF_MAX, 10) || (5 * 60 * 1000);
-    const envFactor = parseFloat(process.env.NEXUS_MQTT_BACKOFF_FACTOR) || 2;
     ctx._adaptiveReconnect = {
-      base: envBase,          // 1s default
-      max: envMax,  // 5m default
-      factor: envFactor,
-      jitter: 0.25,        // 25% random
+      base: resolved.base,
+      max: resolved.max,
+      factor: resolved.factor,
+      jitter: resolved.jitter,
+      resetAfterMs: resolved.resetAfterMs,
       current: 0,
       lastResetTs: 0,
       consecutiveFails: 0  // Track consecutive failures
     };
+  } else {
+    ctx._adaptiveReconnect.base = resolved.base;
+    ctx._adaptiveReconnect.max = resolved.max;
+    ctx._adaptiveReconnect.factor = resolved.factor;
+    ctx._adaptiveReconnect.jitter = resolved.jitter;
+    ctx._adaptiveReconnect.resetAfterMs = resolved.resetAfterMs;
   }
   return ctx._adaptiveReconnect;
 }
@@ -121,6 +136,126 @@ function resetBackoff(state){
   state.current = 0;
   state.lastResetTs = Date.now();
 }
+function getForegroundState(ctx){
+  if (ctx && ctx.globalOptions) {
+    if (ctx.globalOptions.foreground === false) return false;
+    if (ctx.globalOptions.online === false) return false;
+  }
+  return true;
+}
+function startForegroundRefresh(ctx){
+  if (ctx._foregroundRefreshInterval) {
+    clearInterval(ctx._foregroundRefreshInterval);
+    ctx._foregroundRefreshInterval = null;
+  }
+  const options = ctx.globalOptions || {};
+  if (options.foregroundRefreshEnabled === false) return;
+  const minutes = Number.isFinite(options.foregroundRefreshMinutes)
+    ? Math.max(1, options.foregroundRefreshMinutes)
+    : 15;
+  ctx._foregroundRefreshInterval = setInterval(() => {
+    if (!ctx.mqttClient || !ctx.mqttClient.connected) return;
+    try {
+      const foreground = getForegroundState(ctx);
+      ctx.mqttClient.publish("/foreground_state", JSON.stringify({ foreground }), { qos: 0, retain: false });
+      ctx.mqttClient.publish(
+        "/set_client_settings",
+        JSON.stringify({ make_user_available_when_in_foreground: true }),
+        { qos: 0, retain: false }
+      );
+    } catch (err) {
+      if (typeof log.verbose === 'function') {
+        log.verbose("listenMqtt", `Foreground refresh publish failed: ${err.message || err}`);
+      }
+    }
+  }, minutes * 60 * 1000);
+}
+function stopForegroundRefresh(ctx){
+  if (ctx && ctx._foregroundRefreshInterval) {
+    clearInterval(ctx._foregroundRefreshInterval);
+    ctx._foregroundRefreshInterval = null;
+  }
+}
+function getStormGuard(ctx){
+  if(!ctx._mqttStorm){
+    ctx._mqttStorm = {
+      events: [],
+      windowMs: 30 * 60 * 1000,
+      threshold: 12,
+      quietMs: 90 * 1000,
+      extraDelayMs: 120 * 1000,
+      quietUntil: 0,
+      active: false,
+      lastLogTs: 0
+    };
+  }
+  return ctx._mqttStorm;
+}
+function noteStormEvent(ctx){
+  const guard = getStormGuard(ctx);
+  const now = Date.now();
+  guard.events.push(now);
+  guard.events = guard.events.filter(ts => now - ts <= guard.windowMs);
+  if (guard.events.length >= guard.threshold) {
+    guard.active = true;
+    guard.quietUntil = now + guard.quietMs;
+  } else if (guard.active && now > guard.quietUntil) {
+    guard.active = false;
+  }
+  return guard;
+}
+function resetStormGuard(ctx){
+  if(ctx._mqttStorm){
+    ctx._mqttStorm.events = [];
+    ctx._mqttStorm.active = false;
+    ctx._mqttStorm.quietUntil = 0;
+    ctx._mqttStorm.lastLogTs = 0;
+  }
+}
+function shouldLogStorm(guard){
+  if(!guard || !guard.active) return true;
+  const now = Date.now();
+  if(!guard.lastLogTs || now - guard.lastLogTs > guard.quietMs){
+    guard.lastLogTs = now;
+    return true;
+  }
+  return false;
+}
+async function runStormRecovery(ctx, api, defaultFuncs){
+  if(ctx._stormRecoveryRunning) return;
+  ctx._stormRecoveryRunning = true;
+  try {
+    log.warn('listenMqtt', 'Storm recovery triggered: validating session & refreshing tokens.');
+    try {
+      await utils.validateSession(ctx, defaultFuncs, { retries: 1, delayMs: 1000 });
+    } catch (err) {
+      log.warn('listenMqtt', `Storm validateSession failed: ${err.message || err}`);
+    }
+    if (api && typeof api.refreshFb_dtsg === 'function') {
+      try {
+        await api.refreshFb_dtsg();
+      } catch (err) {
+        log.warn('listenMqtt', `Storm fb_dtsg refresh failed: ${err.message || err}`);
+      }
+    }
+    if (ctx.globalSafety && typeof ctx.globalSafety.refreshSafeSession === 'function') {
+      try {
+        await ctx.globalSafety.refreshSafeSession();
+      } catch (err) {
+        log.warn('listenMqtt', `Storm safe session refresh failed: ${err.message || err}`);
+      }
+    }
+  } finally {
+    ctx._stormRecoveryRunning = false;
+  }
+}
+function scheduleStormRecovery(ctx, api, defaultFuncs, guard){
+  if(!guard || !guard.active) return;
+  const now = Date.now();
+  if(ctx._nextStormRecovery && now < ctx._nextStormRecovery) return;
+  ctx._nextStormRecovery = now + guard.quietMs;
+  runStormRecovery(ctx, api, defaultFuncs);
+}
 // Build lazy preflight gating
 function shouldRunPreflight(ctx){
   if(ctx.globalOptions.disablePreflight) return false;
@@ -272,8 +407,8 @@ function listenMqtt(defaultFuncs, api, ctx, globalCallback) {
       }
     })();
   }
-  const chatOn = ctx.globalOptions.online;
-  const foreground = false;
+  const chatOn = (ctx.globalOptions && ctx.globalOptions.online === false) ? false : true;
+  const foreground = getForegroundState(ctx);
   const sessionID = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) + 1;
   const GUID = utils.getGUID();
   const username = {
@@ -388,7 +523,7 @@ function listenMqtt(defaultFuncs, api, ctx, globalCallback) {
       fetchSeqID(defaultFuncs, api, ctx, (err) => {
         if (err) {
           log.warn("listenMqtt", "Failed to refresh SeqID on error, falling back to adaptive reconnect...");
-          scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback);
+          scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback, 'seqid-refresh-failed');
         } else {
           listenMqtt(defaultFuncs, api, ctx, globalCallback);
         }
@@ -403,48 +538,87 @@ function listenMqtt(defaultFuncs, api, ctx, globalCallback) {
   mqttClient.on('close', function () {
     ctx.health.onDisconnect();
     if (ctx.health && typeof ctx.health.incFailure === 'function') { ctx.health.incFailure(); }
+    stopForegroundRefresh(ctx);
 
     const duration = Date.now() - attemptStartTs;
     const seconds = Math.floor(duration / 1000);
 
+    const shortWindowMs = 5 * 60 * 1000;
+    const guard = duration < shortWindowMs ? noteStormEvent(ctx) : getStormGuard(ctx);
+    const allowLog = shouldLogStorm(guard);
+    const stormSuffix = guard && guard.active ? ` [storm:${guard.events.length}/${Math.round(guard.windowMs/60000)}m]` : '';
+    if (guard && guard.active) {
+      scheduleStormRecovery(ctx, api, defaultFuncs, guard);
+    }
     // Treat long-lived connections as normal lifecycle, keep logs calm
     if (duration >= 30 * 60 * 1000) { // >= 30 minutes
-      log.info('listenMqtt', `MQTT connection closed after ${seconds}s (normal lifecycle). Reconnecting...`);
+      if (allowLog) log.info('listenMqtt', `MQTT connection closed after ${seconds}s (normal lifecycle). Reconnecting...${stormSuffix}`);
     } else if (duration >= 5 * 60 * 1000) { // 5-30 minutes
-      log.info('listenMqtt', `MQTT connection closed after ${seconds}s (remote close). Reconnecting with backoff...`);
+      if (allowLog) log.info('listenMqtt', `MQTT connection closed after ${seconds}s (remote close). Reconnecting with backoff...${stormSuffix}`);
+    } else if (duration >= 60 * 1000) {
+      if (allowLog) log.info('listenMqtt', `MQTT connection closed after ${seconds}s (short remote close). Reconnecting with backoff...${stormSuffix}`);
     } else {
-      log.warn('listenMqtt', `MQTT bridge socket closed quickly after ${duration}ms (attempt=${ctx._mqttDiag.attempts}).`);
+      if (allowLog) log.warn('listenMqtt', `MQTT bridge socket closed quickly after ${duration}ms (attempt=${ctx._mqttDiag.attempts}).${stormSuffix}`);
     }
 
     if (!ctx.loggedIn) return; // avoid loops if logged out
     if (ctx.globalOptions.autoReconnect) {
-      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback);
+      const backoffState = getBackoffState(ctx);
+      const resetThreshold = backoffState.resetAfterMs || (3 * 60 * 1000);
+      let reconnectReason = 'close';
+      if (duration >= resetThreshold) {
+        log.info('listenMqtt', `Resetting MQTT backoff after ${seconds}s healthy session.`);
+        resetBackoff(backoffState);
+        resetStormGuard(ctx);
+        reconnectReason = 'close-long';
+      }
+      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback, reconnectReason);
     }
   });
 
   mqttClient.on('disconnect', function(){
     ctx.health.onDisconnect();
     if (ctx.health && typeof ctx.health.incFailure === 'function') { ctx.health.incFailure(); }
+    stopForegroundRefresh(ctx);
 
     const duration = Date.now() - attemptStartTs;
     const seconds = Math.floor(duration / 1000);
 
+    const shortWindowMs = 5 * 60 * 1000;
+    const guard = duration < shortWindowMs ? noteStormEvent(ctx) : getStormGuard(ctx);
+    const allowLog = shouldLogStorm(guard);
+    const stormSuffix = guard && guard.active ? ` [storm:${guard.events.length}/${Math.round(guard.windowMs/60000)}m]` : '';
+    if (guard && guard.active) {
+      scheduleStormRecovery(ctx, api, defaultFuncs, guard);
+    }
     if (duration >= 30 * 60 * 1000) {
-      log.info('listenMqtt', `MQTT disconnected after ${seconds}s (normal lifecycle). Reconnecting...`);
+      if (allowLog) log.info('listenMqtt', `MQTT disconnected after ${seconds}s (normal lifecycle). Reconnecting...${stormSuffix}`);
     } else if (duration >= 5 * 60 * 1000) {
-      log.info('listenMqtt', `MQTT disconnected after ${seconds}s (remote close). Reconnecting with backoff...`);
+      if (allowLog) log.info('listenMqtt', `MQTT disconnected after ${seconds}s (remote close). Reconnecting with backoff...${stormSuffix}`);
+    } else if (duration >= 60 * 1000) {
+      if (allowLog) log.info('listenMqtt', `MQTT disconnected after ${seconds}s (short remote close). Reconnecting with backoff...${stormSuffix}`);
     } else {
-      log.warn('listenMqtt', `MQTT bridge disconnect event after ${duration}ms (attempt=${ctx._mqttDiag.attempts}).`);
+      if (allowLog) log.warn('listenMqtt', `MQTT bridge disconnect event after ${duration}ms (attempt=${ctx._mqttDiag.attempts}).${stormSuffix}`);
     }
 
     if (!ctx.loggedIn) return;
     if (ctx.globalOptions.autoReconnect) {
-      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback);
+      const backoffState = getBackoffState(ctx);
+      const resetThreshold = backoffState.resetAfterMs || (3 * 60 * 1000);
+      let reconnectReason = 'disconnect';
+      if (duration >= resetThreshold) {
+        log.info('listenMqtt', `Resetting MQTT backoff after ${seconds}s healthy session.`);
+        resetBackoff(backoffState);
+        resetStormGuard(ctx);
+        reconnectReason = 'disconnect-long';
+      }
+      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback, reconnectReason);
     }
   });
 
   mqttClient.on("connect", function () {
     resetBackoff(backoff);
+    resetStormGuard(ctx);
     backoff.consecutiveFails = 0;  // Reset consecutive failures on successful connect
     // Reset or wrap MQTT attempt counter so long-lived sessions don't look scary
     ctx._mqttDiag = ctx._mqttDiag || {};
@@ -518,11 +692,12 @@ function listenMqtt(defaultFuncs, api, ctx, globalCallback) {
       JSON.stringify({ make_user_available_when_in_foreground: true }),
       { qos: 1 }
     );
+    startForegroundRefresh(ctx);
     // Replace fixed rTimeout reconnect with health-driven logic
     const rTimeout = setTimeout(function () {
       ctx.health.onError('timeout_no_t_ms');
       mqttClient.end();
-      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback);
+      scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback, 'tms-timeout');
     }, 5000);
     ctx.tmsWait = function () {
       clearTimeout(rTimeout);
@@ -619,11 +794,19 @@ function listenMqtt(defaultFuncs, api, ctx, globalCallback) {
     }, 55000 + Math.floor(Math.random()*20000));
   }
 }
-function scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback){
+function scheduleAdaptiveReconnect(defaultFuncs, api, ctx, globalCallback, reason){
   const state = getBackoffState(ctx);
-  const delay = computeNextDelay(state);
+  const guard = getStormGuard(ctx);
+  let delay = computeNextDelay(state);
+  if (guard && guard.active) {
+    delay = Math.min(state.max, delay + guard.extraDelayMs);
+  }
   ctx.health.onReconnectScheduled(delay);
-  log.warn('listenMqtt', `Reconnecting in ${delay} ms (adaptive backoff)`);
+  const tags = [];
+  if (reason) tags.push(reason);
+  if (guard && guard.active) tags.push('storm');
+  const suffix = tags.length ? ` (${tags.join(', ')})` : '';
+  log.warn('listenMqtt', `Reconnecting in ${delay} ms (adaptive backoff)${suffix}`);
   setTimeout(()=>listenMqtt(defaultFuncs, api, ctx, globalCallback), delay);
 }
 function getTaskResponseData(taskType, payload) {