@push.rocks/smartmta 6.4.1 → 6.5.0

package/ts/security/classes.rustsecuritybridge.ts

@@ -406,7 +406,9 @@ export class RustSecurityBridge extends EventEmitter {
   private _state: BridgeState = BridgeState.Idle;
   private _restartAttempts = 0;
   private _restartTimer: ReturnType<typeof setTimeout> | null = null;
+  private _spawnInFlight = false;
   private _healthCheckTimer: ReturnType<typeof setInterval> | null = null;
+  private _healthCheckStrikes = 0;
   private _deliberateStop = false;
   private _smtpServerConfig: ISmtpServerConfig | null = null;
 
@@ -454,12 +456,15 @@ export class RustSecurityBridge extends EventEmitter {
 
     this.bridge.on('exit', (code: number | null, signal: string | null) => {
       this._running = false;
-      logger.log('warn', `Rust security bridge exited (code=${code}, signal=${signal})`);
+      logger.log('error', `Rust security bridge exited (code=${code}, signal=${signal})`);
 
       if (this._deliberateStop) {
         this.setState(BridgeState.Stopped);
-      } else if (this._state === BridgeState.Running) {
-        // Unexpected exit — attempt restart
+      } else if (!this._restartTimer && !this._spawnInFlight && this._state !== BridgeState.Starting) {
+        // Any unexpected exit triggers recovery, regardless of the state we
+        // were in. Exits during Starting resolve through start() itself;
+        // exits while a restart is already pending/spawning are absorbed by
+        // the in-flight recovery cycle.
         this.attemptRestart();
       }
     });
@@ -540,7 +545,7 @@ export class RustSecurityBridge extends EventEmitter {
       case BridgeState.Restarting:
         throw new Error('Rust bridge is restarting after a crash. Commands will resume once it recovers.');
       case BridgeState.Failed:
-        throw new Error('Rust bridge has failed after exhausting all restart attempts.');
+        throw new Error('Rust bridge is in degraded state after repeated crashes. Recovery attempts continue in the background.');
       case BridgeState.Stopped:
         throw new Error('Rust bridge has been stopped. Call start() to restart it.');
       default:
@@ -563,6 +568,11 @@ export class RustSecurityBridge extends EventEmitter {
 
     this._deliberateStop = false;
     this._restartAttempts = 0;
+    // An explicit start supersedes any pending background recovery attempt
+    if (this._restartTimer) {
+      clearTimeout(this._restartTimer);
+      this._restartTimer = null;
+    }
     this.setState(BridgeState.Starting);
 
     try {
@@ -619,69 +629,97 @@ export class RustSecurityBridge extends EventEmitter {
     const config = RustSecurityBridge._resilienceConfig;
     this._restartAttempts++;
 
-    if (this._restartAttempts > config.maxRestartAttempts) {
-      logger.log('error', `Rust bridge exceeded max restart attempts (${config.maxRestartAttempts}). Giving up.`);
+    const exhausted = this._restartAttempts > config.maxRestartAttempts;
+    if (exhausted) {
+      // Degraded, but never terminal: a mail backend that silently stays dead
+      // is worse than one that keeps retrying at the slow cadence. The Failed
+      // state signals the degradation to embedders (stateChange event) while
+      // recovery attempts continue at restartBackoffMaxMs.
+      logger.log('error', `Rust bridge exhausted ${config.maxRestartAttempts} fast restart attempts — continuing in degraded slow-retry every ${config.restartBackoffMaxMs}ms (attempt ${this._restartAttempts})`);
       this.setState(BridgeState.Failed);
-      return;
+    } else {
+      this.setState(BridgeState.Restarting);
     }
-
-    this.setState(BridgeState.Restarting);
     this.stopHealthCheck();
 
-    const delay = Math.min(
-      config.restartBackoffBaseMs * Math.pow(2, this._restartAttempts - 1),
-      config.restartBackoffMaxMs,
-    );
+    const delay = exhausted
+      ? config.restartBackoffMaxMs
+      : Math.min(
+          config.restartBackoffBaseMs * Math.pow(2, this._restartAttempts - 1),
+          config.restartBackoffMaxMs,
+        );
 
-    logger.log('info', `Rust bridge restart attempt ${this._restartAttempts}/${config.maxRestartAttempts} in ${delay}ms`);
+    logger.log('info', `Rust bridge restart attempt ${this._restartAttempts} in ${delay}ms`);
 
-    this._restartTimer = setTimeout(async () => {
+    this._restartTimer = setTimeout(() => {
       this._restartTimer = null;
+      void this.executeRestartAttempt();
+    }, delay);
+  }
 
-      // Guard: if stop() was called while we were waiting, don't restart
-      if (this._deliberateStop) {
-        this.setState(BridgeState.Stopped);
-        return;
+  private async executeRestartAttempt(): Promise<void> {
+    // Guard: if stop() was called while we were waiting, don't restart
+    if (this._deliberateStop) {
+      this.setState(BridgeState.Stopped);
+      return;
+    }
+
+    this._spawnInFlight = true;
+    try {
+      const ok = await this.bridge.spawn();
+      this._running = ok;
+      if (!ok) {
+        throw new Error('spawn returned false');
       }
 
-      try {
-        const ok = await this.bridge.spawn();
-        this._running = ok;
-
-        if (ok) {
-          logger.log('info', 'Rust bridge restarted successfully');
-          this._restartAttempts = 0;
-          this.setState(BridgeState.Running);
-          this.startHealthCheck();
-          await this.restoreAfterRestart();
-        } else {
-          logger.log('warn', 'Rust bridge restart failed (spawn returned false)');
-          this.attemptRestart();
+      const restored = await this.restoreAfterRestart();
+      if (!restored) {
+        // A bridge without its SMTP listener must not report Running — kill
+        // and go through another restart cycle instead of sitting half-alive.
+        logger.log('error', 'Rust bridge restarted but the SMTP server could not be restored — killing process to retry');
+        try {
+          this.bridge.kill();
+        } catch {
+          // Already dead
         }
-      } catch (err) {
-        logger.log('error', `Rust bridge restart failed: ${(err as Error).message}`);
-        this.attemptRestart();
+        this._running = false;
+        throw new Error('SMTP server restore failed');
       }
-    }, delay);
+
+      logger.log('info', 'Rust bridge restarted successfully');
+      this._restartAttempts = 0;
+      this.setState(BridgeState.Running);
+      this.startHealthCheck();
+    } catch (err) {
+      logger.log('error', `Rust bridge restart failed: ${(err as Error).message}`);
+      this.attemptRestart();
+    } finally {
+      this._spawnInFlight = false;
+    }
   }
 
   /**
    * Restore state after a successful restart:
    * - Re-send startSmtpServer command if the SMTP server was running
+   *
+   * @returns `true` if there was nothing to restore or the restore succeeded.
    */
-  private async restoreAfterRestart(): Promise<void> {
-    if (this._smtpServerConfig) {
-      try {
-        logger.log('info', 'Restoring SMTP server after bridge restart');
-        const result = await this.bridge.sendCommand('startSmtpServer', this._smtpServerConfig);
-        if (result?.started) {
-          logger.log('info', 'SMTP server restored after bridge restart');
-        } else {
-          logger.log('warn', 'SMTP server failed to restore after bridge restart');
-        }
-      } catch (err) {
-        logger.log('error', `Failed to restore SMTP server after restart: ${(err as Error).message}`);
+  private async restoreAfterRestart(): Promise<boolean> {
+    if (!this._smtpServerConfig) {
+      return true;
+    }
+    try {
+      logger.log('info', 'Restoring SMTP server after bridge restart');
+      const result = await this.bridge.sendCommand('startSmtpServer', this._smtpServerConfig);
+      if (result?.started) {
+        logger.log('info', 'SMTP server restored after bridge restart');
+        return true;
       }
+      logger.log('warn', 'SMTP server failed to restore after bridge restart');
+      return false;
+    } catch (err) {
+      logger.log('error', `Failed to restore SMTP server after restart: ${(err as Error).message}`);
+      return false;
     }
   }
 
@@ -693,6 +731,7 @@ export class RustSecurityBridge extends EventEmitter {
     this.stopHealthCheck();
     const config = RustSecurityBridge._resilienceConfig;
 
+    this._healthCheckStrikes = 0;
     this._healthCheckTimer = setInterval(async () => {
       if (this._state !== BridgeState.Running || !this._running) {
         return;
@@ -707,8 +746,17 @@ export class RustSecurityBridge extends EventEmitter {
         if (!(res as any)?.pong) {
           throw new Error('Health check: unexpected ping response');
         }
+        this._healthCheckStrikes = 0;
       } catch (err) {
-        logger.log('warn', `Rust bridge health check failed: ${(err as Error).message}. Killing process to trigger restart.`);
+        // Two consecutive failures before killing: a single missed ping can
+        // be momentary load, and killing mid-transaction is expensive.
+        this._healthCheckStrikes++;
+        if (this._healthCheckStrikes < 2) {
+          logger.log('warn', `Rust bridge health check failed (strike ${this._healthCheckStrikes}/2): ${(err as Error).message}`);
+          return;
+        }
+        logger.log('error', `Rust bridge health check failed twice in a row: ${(err as Error).message}. Killing process to trigger restart.`);
+        this._healthCheckStrikes = 0;
         try {
           this.bridge.kill();
         } catch {