@serve.zone/dcrouter 2.12.4

package/test/suite/smtpserver_reliability/test.rel-01.long-running-operation.ts

@@ -0,0 +1,344 @@
+import * as plugins from '@git.zone/tstest/tapbundle';
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as net from 'net';
+import { startTestServer, stopTestServer } from '../../helpers/server.loader.js';
+
+const TEST_PORT = 2525;
+
+let testServer;
+
+tap.test('prepare server', async () => {
+  testServer = await startTestServer({ port: TEST_PORT });
+  await new Promise(resolve => setTimeout(resolve, 100));
+});
+
+tap.test('REL-01: Long-running operation - Continuous email sending', async (tools) => {
+  const done = tools.defer();
+  const testDuration = 30000; // 30 seconds
+  const operationInterval = 2000; // 2 seconds between operations
+  const startTime = Date.now();
+  const endTime = startTime + testDuration;
+  
+  let operations = 0;
+  let successful = 0;
+  let errors = 0;
+  let connectionIssues = 0;
+  const operationResults: Array<{
+    operation: number;
+    success: boolean;
+    duration: number;
+    error?: string;
+    timestamp: number;
+  }> = [];
+
+  console.log(`Running long-duration test for ${testDuration/1000} seconds...`);
+
+  const performOperation = async (operationId: number): Promise<void> => {
+    const operationStart = Date.now();
+    operations++;
+
+    try {
+      const socket = net.createConnection({
+        host: 'localhost',
+        port: TEST_PORT,
+        timeout: 10000
+      });
+
+      const result = await new Promise<{ success: boolean; error?: string; connectionIssue?: boolean }>((resolve) => {
+        let step = 'connecting';
+        let receivedData = '';
+
+        const timeout = setTimeout(() => {
+          socket.destroy();
+          resolve({
+            success: false,
+            error: `Timeout in step ${step}`,
+            connectionIssue: true
+          });
+        }, 10000);
+
+        socket.on('connect', () => {
+          step = 'connected';
+        });
+
+        socket.on('data', (chunk) => {
+          receivedData += chunk.toString();
+          const lines = receivedData.split('\r\n');
+
+          for (const line of lines) {
+            if (!line.trim()) continue;
+
+            // Check for errors
+            if (line.match(/^[45]\d\d\s/)) {
+              clearTimeout(timeout);
+              socket.destroy();
+              resolve({
+                success: false,
+                error: `SMTP error in ${step}: ${line}`,
+                connectionIssue: false
+              });
+              return;
+            }
+
+            // Process responses
+            if (step === 'connected' && line.startsWith('220')) {
+              step = 'ehlo';
+              socket.write(`EHLO longrun-${operationId}\r\n`);
+            } else if (step === 'ehlo' && line.includes('250 ') && !line.includes('250-')) {
+              step = 'mail_from';
+              socket.write(`MAIL FROM:<sender${operationId}@example.com>\r\n`);
+            } else if (step === 'mail_from' && line.startsWith('250')) {
+              step = 'rcpt_to';
+              socket.write(`RCPT TO:<recipient${operationId}@example.com>\r\n`);
+            } else if (step === 'rcpt_to' && line.startsWith('250')) {
+              step = 'data';
+              socket.write('DATA\r\n');
+            } else if (step === 'data' && line.startsWith('354')) {
+              step = 'email_content';
+              const emailContent = [
+                `From: sender${operationId}@example.com`,
+                `To: recipient${operationId}@example.com`,
+                `Subject: Long Running Test Operation ${operationId}`,
+                `Date: ${new Date().toUTCString()}`,
+                '',
+                `This is test operation ${operationId} for long-running reliability testing.`,
+                `Timestamp: ${Date.now()}`,
+                '.',
+                ''
+              ].join('\r\n');
+              socket.write(emailContent);
+            } else if (step === 'email_content' && line.startsWith('250')) {
+              step = 'quit';
+              socket.write('QUIT\r\n');
+            } else if (step === 'quit' && line.startsWith('221')) {
+              clearTimeout(timeout);
+              socket.end();
+              resolve({
+                success: true
+              });
+              return;
+            }
+          }
+        });
+
+        socket.on('error', (error) => {
+          clearTimeout(timeout);
+          resolve({
+            success: false,
+            error: error.message,
+            connectionIssue: true
+          });
+        });
+
+        socket.on('close', () => {
+          if (step !== 'quit') {
+            clearTimeout(timeout);
+            resolve({
+              success: false,
+              error: 'Connection closed unexpectedly',
+              connectionIssue: true
+            });
+          }
+        });
+      });
+
+      const duration = Date.now() - operationStart;
+
+      if (result.success) {
+        successful++;
+      } else {
+        errors++;
+        if (result.connectionIssue) {
+          connectionIssues++;
+        }
+      }
+
+      operationResults.push({
+        operation: operationId,
+        success: result.success,
+        duration,
+        error: result.error,
+        timestamp: operationStart
+      });
+
+    } catch (error) {
+      errors++;
+      operationResults.push({
+        operation: operationId,
+        success: false,
+        duration: Date.now() - operationStart,
+        error: error instanceof Error ? error.message : 'Unknown error',
+        timestamp: operationStart
+      });
+    }
+  };
+
+  try {
+    // Run operations continuously until end time
+    while (Date.now() < endTime) {
+      const operationStart = Date.now();
+      
+      await performOperation(operations + 1);
+
+      // Calculate wait time for next operation
+      const nextOperation = operationStart + operationInterval;
+      const waitTime = nextOperation - Date.now();
+
+      if (waitTime > 0 && Date.now() < endTime) {
+        await new Promise(resolve => setTimeout(resolve, waitTime));
+      }
+
+      // Progress update every 5 operations
+      if (operations % 5 === 0) {
+        console.log(`Progress: ${operations} operations, ${successful} successful, ${errors} errors`);
+      }
+    }
+
+    // Calculate results
+    const totalDuration = Date.now() - startTime;
+    const successRate = successful / operations;
+    const connectionIssueRate = connectionIssues / operations;
+    const avgOperationTime = operationResults.reduce((sum, r) => sum + r.duration, 0) / operations;
+
+    console.log(`\nLong-Running Operation Results:`);
+    console.log(`Total duration: ${(totalDuration/1000).toFixed(1)}s`);
+    console.log(`Total operations: ${operations}`);
+    console.log(`Successful: ${successful} (${(successRate * 100).toFixed(1)}%)`);
+    console.log(`Errors: ${errors}`);
+    console.log(`Connection issues: ${connectionIssues} (${(connectionIssueRate * 100).toFixed(1)}%)`);
+    console.log(`Average operation time: ${avgOperationTime.toFixed(0)}ms`);
+
+    // Show last few operations for debugging
+    console.log('\nLast 5 operations:');
+    operationResults.slice(-5).forEach(op => {
+      console.log(`  Op ${op.operation}: ${op.success ? 'success' : 'failed'} (${op.duration}ms)${op.error ? ' - ' + op.error : ''}`);
+    });
+
+    // Test passes with 85% success rate and max 10% connection issues
+    expect(successRate).toBeGreaterThanOrEqual(0.85);
+    expect(connectionIssueRate).toBeLessThanOrEqual(0.1);
+    done.resolve();
+  } catch (error) {
+    done.reject(error);
+  }
+});
+
+tap.test('REL-01: Long-running operation - Server stability check', async (tools) => {
+  const done = tools.defer();
+  const checkDuration = 15000; // 15 seconds
+  const checkInterval = 3000; // 3 seconds between checks
+  const startTime = Date.now();
+  const endTime = startTime + checkDuration;
+  
+  const stabilityChecks: Array<{
+    timestamp: number;
+    responseTime: number;
+    success: boolean;
+    error?: string;
+  }> = [];
+
+  console.log(`\nRunning server stability checks for ${checkDuration/1000} seconds...`);
+
+  try {
+    while (Date.now() < endTime) {
+      const checkStart = Date.now();
+      
+      const socket = net.createConnection({
+        host: 'localhost',
+        port: TEST_PORT,
+        timeout: 5000
+      });
+
+      const checkResult = await new Promise<{ success: boolean; responseTime: number; error?: string }>((resolve) => {
+        const connectTime = Date.now();
+        let greetingReceived = false;
+
+        const timeout = setTimeout(() => {
+          socket.destroy();
+          resolve({
+            success: false,
+            responseTime: Date.now() - connectTime,
+            error: 'Timeout waiting for greeting'
+          });
+        }, 5000);
+
+        socket.on('connect', () => {
+          // Connected
+        });
+
+        socket.once('data', (chunk) => {
+          const response = chunk.toString();
+          clearTimeout(timeout);
+          greetingReceived = true;
+          
+          if (response.startsWith('220')) {
+            socket.write('QUIT\r\n');
+            socket.end();
+            resolve({
+              success: true,
+              responseTime: Date.now() - connectTime
+            });
+          } else {
+            socket.end();
+            resolve({
+              success: false,
+              responseTime: Date.now() - connectTime,
+              error: `Unexpected greeting: ${response.substring(0, 50)}`
+            });
+          }
+        });
+
+        socket.on('error', (error) => {
+          clearTimeout(timeout);
+          resolve({
+            success: false,
+            responseTime: Date.now() - connectTime,
+            error: error.message
+          });
+        });
+      });
+
+      stabilityChecks.push({
+        timestamp: checkStart,
+        responseTime: checkResult.responseTime,
+        success: checkResult.success,
+        error: checkResult.error
+      });
+
+      console.log(`Stability check ${stabilityChecks.length}: ${checkResult.success ? 'OK' : 'FAILED'} (${checkResult.responseTime}ms)`);
+
+      // Wait for next check
+      const nextCheck = checkStart + checkInterval;
+      const waitTime = nextCheck - Date.now();
+      if (waitTime > 0 && Date.now() < endTime) {
+        await new Promise(resolve => setTimeout(resolve, waitTime));
+      }
+    }
+
+    // Analyze stability
+    const successfulChecks = stabilityChecks.filter(c => c.success).length;
+    const avgResponseTime = stabilityChecks
+      .filter(c => c.success)
+      .reduce((sum, c) => sum + c.responseTime, 0) / successfulChecks || 0;
+    const maxResponseTime = Math.max(...stabilityChecks.filter(c => c.success).map(c => c.responseTime));
+
+    console.log(`\nStability Check Results:`);
+    console.log(`Total checks: ${stabilityChecks.length}`);
+    console.log(`Successful: ${successfulChecks} (${(successfulChecks/stabilityChecks.length * 100).toFixed(1)}%)`);
+    console.log(`Average response time: ${avgResponseTime.toFixed(0)}ms`);
+    console.log(`Max response time: ${maxResponseTime}ms`);
+
+    // All checks should succeed for stable server
+    expect(successfulChecks).toEqual(stabilityChecks.length);
+    expect(avgResponseTime).toBeLessThan(1000);
+    done.resolve();
+  } catch (error) {
+    done.reject(error);
+  }
+});
+
+tap.test('cleanup server', async () => {
+  await stopTestServer(testServer);
+});
+
+export default tap.start();

package/test/suite/smtpserver_reliability/test.rel-02.restart-recovery.ts

@@ -0,0 +1,328 @@
+import * as plugins from '@git.zone/tstest/tapbundle';
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import * as net from 'net';
+import { startTestServer, stopTestServer } from '../../helpers/server.loader.js';
+
+const TEST_PORT = 2525;
+
+let testServer;
+
+// Helper function to wait for SMTP response
+const waitForResponse = (socket: net.Socket, expectedCode?: string, timeout = 5000): Promise<string> => {
+  return new Promise((resolve, reject) => {
+    let buffer = '';
+    const timer = setTimeout(() => {
+      socket.removeListener('data', handler);
+      reject(new Error(`Timeout waiting for ${expectedCode || 'any'} response`));
+    }, timeout);
+    
+    const handler = (data: Buffer) => {
+      buffer += data.toString();
+      const lines = buffer.split('\r\n');
+      
+      // Check if we have a complete response
+      for (const line of lines) {
+        if (expectedCode) {
+          if (line.startsWith(expectedCode + ' ')) {
+            clearTimeout(timer);
+            socket.removeListener('data', handler);
+            resolve(buffer);
+            return;
+          }
+        } else {
+          // Any complete response line
+          if (line.match(/^\d{3} /)) {
+            clearTimeout(timer);
+            socket.removeListener('data', handler);
+            resolve(buffer);
+            return;
+          }
+        }
+      }
+    };
+    
+    socket.on('data', handler);
+  });
+};
+
+tap.test('prepare server', async () => {
+  testServer = await startTestServer({ port: TEST_PORT });
+  await new Promise(resolve => setTimeout(resolve, 100));
+});
+
+tap.test('REL-02: Restart recovery - Server state after restart', async (tools) => {
+  const done = tools.defer();
+
+  try {
+    console.log('Testing server state and recovery capabilities...');
+
+    // First, establish that server is working normally
+    const socket1 = net.createConnection({
+      host: 'localhost',
+      port: TEST_PORT,
+      timeout: 30000
+    });
+
+    await new Promise<void>((resolve, reject) => {
+      socket1.once('connect', resolve);
+      socket1.once('error', reject);
+    });
+
+    // Read greeting
+    const greeting1 = await waitForResponse(socket1, '220');
+    expect(greeting1).toInclude('220');
+    console.log('Initial connection successful');
+
+    // Send EHLO
+    socket1.write('EHLO testhost\r\n');
+    await waitForResponse(socket1, '250');
+
+    // Complete a transaction
+    socket1.write('MAIL FROM:<sender@example.com>\r\n');
+    const mailResp1 = await waitForResponse(socket1, '250');
+    expect(mailResp1).toInclude('250');
+
+    socket1.write('RCPT TO:<recipient@example.com>\r\n');
+    const rcptResp1 = await waitForResponse(socket1, '250');
+    expect(rcptResp1).toInclude('250');
+
+    socket1.write('DATA\r\n');
+    const dataResp1 = await waitForResponse(socket1, '354');
+    expect(dataResp1).toInclude('354');
+
+    const emailContent = [
+      'From: sender@example.com',
+      'To: recipient@example.com',
+      'Subject: Pre-restart test',
+      '',
+      'Testing server state before restart.',
+      '.',
+      ''
+    ].join('\r\n');
+
+    socket1.write(emailContent);
+    const sendResp1 = await waitForResponse(socket1, '250');
+    expect(sendResp1).toInclude('250');
+
+    socket1.write('QUIT\r\n');
+    await waitForResponse(socket1, '221');
+    socket1.end();
+
+    console.log('Pre-restart transaction completed successfully');
+
+    // Simulate server restart by closing and reopening connections
+    console.log('\nSimulating server restart scenario...');
+    
+    // Wait a moment to simulate restart time
+    await new Promise(resolve => setTimeout(resolve, 2000));
+
+    // Test recovery after simulated restart
+    const socket2 = net.createConnection({
+      host: 'localhost',
+      port: TEST_PORT,
+      timeout: 30000
+    });
+
+    await new Promise<void>((resolve, reject) => {
+      socket2.once('connect', resolve);
+      socket2.once('error', reject);
+    });
+
+    // Read greeting after "restart"
+    const greeting2 = await new Promise<string>((resolve) => {
+      socket2.once('data', (chunk) => {
+        resolve(chunk.toString());
+      });
+    });
+
+    expect(greeting2).toInclude('220');
+    console.log('Post-restart connection successful');
+
+    // Verify server is fully functional after restart
+    socket2.write('EHLO testhost-postrestart\r\n');
+    await waitForResponse(socket2, '250');
+
+    // Complete another transaction to verify full recovery
+    socket2.write('MAIL FROM:<sender2@example.com>\r\n');
+    const mailResp2 = await waitForResponse(socket2, '250');
+    expect(mailResp2).toInclude('250');
+
+    socket2.write('RCPT TO:<recipient2@example.com>\r\n');
+    const rcptResp2 = await waitForResponse(socket2, '250');
+    expect(rcptResp2).toInclude('250');
+
+    socket2.write('DATA\r\n');
+    const dataResp2 = await waitForResponse(socket2, '354');
+    expect(dataResp2).toInclude('354');
+
+    const postRestartEmail = [
+      'From: sender2@example.com',
+      'To: recipient2@example.com',
+      'Subject: Post-restart recovery test',
+      '',
+      'Testing server recovery after restart.',
+      '.',
+      ''
+    ].join('\r\n');
+
+    socket2.write(postRestartEmail);
+    const sendResp2 = await waitForResponse(socket2, '250');
+    expect(sendResp2).toInclude('250');
+
+    socket2.write('QUIT\r\n');
+    await waitForResponse(socket2, '221');
+    socket2.end();
+
+    console.log('Post-restart transaction completed successfully');
+    console.log('Server recovered successfully from restart');
+
+    done.resolve();
+  } catch (error) {
+    done.reject(error);
+  }
+});
+
+tap.test('REL-02: Restart recovery - Multiple rapid reconnections', async (tools) => {
+  const done = tools.defer();
+  const rapidConnections = 10;
+  let successfulReconnects = 0;
+
+  try {
+    console.log(`\nTesting rapid reconnection after disruption (${rapidConnections} attempts)...`);
+
+    for (let i = 0; i < rapidConnections; i++) {
+      try {
+        const socket = net.createConnection({
+          host: 'localhost',
+          port: TEST_PORT,
+          timeout: 5000
+        });
+
+        await new Promise<void>((resolve, reject) => {
+          const timeout = setTimeout(() => {
+            socket.destroy();
+            reject(new Error('Connection timeout'));
+          }, 5000);
+
+          socket.once('connect', () => {
+            clearTimeout(timeout);
+            resolve();
+          });
+          socket.once('error', (err) => {
+            clearTimeout(timeout);
+            reject(err);
+          });
+        });
+
+        // Read greeting
+        try {
+          const greeting = await waitForResponse(socket, '220', 3000);
+          if (greeting.includes('220')) {
+            successfulReconnects++;
+            socket.write('QUIT\r\n');
+            await waitForResponse(socket, '221', 1000).catch(() => {});
+            socket.end();
+          } else {
+            socket.destroy();
+          }
+        } catch (error) {
+          socket.destroy();
+          throw error;
+        }
+
+        // Very short delay between attempts
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+      } catch (error) {
+        console.log(`Reconnection ${i + 1} failed:`, error.message);
+      }
+    }
+
+    const reconnectRate = successfulReconnects / rapidConnections;
+    console.log(`Successful reconnections: ${successfulReconnects}/${rapidConnections} (${(reconnectRate * 100).toFixed(1)}%)`);
+
+    // Expect high success rate for good recovery
+    expect(reconnectRate).toBeGreaterThanOrEqual(0.8);
+    done.resolve();
+  } catch (error) {
+    done.reject(error);
+  }
+});
+
+tap.test('REL-02: Restart recovery - State persistence check', async (tools) => {
+  const done = tools.defer();
+
+  try {
+    console.log('\nTesting server state persistence across connections...');
+
+    // Create initial connection and start transaction
+    const socket1 = net.createConnection({
+      host: 'localhost',
+      port: TEST_PORT,
+      timeout: 30000
+    });
+
+    await new Promise<void>((resolve, reject) => {
+      socket1.once('connect', resolve);
+      socket1.once('error', reject);
+    });
+
+    // Read greeting
+    await waitForResponse(socket1, '220');
+
+    // Send EHLO
+    socket1.write('EHLO persistence-test\r\n');
+    await waitForResponse(socket1, '250');
+
+    // Start transaction but don't complete it
+    socket1.write('MAIL FROM:<incomplete@example.com>\r\n');
+    const mailResp = await waitForResponse(socket1, '250');
+    expect(mailResp).toInclude('250');
+
+    // Abruptly close connection
+    socket1.destroy();
+    console.log('Abruptly closed connection with incomplete transaction');
+
+    // Wait briefly
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
+    // Create new connection and verify server recovered
+    const socket2 = net.createConnection({
+      host: 'localhost',
+      port: TEST_PORT,
+      timeout: 30000
+    });
+
+    await new Promise<void>((resolve, reject) => {
+      socket2.once('connect', resolve);
+      socket2.once('error', reject);
+    });
+
+    // Read greeting
+    await waitForResponse(socket2, '220');
+
+    // Send EHLO
+    socket2.write('EHLO recovery-test\r\n');
+    await waitForResponse(socket2, '250');
+
+    // Try new transaction - should work without issues from previous incomplete one
+    socket2.write('MAIL FROM:<recovery@example.com>\r\n');
+    const mailResponse = await waitForResponse(socket2, '250');
+    expect(mailResponse).toInclude('250');
+    console.log('Server recovered successfully - new transaction started without issues');
+
+    socket2.write('QUIT\r\n');
+    await waitForResponse(socket2, '221');
+    socket2.end();
+
+    done.resolve();
+  } catch (error) {
+    done.reject(error);
+  }
+});
+
+tap.test('cleanup server', async () => {
+  await stopTestServer(testServer);
+});
+
+export default tap.start();