http-log-replay 1.0.0

package/replayer.js

@@ -0,0 +1,232 @@
+/* eslint-disable no-console */
+const axios = require('axios');
+const fs = require('fs');
+const path = require('path');
+const { diffJson } = require('diff');
+require('colors');
+const readline = require('readline');
+
+async function replayAndDiff(logFile, primaryUrl, secondaryUrl, reportFile, ignoreFields = [], injectedHeaders = {}, excludeEndpoints = [], onEvent = () => { }, concurrency = 1) {
+  console.log(`\n🔄 Replaying logs from: ${logFile}`);
+  console.log(`🅰️  Primary:   ${primaryUrl}`);
+  console.log(`🅱️  Secondary: ${secondaryUrl}`);
+  console.log(`🚀 Concurrency: ${concurrency}`);
+
+  const logs = [];
+  const results = [];
+  let passed = 0;
+  let failed = 0;
+
+  // 1. Read the JSONL file safely
+  try {
+    const fileStream = fs.createReadStream(logFile);
+    const rl = readline.createInterface({
+      input: fileStream,
+      crlfDelay: Infinity
+    });
+
+    for await (const line of rl) {
+      if (line.trim()) {
+        try {
+          const entry = JSON.parse(line);
+          if (entry.status >= 200 && entry.status < 300) {
+            logs.push(entry);
+          }
+        } catch (e) {
+          console.warn("⚠️ Skipping malformed log line".yellow);
+        }
+      }
+    }
+  } catch (e) {
+    console.error(`❌ Failed to read log file: ${e.message}`.red);
+    onEvent({ type: 'error', message: `Failed to read log file: ${e.message}` });
+    return;
+  }
+
+  console.log(`📊 Loaded ${logs.length} valid requests to replay.`.cyan);
+  onEvent({ type: 'start', total: logs.length });
+
+  // 2. Iterate and Replay with Concurrency
+  const processLog = async (entry, index) => {
+    const endpoint = entry.url;
+
+    // SKIP LOGIC
+    if (excludeEndpoints.includes(endpoint)) {
+      console.log(`⏩ Skipping excluded endpoint: ${endpoint}`.gray);
+      return null;
+    }
+
+    console.log(`\n[${index + 1}/${logs.length}] ${entry.method} ${endpoint}`.bold);
+
+    const headers = {
+      ...injectedHeaders,
+      'Content-Type': 'application/json',
+      'User-Agent': 'Traffic-Replayer/1.0'
+    };
+
+    const config = {
+      method: entry.method,
+      headers: headers,
+      data: entry.requestBody ? JSON.parse(entry.requestBody) : undefined,
+      validateStatus: () => true
+    };
+
+    try {
+      const start1 = Date.now();
+      const res1 = await axios({ ...config, url: `${primaryUrl}${endpoint}` });
+      const time1 = Date.now() - start1;
+
+      const start2 = Date.now();
+      const res2 = await axios({ ...config, url: `${secondaryUrl}${endpoint}` });
+      const time2 = Date.now() - start2;
+
+      const cleanBody1 = clean(res1.data, ignoreFields);
+      const cleanBody2 = clean(res2.data, ignoreFields);
+
+      const differences = diffJson(cleanBody1, cleanBody2);
+      const hasDiff = differences.length > 1;
+
+      const isSuccess = !hasDiff && res1.status === res2.status;
+
+      if (isSuccess) {
+        console.log(`✅ MATCH (${res1.status})`.green);
+        passed++;
+      } else {
+        console.log(`❌ MISMATCH`.red);
+        failed++;
+      }
+
+      const resultEntry = {
+        id: index + 1,
+        method: entry.method,
+        url: endpoint,
+        status1: res1.status,
+        status2: res2.status,
+        time1,
+        time2,
+        diff: hasDiff ? differences : null,
+        match: isSuccess
+      };
+
+      results.push(resultEntry);
+      onEvent({ type: 'progress', current: index + 1, total: logs.length, result: resultEntry });
+      return resultEntry;
+
+    } catch (err) {
+      console.error(`💥 Error replaying ${endpoint}: ${err.message}`.red);
+      onEvent({ type: 'error', message: `Error on ${endpoint}: ${err.message}` });
+      return null;
+    }
+  };
+
+  // Queue mechanism: Process logic in chunks based on concurrency
+  for (let i = 0; i < logs.length; i += concurrency) {
+    const batch = logs.slice(i, i + concurrency);
+    // Map batch to promises
+    await Promise.all(batch.map((log, idx) => processLog(log, i + idx)));
+  }
+
+  generateReport(results, reportFile, primaryUrl, secondaryUrl);
+
+  console.log(`\n🏁 Replay Complete. passed: ${passed}, failed: ${failed}`.bold);
+
+  onEvent({ type: 'complete', passed, failed, report: reportFile, results: results });
+}
+
+function clean(obj, ignoreList) {
+  if (!obj || typeof obj !== 'object') return obj;
+  const copy = JSON.parse(JSON.stringify(obj));
+  const sanitize = (o) => {
+    for (const key in o) {
+      if (ignoreList.includes(key)) {
+        delete o[key];
+      } else if (typeof o[key] === 'object' && o[key] !== null) {
+        sanitize(o[key]);
+      }
+    }
+  };
+  sanitize(copy);
+  return copy;
+}
+
+function generateReport(results, filePath, url1, url2) {
+  const fs = require('fs');
+
+  const html = `
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>Traffic Replay Report</title>
+        <style>
+            body { font-family: sans-serif; padding: 20px; background: #f4f4f9; }
+            h1 { text-align: center; }
+            .summary { display: flex; justify-content: center; gap: 20px; margin-bottom: 20px; }
+            .card { background: white; padding: 20px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
+            .success { border-left: 5px solid #2ecc71; color: #2ecc71; }
+            .failure { border-left: 5px solid #e74c3c; color: #e74c3c; }
+            table { width: 100%; border-collapse: collapse; background: white; }
+            th, td { padding: 10px; border-bottom: 1px solid #ddd; text-align: left; }
+            th { background: #eee; }
+            .diff-pre { background: #2d2d2d; color: #ccc; padding: 10px; border-radius: 4px; overflow-x: auto; }
+            .diff-added { color: #2ecc71; }
+            .diff-removed { color: #e74c3c; }
+            tr.fail-row { background-color: #ffe6e6; }
+        </style>
+    </head>
+    <body>
+        <h1>🚦 Traffic Replay Report</h1>
+        <div class="summary">
+            <div class="card success">
+                <h2>Passed</h2>
+                <p>${results.filter(r => r.match).length}</p>
+            </div>
+            <div class="card failure">
+                <h2>Failed</h2>
+                <p>${results.filter(r => !r.match).length}</p>
+            </div>
+        </div>
+
+        <table>
+            <thead>
+                <tr>
+                    <th>Method</th>
+                    <th>URL</th>
+                    <th>Env 1 Status</th>
+                    <th>Env 2 Status</th>
+                    <th>Time (ms)</th>
+                    <th>Result</th>
+                </tr>
+            </thead>
+            <tbody>
+                ${results.map(r => `
+                    <tr class="${!r.match ? 'fail-row' : ''}">
+                        <td>${r.method}</td>
+                        <td>${r.url}</td>
+                        <td>${r.status1}</td>
+                        <td>${r.status2}</td>
+                        <td>${r.time1} / ${r.time2}</td>
+                        <td>${r.match ? '✅ Match' : '❌ Mismatch'}</td>
+                    </tr>
+                    ${!r.match && r.diff ? `
+                    <tr>
+                        <td colspan="6">
+                            <pre class="diff-pre">${r.diff.map(d =>
+    d.added ? `<span class="diff-added">+ ${d.value}</span>` :
+      d.removed ? `<span class="diff-removed">- ${d.value}</span>` :
+        `<span>  ${d.value}</span>`
+  ).join('')}</pre>
+                        </td>
+                    </tr>
+                    ` : ''}
+                `).join('')}
+            </tbody>
+        </table>
+    </body>
+    </html>
+    `;
+
+  fs.writeFileSync(filePath, html);
+  console.log(`📄 Report saved to ${filePath}`.green);
+}
+
+module.exports = replayAndDiff;

package/server.js

@@ -0,0 +1,129 @@
+/* eslint-disable no-console */
+// ... existing imports
+const express = require('express');
+const http = require('http');
+const { Server } = require('socket.io');
+const cors = require('cors');
+const recorder = require('./recorder');
+const replayer = require('./replayer');
+const generator = require('./traffic-generator');
+const path = require('path');
+const fs = require('fs');
+
+function startServer(port = 4200) {
+  const app = express();
+  const server = http.createServer(app);
+  const io = new Server(server, { cors: { origin: '*' } });
+
+  app.use(cors());
+  app.use(express.json());
+
+  // --- Recorder API ---
+  app.post('/api/record/start', (req, res) => {
+    const { target, port, file } = req.body;
+    try {
+      const msg = recorder.start(target, port, file, (log) => {
+        io.emit('record-log', log);
+      });
+      res.json({ status: 'ok', message: msg });
+    } catch (e) {
+      res.status(400).json({ error: e.message });
+    }
+  });
+
+  app.post('/api/record/stop', (req, res) => {
+    recorder.stop();
+    res.json({ status: 'ok', message: 'Stopped' });
+  });
+
+  // --- Replayer API (FIXED) ---
+  app.post('/api/replay', (req, res) => {
+    // Added 'exclude' to destructuring
+    const { file, env1, env2, ignore, auth, exclude } = req.body;
+
+    const injectedHeaders = {};
+    if (auth && auth.trim() !== '') {
+      injectedHeaders['Authorization'] = auth;
+    }
+
+    // Fixed Argument Order:
+    // 1. file, 2. env1, 3. env2, 4. reportPath, 5. ignore, 6. headers, 7. EXCLUDE, 8. callback
+    replayer(
+      file,
+      env1,
+      env2,
+      'last-report.html',
+      ignore || [],
+      injectedHeaders,
+      exclude || [], // <--- Pass exclude array here
+      (event) => {
+        io.emit('replay-event', event);
+      }
+    );
+
+    res.json({ status: 'ok', message: 'Replay started' });
+  });
+
+  // --- Generator API ---
+  app.post('/api/generate', async (req, res) => {
+    // 1. Get arguments (Added 'target' and 'port' so we can start the recorder)
+    const { proxyUrl, swaggerFile, exclude, target, port } = req.body;
+
+    const targetProxy = proxyUrl || 'http://localhost:3000';
+    const targetFile = swaggerFile || './full_documentation.json';
+    const targetExclude = Array.isArray(exclude) ? exclude : [];
+    const recTarget = target || 'http://localhost:8080';
+    const recPort = port || 3000;
+
+    console.log('🚀 Starting Traffic Generation Workflow...');
+
+    // --- STEP A: Start the Recorder (The Proxy) ---
+    try {
+      // We force start the recorder so there is something listening on port 3000
+      recorder.start(recTarget, recPort, 'ui-traffic.jsonl', (log) => {
+        io.emit('record-log', log);
+      });
+      console.log(`🎙️  Auto-started Recorder on port ${recPort} -> ${recTarget}`);
+    } catch (e) {
+      console.log('⚠️  Recorder might already be running, proceeding...');
+    }
+
+    res.json({ status: 'started', message: 'Traffic generation started' });
+
+    try {
+      // --- STEP B: Run the Generator ---
+      // Wait a moment for the server to bind the port
+      await new Promise((r) => setTimeout(r, 500));
+
+      await generator.run(targetProxy, targetFile, targetExclude, (log) => {
+        // If the log is an object, emit it; if string, just log console
+        if (typeof log === 'object' && log.message) {
+          // specific generator status updates
+        }
+      });
+
+      console.log('🛑 Generation finished. Stopping Recorder...');
+
+      // --- STEP C: Stop the Recorder ---
+      recorder.stop();
+      io.emit('record-stopped');
+    } catch (e) {
+      console.error('Generator Error:', e);
+      recorder.stop(); // Ensure we stop even if generator crashes
+      io.emit('record-stopped');
+    }
+  });
+
+  // ... Serve Angular logic (keep existing) ...
+  const frontendPath = path.join(__dirname, 'ui/dist/ui/browser');
+  if (fs.existsSync(frontendPath)) {
+    app.use(express.static(frontendPath));
+    app.get('/', (req, res) => res.sendFile(path.join(frontendPath, 'index.html')));
+  }
+
+  server.listen(port, () => {
+    console.log(`\n🌐 TrafficMirror running at http://localhost:${port}`);
+  });
+}
+
+module.exports = startServer;

package/traffic-generator.js

@@ -0,0 +1,149 @@
+/* eslint-disable no-console */
+const fs = require('fs');
+const http = require('http');
+const https = require('https');
+
+class TrafficGenerator {
+  /**
+   * Fixed signature to match index.js:
+   * 1. proxyUrl (from options.target)
+   * 2. swaggerPath (from options.file)
+   * 3. exclude (from options.exclude)
+   * 4. onProgress (the callback)
+   * 5. sourceUrl (from options.source)
+   */
+  async run(proxyUrl, swaggerPath, exclude = [], onProgress = () => { }, sourceUrl) {
+    // --- STEP 1: Health Checks ---
+    onProgress({ message: `🩺 Starting Pre-flight Health Checks...` });
+
+    // Check Proxy Health
+    onProgress({ message: `   - Checking Proxy: ${proxyUrl}` });
+    const isProxyUp = await this.checkConnection(proxyUrl);
+
+    // Check Source Health (if provided)
+    let isSourceUp = true;
+    if (sourceUrl) {
+      onProgress({ message: `   - Checking Source: ${sourceUrl}` });
+      isSourceUp = await this.checkConnection(sourceUrl);
+    }
+
+    if (!isProxyUp) {
+      const msg = `🛑 ABORTING: Proxy is unreachable at ${proxyUrl}`;
+      onProgress({ message: msg });
+      throw new Error(msg);
+    }
+
+    if (sourceUrl && !isSourceUp) {
+      const msg = `🛑 ABORTING: Source Server is unreachable at ${sourceUrl}`;
+      onProgress({ message: msg });
+      throw new Error(msg);
+    }
+
+    onProgress({ message: `✅ Systems Nominal. Proxy and Source are UP.` });
+    // -----------------------------
+
+    onProgress({ message: `📖 Reading Swagger file: ${swaggerPath}` });
+
+    let doc;
+    try {
+      if (!fs.existsSync(swaggerPath)) {
+        throw new Error(`Swagger file not found at ${swaggerPath}`);
+      }
+      const raw = fs.readFileSync(swaggerPath);
+      doc = JSON.parse(raw);
+    } catch (e) {
+      throw new Error(`Failed to load Swagger: ${e.message}`);
+    }
+
+    // Validate Swagger structure
+    if (!doc.paths) {
+      throw new Error("Invalid Swagger file: 'paths' property missing.");
+    }
+
+    const paths = Object.keys(doc.paths);
+    const getEndpoints = paths.filter((p) => doc.paths[p].get);
+
+    onProgress({
+      message: `🔍 Found ${getEndpoints.length} GET endpoints. Applying filters...`,
+    });
+
+    const rawExclude = Array.isArray(exclude) ? exclude : [exclude];
+    const cleanExclude = rawExclude.map((e) => (e ? e.trim() : '')).filter((e) => e !== '');
+
+    const validEndpoints = getEndpoints.filter((endpoint) => {
+      if (cleanExclude.includes(endpoint)) return false;
+      const isExcluded = cleanExclude.some((excludedItem) => {
+        return excludedItem.endsWith(endpoint) || endpoint.endsWith(excludedItem);
+      });
+      if (isExcluded) return false;
+      return true;
+    });
+
+    onProgress({
+      message: `⚡ Generating traffic for ${validEndpoints.length
+        } endpoints (Skipped ${getEndpoints.length - validEndpoints.length})`,
+    });
+
+    // Ensure we send traffic to the PROXY
+    const cleanProxyUrl = proxyUrl.replace(/\/$/, '');
+    let count = 0;
+
+    for (const endpoint of validEndpoints) {
+      // Skip parameterized endpoints like /users/{id} for now as we can't guess IDs
+      if (endpoint.includes('{')) {
+        onProgress({ message: `⚠️ Skipping parameterized path: ${endpoint}` });
+        continue;
+      }
+
+      const url = `${cleanProxyUrl}/api${endpoint}`;
+      console.log(`[Generator] Requesting: ${url}`);
+      count++;
+
+      try {
+        await this.sendRequest(url);
+        onProgress({
+          message: `[${count}/${validEndpoints.length}] 🚀 HIT: ${endpoint}`,
+        });
+        // Small delay to prevent overwhelming the server
+        await new Promise((r) => setTimeout(r, 50));
+      } catch (err) {
+        onProgress({ message: `❌ FAIL: ${endpoint} - ${err.message}` });
+      }
+    }
+
+    onProgress({ message: '✅ Traffic generation complete!' });
+    return true;
+  }
+
+  checkConnection(url) {
+    return new Promise((resolve) => {
+      if (!url) resolve(true);
+
+      const client = url.startsWith('https') ? https : http;
+      const req = client.get(url, (res) => {
+        res.resume();
+        resolve(true);
+      });
+
+      req.on('error', () => resolve(false));
+
+      req.setTimeout(5000, () => {
+        req.destroy();
+        resolve(false);
+      });
+    });
+  }
+
+  sendRequest(url) {
+    return new Promise((resolve, reject) => {
+      const client = url.startsWith('https') ? https : http;
+      const req = client.get(url, { headers: { 'User-Agent': 'Traffic-Mirror-Bot' } }, (res) => {
+        res.resume();
+        resolve();
+      });
+      req.on('error', (e) => reject(e));
+    });
+  }
+}
+
+module.exports = new TrafficGenerator();