@bernierllc/email-webhook-events 1.0.0 → 1.2.0

package/__tests__/integration/sendgrid-webhook-integration.test.ts

@@ -0,0 +1,272 @@
+/*
+Copyright (c) 2025 Bernier LLC
+
+This file is licensed to the client under a limited-use license.
+The client may use and modify this code *only within the scope of the project it was delivered for*.
+Redistribution or use in other products or commercial offerings is not permitted without written consent from Bernier LLC.
+*/
+
+/**
+ * SendGrid Webhook Integration Tests
+ *
+ * These tests use REAL SendGrid API and webhooks to:
+ * - Send emails via @bernierllc/email-manager
+ * - Receive webhooks from SendGrid via ngrok tunnel
+ * - Process webhooks through @bernierllc/email-webhook-events
+ * - Capture real webhook payloads for mock generation
+ *
+ * Prerequisites:
+ * - .env.test file with valid API keys and configuration
+ * - SendGrid webhook URL configured to ngrok URL
+ * - RUN_INTEGRATION_TESTS=true environment variable set
+ *
+ * Run with: npm test -- sendgrid-webhook-integration
+ */
+
+import * as dotenv from 'dotenv';
+import * as path from 'path';
+// Jest provides describe, it, expect, beforeAll, afterAll, beforeEach as globals
+import { EmailManager } from '@bernierllc/email-manager';
+import { EmailWebhookEventsService, EmailProvider, EmailEventType } from '../../src/index';
+import { WebhookTestServer, CapturedWebhook } from './webhook-server';
+import { NgrokTunnelManager } from './ngrok-tunnel-manager';
+
+// Load test environment variables from .env.test (at repo root)
+dotenv.config({ path: path.join(__dirname, '..', '..', '..', '..', '..', '.env.test') });
+
+// Skip these tests if not explicitly enabled or if required env vars are missing
+const SKIP_INTEGRATION_TESTS = 
+  !process.env.RUN_INTEGRATION_TESTS || 
+  !process.env.SENDGRID_API_KEY ||
+  !process.env.SENDGRID_FROM_EMAIL ||
+  !process.env.NGROK_AUTH_TOKEN ||
+  !process.env.TEST_EMAIL;
+
+const describeOrSkip = SKIP_INTEGRATION_TESTS ? describe.skip : describe;
+
+describeOrSkip('SendGrid Webhook Integration Tests', () => {
+  let webhookServer: WebhookTestServer;
+  let ngrokTunnel: NgrokTunnelManager;
+  let webhookUrl: string;
+  let emailManager: EmailManager;
+  let webhookService: EmailWebhookEventsService;
+
+  beforeAll(async () => {
+    // Start webhook server
+    const port = parseInt(process.env.WEBHOOK_SERVER_PORT || '3000', 10);
+    webhookServer = new WebhookTestServer(port);
+    await webhookServer.start();
+
+    // Start ngrok tunnel programmatically
+    ngrokTunnel = new NgrokTunnelManager();
+    webhookUrl = await ngrokTunnel.start({
+      port,
+      domain: process.env.NGROK_RESERVED_DOMAIN,
+      authtoken: process.env.NGROK_AUTH_TOKEN
+    });
+    
+    console.log(`\n📡 Webhook URL: ${webhookUrl}/webhooks/sendgrid`);
+    console.log(`📋 Configure SendGrid webhook URL to: ${webhookUrl}/webhooks/sendgrid\n`);
+
+    // Initialize email manager
+    emailManager = new EmailManager({
+      providers: [],
+      scheduling: { 
+        enabled: false,
+        checkIntervalMs: 60000,
+        maxRetries: 3,
+        retryDelayMs: 1000
+      } // Disable scheduler for tests
+    });
+
+    // Manually add provider and await to ensure it's registered before sending
+    const addResult = await emailManager.addProvider({
+      id: 'sendgrid-test',
+      name: 'SendGrid Test',
+      type: 'sendgrid',
+      config: {
+        apiKey: process.env.SENDGRID_API_KEY!,
+        fromEmail: process.env.SENDGRID_FROM_EMAIL!,
+        fromName: 'Webhook Integration Test'
+      },
+      isActive: true,
+      priority: 1
+    });
+    
+    if (!addResult.success) {
+      throw new Error(`Failed to add SendGrid provider: ${addResult.errors?.join(', ')}`);
+    }
+    
+    console.log('✅ SendGrid provider added successfully');
+
+    // Initialize webhook service
+    webhookService = new EmailWebhookEventsService({
+      providers: [{
+        provider: EmailProvider.SENDGRID,
+        enabled: true,
+        webhookUrl: '/webhooks/sendgrid',
+        secretKey: process.env.SENDGRID_WEBHOOK_SECRET || '',
+        signatureHeader: 'X-Twilio-Email-Event-Webhook-Signature',
+        signatureAlgorithm: 'sha256'
+      }],
+      persistence: { 
+        enabled: true, 
+        adapter: 'memory' 
+      },
+      analytics: { 
+        realTimeEnabled: true,
+        aggregationInterval: 5000,
+        retentionDays: 90
+      }
+    });
+    await webhookService.initialize();
+  }, 60000); // 60 second timeout for setup
+
+  afterAll(async () => {
+    await ngrokTunnel.stop();
+    await webhookServer.stop();
+  }, 30000);
+
+  beforeEach(() => {
+    webhookServer.clearCapturedWebhooks();
+  });
+
+  // Helper function to wait for webhook
+  async function waitForWebhook(
+    eventType: string,
+    timeout: number = 60000
+  ): Promise<CapturedWebhook | null> {
+    const startTime = Date.now();
+    
+    while (Date.now() - startTime < timeout) {
+      const webhooks = webhookServer.getCapturedWebhooks();
+      const matching = webhooks.find(w => w.body.event === eventType);
+      
+      if (matching) {
+        return matching;
+      }
+      
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+    
+    return null;
+  }
+
+  it('should receive and process delivered webhook', async () => {
+    const testEmail = process.env.TEST_EMAIL!;
+    
+    // Send email
+    const sendResult = await emailManager.sendEmail({
+      to: testEmail,
+      subject: 'Test Email for Webhook Testing - Delivered',
+      html: '<h1>Test</h1><p>This is a test email for delivered webhook testing.</p>',
+      text: 'Test: This is a test email for delivered webhook testing.'
+    });
+
+    expect(sendResult.success).toBe(true);
+    expect(sendResult.messageId).toBeDefined();
+
+    // Wait for webhook (with timeout)
+    const deliveredWebhook = await waitForWebhook('delivered', 60000);
+    
+    expect(deliveredWebhook).toBeDefined();
+    expect(deliveredWebhook?.body.event).toBe('delivered');
+    expect(deliveredWebhook?.body.email).toBe(testEmail);
+
+    // Process webhook through service
+    const result = await webhookService.processWebhook({
+      provider: EmailProvider.SENDGRID,
+      signature: deliveredWebhook!.headers['x-twilio-email-event-webhook-signature'] || '',
+      payload: deliveredWebhook!.body,
+      headers: deliveredWebhook!.headers
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.event).toBeDefined();
+    expect(result.event!.type).toBe(EmailEventType.DELIVERED);
+    expect(result.event!.recipient).toBe(testEmail);
+  }, 120000); // 2 minute timeout for email delivery
+
+  it('should capture webhook payload structure', async () => {
+    const testEmail = process.env.TEST_EMAIL!;
+    
+    // Send email
+    await emailManager.sendEmail({
+      to: testEmail,
+      subject: 'Test Email for Payload Capture',
+      html: '<h1>Payload Test</h1><p>Capturing webhook payload structure.</p>',
+      text: 'Payload Test: Capturing webhook payload structure.'
+    });
+
+    // Wait for delivered webhook
+    const deliveredWebhook = await waitForWebhook('delivered', 60000);
+    
+    expect(deliveredWebhook).toBeDefined();
+    
+    // Verify payload structure
+    expect(deliveredWebhook!.body).toHaveProperty('event');
+    expect(deliveredWebhook!.body).toHaveProperty('email');
+    expect(deliveredWebhook!.body).toHaveProperty('timestamp');
+    expect(deliveredWebhook!.body).toHaveProperty('sg_message_id');
+    
+    // Verify headers
+    expect(deliveredWebhook!.headers).toBeDefined();
+    
+    console.log('\n📦 Captured webhook payload structure:');
+    console.log(JSON.stringify(deliveredWebhook!.body, null, 2));
+  }, 120000);
+
+  it('should handle multiple events in single webhook', async () => {
+    const testEmail = process.env.TEST_EMAIL!;
+    
+    // Send email
+    await emailManager.sendEmail({
+      to: testEmail,
+      subject: 'Test Email for Multiple Events',
+      html: '<h1>Multiple Events</h1><p>Testing multiple webhook events.</p>',
+      text: 'Multiple Events: Testing multiple webhook events.'
+    });
+
+    // Wait for delivered webhook
+    const deliveredWebhook = await waitForWebhook('delivered', 60000);
+    
+    expect(deliveredWebhook).toBeDefined();
+    
+    // SendGrid may send multiple events in one webhook
+    // Verify we can handle both single and multiple events
+    const allWebhooks = webhookServer.getCapturedWebhooks();
+    expect(allWebhooks.length).toBeGreaterThan(0);
+  }, 120000);
+
+  // Note: Open, click, bounce, spamreport, and unsubscribe events
+  // require manual interaction or specific test conditions.
+  // These can be tested manually or with email client automation.
+  
+  it('should process webhook with signature validation', async () => {
+    const testEmail = process.env.TEST_EMAIL!;
+    
+    // Send email
+    await emailManager.sendEmail({
+      to: testEmail,
+      subject: 'Test Email for Signature Validation',
+      html: '<h1>Signature Test</h1><p>Testing webhook signature validation.</p>',
+      text: 'Signature Test: Testing webhook signature validation.'
+    });
+
+    // Wait for delivered webhook
+    const deliveredWebhook = await waitForWebhook('delivered', 60000);
+    
+    expect(deliveredWebhook).toBeDefined();
+    
+    // Verify signature header is present (if SendGrid provides it)
+    const signatureHeader = deliveredWebhook!.headers['x-twilio-email-event-webhook-signature'];
+    
+    if (signatureHeader) {
+      expect(signatureHeader).toBeDefined();
+      console.log('\n🔐 Webhook signature header present:', signatureHeader.substring(0, 20) + '...');
+    } else {
+      console.log('\n⚠️  Webhook signature header not present (may not be configured in SendGrid)');
+    }
+  }, 120000);
+});
+

package/__tests__/integration/webhook-server.ts

@@ -0,0 +1,188 @@
+/*
+Copyright (c) 2025 Bernier LLC
+
+This file is licensed to the client under a limited-use license.
+The client may use and modify this code *only within the scope of the project it was delivered for*.
+Redistribution or use in other products or commercial offerings is not permitted without written consent from Bernier LLC.
+*/
+
+import express from 'express';
+import { writeFileSync, readFileSync, mkdirSync, existsSync } from 'fs';
+import { join } from 'path';
+
+export interface CapturedWebhook {
+  headers: Record<string, string>;
+  body: any;
+  timestamp: string;
+  eventType: string;
+}
+
+export class WebhookTestServer {
+  private app: express.Application;
+  private server: any;
+  private capturedWebhooks: CapturedWebhook[] = [];
+  private fixturesDir: string;
+  private port: number;
+
+  constructor(port: number = 3000) {
+    this.port = port;
+    this.app = express();
+    this.fixturesDir = join(__dirname, '../fixtures/sendgrid-webhooks');
+    
+    // Ensure fixtures directory exists
+    if (!existsSync(this.fixturesDir)) {
+      mkdirSync(this.fixturesDir, { recursive: true });
+    }
+
+    // Middleware to capture raw body
+    this.app.use(express.json({ verify: (req: any, res, buf) => {
+      req.rawBody = buf.toString('utf8');
+    }}));
+
+    // Log all incoming requests
+    this.app.use((req, res, next) => {
+      console.log(`📥 ${req.method} ${req.path}`);
+      next();
+    });
+
+    // Webhook endpoint - handle both root and /webhooks/sendgrid paths
+    const webhookHandler = (req: express.Request, res: express.Response) => {
+      console.log(`✅ Webhook received at ${req.path}`);
+      const events = Array.isArray(req.body) ? req.body : [req.body];
+      
+      events.forEach((event: any) => {
+        console.log(`   📨 Event: ${event.event} for ${event.email}`);
+        const captured: CapturedWebhook = {
+          headers: req.headers as Record<string, string>,
+          body: event,
+          timestamp: new Date().toISOString(),
+          eventType: event.event || 'unknown'
+        };
+        
+        this.capturedWebhooks.push(captured);
+        this.saveWebhookPayload(captured);
+      });
+
+      res.status(200).json({ received: true, count: events.length });
+    };
+
+    // Handle webhooks at root (current SendGrid config)
+    this.app.post('/', webhookHandler);
+    
+    // Also handle webhooks at /webhooks/sendgrid (recommended path)
+    this.app.post('/webhooks/sendgrid', webhookHandler);
+
+    // Legacy endpoint handler (keep for backward compatibility)
+    this.app.post('/webhooks/sendgrid-old', (req, res) => {
+      const events = Array.isArray(req.body) ? req.body : [req.body];
+      
+      events.forEach((event: any) => {
+        const captured: CapturedWebhook = {
+          headers: req.headers as Record<string, string>,
+          body: event,
+          timestamp: new Date().toISOString(),
+          eventType: event.event || 'unknown'
+        };
+        
+        this.capturedWebhooks.push(captured);
+        this.saveWebhookPayload(captured);
+      });
+
+      res.status(200).json({ received: true, count: events.length });
+    });
+
+    // Health check
+    this.app.get('/health', (req, res) => {
+      res.json({ status: 'ok', captured: this.capturedWebhooks.length });
+    });
+
+    // Catch-all for any other POST (in case SendGrid hits a different path)
+    this.app.post('*', (req, res) => {
+      console.log(`⚠️  Unhandled POST to: ${req.path}`);
+      console.log('Body:', JSON.stringify(req.body, null, 2));
+      
+      // Still capture it as a webhook
+      const events = Array.isArray(req.body) ? req.body : [req.body];
+      events.forEach((event: any) => {
+        const captured: CapturedWebhook = {
+          headers: req.headers as Record<string, string>,
+          body: event,
+          timestamp: new Date().toISOString(),
+          eventType: event.event || 'unknown'
+        };
+        this.capturedWebhooks.push(captured);
+        this.saveWebhookPayload(captured);
+      });
+
+      res.status(200).json({ received: true, path: req.path });
+    });
+
+    // 404 handler
+    this.app.use((req, res) => {
+      console.log(`❌ 404: ${req.method} ${req.path}`);
+      res.status(404).json({ error: 'Not found', path: req.path, method: req.method });
+    });
+  }
+
+  private saveWebhookPayload(webhook: CapturedWebhook): void {
+    const eventType = webhook.eventType;
+    const filePath = join(this.fixturesDir, `${eventType}.json`);
+    
+    let existing: CapturedWebhook[] = [];
+    if (existsSync(filePath)) {
+      try {
+        existing = JSON.parse(readFileSync(filePath, 'utf8'));
+      } catch (error) {
+        // If file is corrupted, start fresh
+        console.warn(`Failed to read existing webhook file ${filePath}, starting fresh`);
+        existing = [];
+      }
+    }
+    
+    existing.push(webhook);
+    writeFileSync(filePath, JSON.stringify(existing, null, 2));
+  }
+
+  async start(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        this.server = this.app.listen(this.port, () => {
+          console.log(`Webhook test server running on port ${this.port}`);
+          resolve();
+        });
+
+        this.server.on('error', (error: Error) => {
+          reject(error);
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
+  async stop(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.server) {
+        this.server.close(() => {
+          console.log('Webhook test server stopped');
+          resolve();
+        });
+      } else {
+        resolve();
+      }
+    });
+  }
+
+  getCapturedWebhooks(): CapturedWebhook[] {
+    return [...this.capturedWebhooks];
+  }
+
+  clearCapturedWebhooks(): void {
+    this.capturedWebhooks = [];
+  }
+
+  getPort(): number {
+    return this.port;
+  }
+}
+

package/__tests__/mocks/README.md

@@ -0,0 +1,172 @@
+# SendGrid Mock Fixtures
+
+> **Philosophy: "Only mock services that are not owned by us"**
+
+These mock fixtures are based on **REAL SendGrid webhook payload structures** captured during integration testing. They accurately reflect the actual format and fields that SendGrid sends.
+
+> ⚠️ **SANITIZED DATA**: All email addresses, IPs, message IDs, signatures, API keys, and other identifiable data have been replaced with fake test values (using RFC 5737 test IPs like `192.0.2.x` and `example.com` domains). The **structure** is real, the **values** are safe for version control.
+
+## Testing Strategy
+
+| Service Type | Mock Strategy | Example |
+|--------------|---------------|---------|
+| External APIs | ✅ Mock | SendGrid, Stripe, AWS |
+| Internal packages | ❌ Use Real | @bernierllc/*, @unicorn-love/* |
+
+### Why This Approach?
+
+1. **Confidence**: Tests using real internal packages catch real integration issues
+2. **Accuracy**: Mocks based on real payloads prevent "mock drift"
+3. **Speed**: Mocking external APIs is fast and doesn't require network
+4. **Reliability**: No flaky tests from external service downtime
+
+## Available Mocks
+
+### Webhook Payloads (Receiving Side)
+
+For testing webhook processing:
+
+```typescript
+import { 
+  createDeliveredEvent,
+  createOpenEvent,
+  createBounceEvent,
+  createClickEvent,
+  createWebhookPayload,
+  REAL_DELIVERED_EVENT,
+  REAL_OPEN_EVENT
+} from '../mocks';
+
+// Create a mock delivered webhook
+const deliveredEvent = createDeliveredEvent({
+  email: 'user@example.com'
+});
+
+// Use real captured payload as-is
+const realPayload = REAL_DELIVERED_EVENT;
+
+// Create complete webhook with headers
+const webhook = createWebhookPayload(deliveredEvent);
+```
+
+### API Responses (Sending Side)
+
+For testing email sending:
+
+```typescript
+import {
+  createSuccessResponse,
+  createRateLimitResponse,
+  createValidationErrorResponse,
+  mockSuccessfulSend,
+  mockRateLimitThenSuccess
+} from '../mocks';
+
+// Mock successful send
+const successResponse = createSuccessResponse();
+
+// Mock rate limit then success (for retry testing)
+const fetchMock = mockRateLimitThenSuccess();
+```
+
+## Real Captured Payloads
+
+These constants contain **actual payloads** from SendGrid:
+
+| Constant | Description |
+|----------|-------------|
+| `REAL_SENDGRID_HEADERS` | Actual HTTP headers from webhook requests |
+| `REAL_DELIVERED_EVENT` | Real "delivered" event payload |
+| `REAL_OPEN_EVENT` | Real "open" event payload |
+| `REAL_DEFERRED_EVENT` | Real "deferred" event payload |
+
+## Test Scenarios
+
+Pre-built scenarios for common test cases:
+
+```typescript
+import {
+  createEmailLifecycleEvents,
+  createFailedDeliveryEvents,
+  createSpamScenarioEvents
+} from '../mocks';
+
+// Email lifecycle: delivered -> open -> click
+const lifecycle = createEmailLifecycleEvents('user@example.com');
+
+// Failed delivery: deferred -> deferred -> bounce
+const failedDelivery = createFailedDeliveryEvents('bad@example.com');
+
+// Spam scenario: delivered -> open -> spam report
+const spamReport = createSpamScenarioEvents('user@example.com');
+```
+
+## File Structure
+
+```
+__tests__/mocks/
+├── index.ts                      # Central exports
+├── sendgrid-webhook-payloads.ts  # Webhook receiving mocks
+├── sendgrid-api-responses.ts     # API sending mocks
+└── README.md                     # This file
+
+__tests__/fixtures/sendgrid-webhooks/
+├── delivered.json                # Raw captured payloads
+├── open.json
+├── deferred.json
+└── README.md
+```
+
+## Updating Mocks
+
+If SendGrid changes their payload format:
+
+1. Run the integration tests to capture new payloads:
+   ```bash
+   RUN_INTEGRATION_TESTS=true npm test -- sendgrid-webhook-integration --run
+   ```
+
+2. New payloads are saved to `__tests__/fixtures/sendgrid-webhooks/`
+
+3. Update the mock factories in `sendgrid-webhook-payloads.ts` with any new fields
+
+## Example Test
+
+```typescript
+// Jest provides describe, it, expect as globals; use jest.fn() instead of vi.fn()
+import { EmailWebhookEventsService } from '../../src';
+import { createDeliveredEvent, createWebhookPayload } from '../mocks';
+
+describe('EmailWebhookEventsService', () => {
+  it('should process a delivered webhook', async () => {
+    // Arrange - use mock based on real data
+    const event = createDeliveredEvent({ email: 'test@example.com' });
+    const payload = createWebhookPayload(event);
+    
+    const service = new EmailWebhookEventsService({...});
+    await service.initialize();
+    
+    // Act - use REAL service implementation
+    const result = await service.processWebhook({
+      provider: EmailProvider.SENDGRID,
+      payload: payload.body,
+      headers: payload.headers
+    });
+    
+    // Assert
+    expect(result.success).toBe(true);
+    expect(result.event?.type).toBe('delivered');
+  });
+});
+```
+
+## Contributing
+
+When adding new event types or updating mocks:
+
+1. Run integration tests to capture real payloads
+2. Add the new type interface
+3. Create a factory function with sensible defaults
+4. Add the type to the `SendGridWebhookEvent` union
+5. Export from `index.ts`
+