insforge 0.3.3 → 1.3.0

package/backend/src/infra/database/migrations/017_create-realtime-schema.sql

@@ -0,0 +1,233 @@
+-- Migration 017: Create Realtime Schema
+--
+-- Creates the realtime schema with:
+-- 1. channels table - Channel definitions with webhook configuration
+-- 2. messages table - All realtime messages with delivery statistics
+-- 3. publish() function - Called by developer triggers to publish events
+--
+-- Permission Model (Supabase pattern):
+-- - SELECT on channels = 'subscribe' permission (subscribe to channel)
+-- - INSERT on messages = 'publish' permission (publish to channel)
+-- Developers define RLS policies on channels/messages table to control access.
+
+-- ============================================================================
+-- CREATE SCHEMA
+-- ============================================================================
+
+CREATE SCHEMA IF NOT EXISTS realtime;
+
+-- ============================================================================
+-- CHANNELS TABLE
+-- ============================================================================
+-- Stores channel definitions with delivery configuration.
+-- RLS policies control subscribe permissions.
+-- - SELECT policy = 'subscribe' permission (can subscribe to channel)
+-- Channel names use : as separator and % for wildcards (LIKE pattern).
+-- Examples: "orders", "order:%", "chat:%:messages"
+
+CREATE TABLE IF NOT EXISTS realtime.channels (
+  id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
+
+  -- Channel name pattern (e.g., "orders", "order:%", "chat:%:messages")
+  -- Convention: use : as separator, % for wildcards (LIKE pattern)
+  pattern TEXT UNIQUE NOT NULL,
+
+  -- Human-readable description
+  description TEXT,
+
+  -- Webhook URLs to POST events to (NULL or empty array = no webhooks)
+  webhook_urls TEXT[],
+
+  -- Whether this channel is active
+  enabled BOOLEAN DEFAULT TRUE NOT NULL,
+
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  updated_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- ============================================================================
+-- MESSAGES TABLE
+-- ============================================================================
+-- Stores all realtime messages published through the system.
+-- RLS policies on this table control publish permissions:
+-- - INSERT policy = 'publish' permission (can publish to channel)
+
+CREATE TABLE IF NOT EXISTS realtime.messages (
+  id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
+
+  -- Event metadata
+  event_name TEXT NOT NULL,
+
+  -- Channel reference (SET NULL on delete to preserve history)
+  channel_id UUID REFERENCES realtime.channels(id) ON DELETE SET NULL,
+  channel_name TEXT NOT NULL, -- Denormalized for query convenience after channel deletion
+
+  -- Event payload (stored for audit/replay purposes)
+  payload JSONB DEFAULT '{}'::jsonb NOT NULL,
+
+  -- Sender information
+  -- 'system' = triggered by database trigger (via publish() function)
+  -- 'user' = published by client via WebSocket
+  sender_type TEXT DEFAULT 'system' NOT NULL CHECK (sender_type IN ('system', 'user')),
+  sender_id UUID, -- User ID for 'user' type, NULL for 'system' type
+
+  -- Delivery statistics for WebSocket
+  ws_audience_count INTEGER DEFAULT 0 NOT NULL, -- How many clients were subscribed
+
+  -- Delivery statistics for Webhooks
+  wh_audience_count INTEGER DEFAULT 0 NOT NULL, -- How many webhook URLs configured
+  wh_delivered_count INTEGER DEFAULT 0 NOT NULL, -- How many succeeded (2xx response)
+
+  created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- ============================================================================
+-- INDEXES
+-- ============================================================================
+
+CREATE INDEX IF NOT EXISTS idx_realtime_channels_pattern ON realtime.channels(pattern);
+CREATE INDEX IF NOT EXISTS idx_realtime_channels_enabled ON realtime.channels(enabled);
+CREATE INDEX IF NOT EXISTS idx_realtime_messages_channel_id ON realtime.messages(channel_id);
+CREATE INDEX IF NOT EXISTS idx_realtime_messages_channel_name ON realtime.messages(channel_name);
+CREATE INDEX IF NOT EXISTS idx_realtime_messages_created_at ON realtime.messages(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_realtime_messages_event_name ON realtime.messages(event_name);
+CREATE INDEX IF NOT EXISTS idx_realtime_messages_sender ON realtime.messages(sender_type, sender_id);
+
+-- ============================================================================
+-- UPDATED_AT TRIGGER
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION realtime.update_updated_at()
+RETURNS TRIGGER AS $$
+BEGIN
+  NEW.updated_at = NOW();
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_channels_updated_at ON realtime.channels;
+CREATE TRIGGER trg_channels_updated_at
+BEFORE UPDATE ON realtime.channels
+FOR EACH ROW EXECUTE FUNCTION realtime.update_updated_at();
+
+-- ============================================================================
+-- ROW LEVEL SECURITY (DISABLED BY DEFAULT)
+-- ============================================================================
+-- RLS is disabled by default for the best developer experience.
+-- Channels and messages are open to all authenticated/anon users out of the box.
+--
+-- To enable access control, developers can:
+--   1. Enable RLS: ALTER TABLE realtime.channels ENABLE ROW LEVEL SECURITY;
+--   2. Add policies using the helper function realtime.channel_name()
+--
+-- See documentation for policy examples.
+
+-- ============================================================================
+-- HELPER FUNCTIONS FOR RLS POLICIES
+-- ============================================================================
+
+-- Returns the channel name being accessed (set by backend during permission checks)
+-- Developers use this in policies instead of writing current_setting directly
+CREATE OR REPLACE FUNCTION realtime.channel_name()
+RETURNS TEXT AS $$
+  SELECT current_setting('realtime.channel_name', true);
+$$ LANGUAGE SQL STABLE;
+
+-- ============================================================================
+-- PUBLISH FUNCTION
+-- ============================================================================
+-- Called by developer triggers to publish events to channels.
+-- This function can only be executed by the backend (SECURITY DEFINER).
+--
+-- Usage in a trigger:
+--   PERFORM realtime.publish(
+--     'order:' || NEW.id::text,  -- channel name (resolved instance)
+--     'order_updated',           -- event name
+--     jsonb_build_object('id', NEW.id, 'status', NEW.status)  -- payload
+--   );
+
+CREATE OR REPLACE FUNCTION realtime.publish(
+  p_channel_name TEXT,
+  p_event_name TEXT,
+  p_payload JSONB
+)
+RETURNS UUID AS $$
+DECLARE
+  v_channel_id UUID;
+  v_message_id UUID;
+BEGIN
+  -- Find matching channel: exact match first, then wildcard pattern match
+  -- For wildcard patterns like "order:%", check if p_channel_name LIKE pattern
+  SELECT id INTO v_channel_id
+  FROM realtime.channels
+  WHERE enabled = TRUE
+    AND (pattern = p_channel_name OR p_channel_name LIKE pattern)
+  ORDER BY pattern = p_channel_name DESC
+  LIMIT 1;
+
+  -- If no channel found, raise a warning and return NULL
+  IF v_channel_id IS NULL THEN
+    RAISE WARNING 'Realtime: No matching channel found for "%"', p_channel_name;
+    RETURN NULL;
+  END IF;
+
+  -- Insert message record (system-triggered, so sender_type = 'system')
+  INSERT INTO realtime.messages (
+    event_name,
+    channel_id,
+    channel_name,
+    payload,
+    sender_type
+  ) VALUES (
+    p_event_name,
+    v_channel_id,
+    p_channel_name,
+    p_payload,
+    'system'
+  )
+  RETURNING id INTO v_message_id;
+
+  RETURN v_message_id;
+END;
+$$ LANGUAGE plpgsql SECURITY DEFINER;
+
+-- Revoke execute from public, only backend can call this
+REVOKE ALL ON FUNCTION realtime.publish FROM PUBLIC;
+
+-- ============================================================================
+-- TRIGGER FOR PG_NOTIFY
+-- ============================================================================
+-- Trigger function that sends pg_notify for every new message (both system and client).
+
+CREATE OR REPLACE FUNCTION realtime.notify_on_message_insert()
+RETURNS TRIGGER AS $$
+BEGIN
+  -- Send only message_id to bypass pg_notify 8KB payload limit
+  -- Backend will fetch full message from DB
+  PERFORM pg_notify('realtime_message', NEW.id::text);
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Create trigger on messages table
+DROP TRIGGER IF EXISTS trg_message_notify ON realtime.messages;
+CREATE TRIGGER trg_message_notify
+  AFTER INSERT ON realtime.messages
+  FOR EACH ROW
+  EXECUTE FUNCTION realtime.notify_on_message_insert();
+
+-- ============================================================================
+-- GRANTS
+-- ============================================================================
+
+-- Grant schema access to both authenticated and anonymous users
+GRANT USAGE ON SCHEMA realtime TO authenticated, anon;
+
+-- Grant SELECT on channels table (allows subscribe)
+GRANT SELECT ON realtime.channels TO authenticated, anon;
+
+-- Grant INSERT on messages table (allows publish)
+GRANT INSERT ON realtime.messages TO authenticated, anon;
+
+-- Grant execution permission on helper function (used when RLS is enabled)
+GRANT EXECUTE ON FUNCTION realtime.channel_name() TO authenticated, anon;

package/backend/src/infra/realtime/realtime.manager.ts

@@ -0,0 +1,246 @@
+import type { Client } from 'pg';
+import { SocketManager } from '@/infra/socket/socket.manager.js';
+import { WebhookSender } from './webhook-sender.js';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import { RealtimeChannelService } from '@/services/realtime/realtime-channel.service.js';
+import { RealtimeMessageService } from '@/services/realtime/realtime-message.service.js';
+import logger from '@/utils/logger.js';
+import type { RealtimeMessage, RealtimeChannel, WebhookMessage } from '@insforge/shared-schemas';
+import type { DeliveryResult } from '@/types/realtime.js';
+
+/**
+ * RealtimeManager - Listens to pg_notify and publishes messages to WebSocket/webhooks
+ *
+ * This is a singleton that:
+ * 1. Maintains a dedicated PostgreSQL connection for LISTEN
+ * 2. Receives notifications from realtime.publish() function
+ * 3. Publishes messages to WebSocket clients (via Socket.IO rooms)
+ * 4. Publishes messages to webhook URLs (via HTTP POST)
+ * 5. Updates message records with delivery statistics
+ */
+export class RealtimeManager {
+  private static instance: RealtimeManager;
+  private listenerClient: Client | null = null;
+  private isConnected = false;
+  private reconnectTimeout: NodeJS.Timeout | null = null;
+  private reconnectAttempts = 0;
+  private readonly maxReconnectAttempts = 10;
+  private readonly baseReconnectDelay = 5000;
+  private webhookSender: WebhookSender;
+
+  private constructor() {
+    this.webhookSender = new WebhookSender();
+  }
+
+  static getInstance(): RealtimeManager {
+    if (!RealtimeManager.instance) {
+      RealtimeManager.instance = new RealtimeManager();
+    }
+    return RealtimeManager.instance;
+  }
+
+  /**
+   * Initialize the realtime manager and start listening for pg_notify
+   */
+  async initialize(): Promise<void> {
+    if (this.isConnected) {
+      return;
+    }
+
+    // Create a dedicated client for LISTEN (cannot use pooled connections)
+    this.listenerClient = DatabaseManager.getInstance().createClient();
+
+    try {
+      await this.listenerClient.connect();
+      await this.listenerClient.query('LISTEN realtime_message');
+      this.isConnected = true;
+      this.reconnectAttempts = 0;
+
+      this.listenerClient.on('notification', (msg) => {
+        if (msg.channel === 'realtime_message' && msg.payload) {
+          void this.handlePGNotification(msg.payload);
+        }
+      });
+
+      this.listenerClient.on('error', (error) => {
+        logger.error('RealtimeManager connection error', { error: error.message });
+        this.handleDisconnect();
+      });
+
+      this.listenerClient.on('end', () => {
+        logger.warn('RealtimeManager connection ended');
+        this.handleDisconnect();
+      });
+
+      logger.info('RealtimeManager initialized and listening');
+    } catch (error) {
+      logger.error('Failed to initialize RealtimeManager', { error });
+      this.handleDisconnect();
+    }
+  }
+
+  /**
+   * Handle incoming pg_notify notification
+   * Payload is just the message_id (UUID string) to bypass 8KB limit
+   */
+  private async handlePGNotification(messageId: string): Promise<void> {
+    try {
+      // 1. Fetch message and channel in parallel
+      // channelId is guaranteed non-null for fresh messages (publish/insertMessage validate channel)
+      const message = await RealtimeMessageService.getInstance().getById(messageId);
+
+      if (!message || !message.channelId) {
+        logger.warn('Message not found or invalid for realtime notification', { messageId });
+        return;
+      }
+
+      // 2. Look up channel configuration (for enabled check and webhook URLs)
+      const channel = await RealtimeChannelService.getInstance().getById(message.channelId);
+
+      if (!channel?.enabled) {
+        logger.debug('Channel not found or disabled, skipping', { channelId: message.channelId });
+        return;
+      }
+
+      // 3. Publish to WebSocket and/or Webhooks
+      const result = await this.publishMessage(message, channel);
+
+      // 4. Update message record with delivery stats
+      await RealtimeMessageService.getInstance().updateDeliveryStats(messageId, result);
+
+      logger.debug('Realtime message published', {
+        messageId,
+        channelName: message.channelName,
+        eventName: message.eventName,
+        ...result,
+      });
+    } catch (error) {
+      logger.error('Failed to publish realtime message', { error, messageId });
+    }
+  }
+
+  /**
+   * Publish message to WebSocket clients and webhook URLs
+   */
+  private async publishMessage(
+    message: RealtimeMessage,
+    channel: RealtimeChannel
+  ): Promise<DeliveryResult> {
+    const result: DeliveryResult = {
+      wsAudienceCount: 0,
+      whAudienceCount: 0,
+      whDeliveredCount: 0,
+    };
+
+    // Publish to WebSocket clients
+    result.wsAudienceCount = this.publishToWebSocket(message);
+
+    // Publish to Webhook URLs if configured
+    if (channel.webhookUrls && channel.webhookUrls.length > 0) {
+      const webhookPayload: WebhookMessage = {
+        messageId: message.id,
+        channel: message.channelName,
+        eventName: message.eventName,
+        payload: message.payload,
+      };
+      const whResult = await this.publishToWebhooks(channel.webhookUrls, webhookPayload);
+      result.whAudienceCount = whResult.audienceCount;
+      result.whDeliveredCount = whResult.deliveredCount;
+    }
+
+    return result;
+  }
+
+  /**
+   * Publish message to WebSocket clients subscribed to the channel
+   * Returns the number of clients in the room (audience count)
+   */
+  private publishToWebSocket(message: RealtimeMessage): number {
+    const socketManager = SocketManager.getInstance();
+    const roomName = `realtime:${message.channelName}`;
+
+    const audienceCount = socketManager.getRoomSize(roomName);
+
+    if (audienceCount > 0) {
+      socketManager.broadcastToRoom(
+        roomName,
+        message.eventName,
+        message.payload,
+        message.senderType,
+        message.senderId ?? undefined,
+        message.id
+      );
+    }
+
+    return audienceCount;
+  }
+
+  /**
+   * Publish message to all configured webhook URLs
+   */
+  private async publishToWebhooks(
+    urls: string[],
+    message: WebhookMessage
+  ): Promise<{ audienceCount: number; deliveredCount: number }> {
+    const audienceCount = urls.length;
+    const results = await this.webhookSender.sendToAll(urls, message);
+    const deliveredCount = results.filter((r) => r.success).length;
+
+    return { audienceCount, deliveredCount };
+  }
+
+  /**
+   * Handle disconnection and attempt reconnection
+   */
+  private handleDisconnect(): void {
+    this.isConnected = false;
+
+    if (this.listenerClient) {
+      this.listenerClient.removeAllListeners();
+      this.listenerClient = null;
+    }
+
+    // Reconnect with exponential backoff
+    if (this.reconnectAttempts < this.maxReconnectAttempts) {
+      const delay = this.baseReconnectDelay * Math.pow(2, this.reconnectAttempts);
+      this.reconnectAttempts++;
+
+      if (!this.reconnectTimeout) {
+        this.reconnectTimeout = setTimeout(() => {
+          this.reconnectTimeout = null;
+          logger.info(
+            `Attempting to reconnect RealtimeManager (attempt ${this.reconnectAttempts})...`
+          );
+          void this.initialize();
+        }, delay);
+      }
+    } else {
+      logger.error('RealtimeManager max reconnect attempts reached');
+    }
+  }
+
+  /**
+   * Close the realtime manager connection
+   */
+  async close(): Promise<void> {
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+      this.reconnectTimeout = null;
+    }
+
+    if (this.listenerClient) {
+      this.listenerClient.removeAllListeners();
+      await this.listenerClient.end();
+      this.listenerClient = null;
+      this.isConnected = false;
+      logger.info('RealtimeManager closed');
+    }
+  }
+
+  /**
+   * Check if the manager is connected and healthy
+   */
+  isHealthy(): boolean {
+    return this.isConnected;
+  }
+}

package/backend/src/infra/realtime/webhook-sender.ts

@@ -0,0 +1,82 @@
+import axios, { AxiosError } from 'axios';
+import logger from '@/utils/logger.js';
+import type { WebhookMessage } from '@insforge/shared-schemas';
+
+export interface WebhookResult {
+  url: string;
+  success: boolean;
+  statusCode?: number;
+  error?: string;
+}
+
+/**
+ * WebhookSender - Handles HTTP delivery of realtime messages to webhook endpoints
+ */
+export class WebhookSender {
+  private readonly timeout = 10000; // 10 seconds
+  private readonly maxRetries = 2;
+
+  /**
+   * Send message to all webhook URLs in parallel
+   */
+  async sendToAll(urls: string[], message: WebhookMessage): Promise<WebhookResult[]> {
+    const promises = urls.map((url) => this.send(url, message));
+    return Promise.all(promises);
+  }
+
+  /**
+   * Send message to a single webhook URL with retry logic
+   */
+  private async send(url: string, message: WebhookMessage): Promise<WebhookResult> {
+    let lastError: string | undefined;
+
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      try {
+        const response = await axios.post(url, message.payload, {
+          timeout: this.timeout,
+          headers: {
+            'Content-Type': 'application/json',
+            'X-InsForge-Event': message.eventName,
+            'X-InsForge-Channel': message.channel,
+            'X-InsForge-Message-Id': message.messageId,
+          },
+        });
+
+        return {
+          url,
+          success: response.status >= 200 && response.status < 300,
+          statusCode: response.status,
+        };
+      } catch (error) {
+        const axiosError = error as AxiosError;
+        lastError = axiosError.message;
+
+        if (axiosError.response) {
+          // Server responded with error status - don't retry
+          return {
+            url,
+            success: false,
+            statusCode: axiosError.response.status,
+            error: `HTTP ${axiosError.response.status}`,
+          };
+        }
+
+        // Network error - retry with backoff
+        if (attempt < this.maxRetries) {
+          await this.delay(1000 * (attempt + 1)); // 1s, 2s
+        }
+      }
+    }
+    logger.warn('Webhook delivery failed after retries', { url, error: lastError });
+
+    return {
+      url,
+      success: false,
+      error: lastError,
+    };
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}

package/backend/src/{core/secrets/encryption.ts → infra/security/encryption.manager.ts}

@@ -1,9 +1,10 @@
 import crypto from 'crypto';
 
 /**
- * Encryption utilities for secrets management
+ * EncryptionManager - Handles encryption/decryption operations
+ * Infrastructure layer for secrets encryption
  */
-export class EncryptionUtils {
+export class EncryptionManager {
   private static encryptionKey: Buffer | null = null;
 
   private static getEncryptionKey(): Buffer {