insforge 1.2.10 → 1.4.8

package/backend/src/infra/database/migrations/019_create-deployments-table.sql

@@ -0,0 +1,36 @@
+-- Migration: 019 - Create deployments table in system schema
+
+-- Create deployments table for tracking deployment requests and their status
+-- Designed to be provider-agnostic (Vercel, Netlify, Cloudflare, etc.)
+--
+-- Status flow:
+--   WAITING -> UPLOADING -> (Vercel statuses: QUEUED/BUILDING/READY/ERROR/CANCELED)
+--   InsForge statuses:
+--   - WAITING: Record created, waiting for client to upload zip to S3
+--   - UPLOADING: Server is downloading from S3 and uploading to Vercel
+--   Vercel statuses (stored directly):
+--   - QUEUED: Deployment queued
+--   - BUILDING: Deployment building
+--   - READY: Deployment ready
+--   - ERROR: Deployment failed
+--   - CANCELED: Deployment canceled
+CREATE TABLE IF NOT EXISTS system.deployments (
+  id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
+  provider TEXT NOT NULL DEFAULT 'vercel',
+  provider_deployment_id TEXT UNIQUE,  -- Provider's deployment ID, null until deployment starts
+  status TEXT NOT NULL DEFAULT 'WAITING',
+  url TEXT,
+  metadata JSONB,
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  updated_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Create indexes for better query performance
+CREATE INDEX IF NOT EXISTS idx_deployments_status ON system.deployments(status);
+CREATE INDEX IF NOT EXISTS idx_deployments_provider ON system.deployments(provider);
+CREATE INDEX IF NOT EXISTS idx_deployments_created_at ON system.deployments(created_at DESC);
+
+-- Add trigger for updated_at
+DROP TRIGGER IF EXISTS update_system_deployments_updated_at ON system.deployments;
+CREATE TRIGGER update_system_deployments_updated_at BEFORE UPDATE ON system.deployments
+FOR EACH ROW EXECUTE FUNCTION system.update_updated_at();

package/backend/src/infra/database/migrations/020_add-audio-modality.sql

@@ -0,0 +1,11 @@
+-- Migration: 020 - Expand modality constraints to all OpenRouter modalities
+-- Supports: text, image, audio, video, file
+-- Filtering is handled in application layer (shared-schemas + helpers.ts)
+
+ALTER TABLE ai.configs DROP CONSTRAINT IF EXISTS check_input_modality_valid;
+ALTER TABLE ai.configs ADD CONSTRAINT check_input_modality_valid
+CHECK (input_modality <@ '{text,image,audio,video,file}'::TEXT[]);
+
+ALTER TABLE ai.configs DROP CONSTRAINT IF EXISTS check_output_modality_valid;
+ALTER TABLE ai.configs ADD CONSTRAINT check_output_modality_valid
+CHECK (output_modality <@ '{text,image,audio,video,file}'::TEXT[]);

package/backend/src/infra/database/migrations/bootstrap/bootstrap-migrations.js

@@ -0,0 +1,103 @@
+/**
+ * Bootstrap script for migrations table migration
+ *
+ * This script handles the one-time migration of the node-pg-migrate tracking table
+ * from `public._migrations` to `system.migrations`.
+ *
+ * Why this is needed:
+ * - node-pg-migrate checks for the migrations table BEFORE running any migrations
+ * - If we try to move the table inside a migration file, node-pg-migrate will have
+ *   already looked for `system.migrations`, not found it, and created an empty one
+ * - This would cause all migrations to appear as "pending" and fail
+ *
+ * This script runs BEFORE node-pg-migrate and handles the table move gracefully.
+ */
+
+import pg from 'pg';
+// Note: This imports a TypeScript file. This works because the script is run with `tsx`
+// (see package.json migrate:bootstrap script), which can handle TypeScript imports.
+// The relative path goes up 4 levels: bootstrap -> migrations -> database -> infra -> src, then into utils.
+import logger from '@/utils/logger.js';
+
+const { Pool } = pg;
+
+async function bootstrapMigrations() {
+  // Use DATABASE_URL from environment (set by dotenv-cli in npm scripts)
+  const connectionString = process.env.DATABASE_URL;
+
+  if (!connectionString) {
+    logger.error('DATABASE_URL environment variable is not set');
+    process.exit(1);
+  }
+
+  const pool = new Pool({ connectionString });
+
+  try {
+    const client = await pool.connect();
+
+    try {
+      // Check if old _migrations table exists in public schema
+      const oldTableExists = await client.query(`
+        SELECT EXISTS (
+          SELECT 1 FROM information_schema.tables
+          WHERE table_schema = 'public' AND table_name = '_migrations'
+        ) as exists
+      `);
+
+      // Check if new system.migrations table already exists
+      const newTableExists = await client.query(`
+        SELECT EXISTS (
+          SELECT 1 FROM information_schema.tables
+          WHERE table_schema = 'system' AND table_name = 'migrations'
+        ) as exists
+      `);
+
+      if (oldTableExists.rows[0].exists && !newTableExists.rows[0].exists) {
+        logger.info('Bootstrap: Moving _migrations table to system.migrations...');
+
+        // Create system schema if it doesn't exist
+        await client.query('CREATE SCHEMA IF NOT EXISTS system');
+
+        // Move the table in a transaction to avoid partial state
+        await client.query('BEGIN');
+        try {
+          await client.query('ALTER TABLE public._migrations SET SCHEMA system');
+          await client.query('ALTER TABLE system._migrations RENAME TO migrations');
+          await client.query('COMMIT');
+        } catch (error) {
+          await client.query('ROLLBACK');
+          throw error;
+        }
+
+        logger.info('Bootstrap: Successfully moved _migrations to system.migrations');
+      } else if (newTableExists.rows[0].exists) {
+        // Already migrated, nothing to do
+        logger.info('Bootstrap: system.migrations already exists, skipping');
+      } else if (!oldTableExists.rows[0].exists && !newTableExists.rows[0].exists) {
+        // Fresh install - create system schema so node-pg-migrate can create its table there
+        logger.info('Bootstrap: No existing migrations table, fresh install');
+        await client.query('CREATE SCHEMA IF NOT EXISTS system');
+        logger.info('Bootstrap: Created system schema for migrations');
+      }
+    } finally {
+      client.release();
+    }
+  } catch (error) {
+    logger.error('Bootstrap migration failed', {
+      error: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+    process.exitCode = 1;
+  } finally {
+    await pool.end();
+  }
+}
+
+bootstrapMigrations().catch((error) => {
+  const message = error instanceof Error ? error.message : String(error);
+  logger.error('Bootstrap migration failed', {
+    error: message,
+    stack: error instanceof Error ? error.stack : undefined,
+  });
+  process.exitCode = 1;
+});

package/backend/src/infra/realtime/realtime.manager.ts

@@ -0,0 +1,246 @@
+import type { Client } from 'pg';
+import { SocketManager } from '@/infra/socket/socket.manager.js';
+import { WebhookSender } from './webhook-sender.js';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import { RealtimeChannelService } from '@/services/realtime/realtime-channel.service.js';
+import { RealtimeMessageService } from '@/services/realtime/realtime-message.service.js';
+import logger from '@/utils/logger.js';
+import type { RealtimeMessage, RealtimeChannel, WebhookMessage } from '@insforge/shared-schemas';
+import type { DeliveryResult } from '@/types/realtime.js';
+
+/**
+ * RealtimeManager - Listens to pg_notify and publishes messages to WebSocket/webhooks
+ *
+ * This is a singleton that:
+ * 1. Maintains a dedicated PostgreSQL connection for LISTEN
+ * 2. Receives notifications from realtime.publish() function
+ * 3. Publishes messages to WebSocket clients (via Socket.IO rooms)
+ * 4. Publishes messages to webhook URLs (via HTTP POST)
+ * 5. Updates message records with delivery statistics
+ */
+export class RealtimeManager {
+  private static instance: RealtimeManager;
+  private listenerClient: Client | null = null;
+  private isConnected = false;
+  private reconnectTimeout: NodeJS.Timeout | null = null;
+  private reconnectAttempts = 0;
+  private readonly maxReconnectAttempts = 10;
+  private readonly baseReconnectDelay = 5000;
+  private webhookSender: WebhookSender;
+
+  private constructor() {
+    this.webhookSender = new WebhookSender();
+  }
+
+  static getInstance(): RealtimeManager {
+    if (!RealtimeManager.instance) {
+      RealtimeManager.instance = new RealtimeManager();
+    }
+    return RealtimeManager.instance;
+  }
+
+  /**
+   * Initialize the realtime manager and start listening for pg_notify
+   */
+  async initialize(): Promise<void> {
+    if (this.isConnected) {
+      return;
+    }
+
+    // Create a dedicated client for LISTEN (cannot use pooled connections)
+    this.listenerClient = DatabaseManager.getInstance().createClient();
+
+    try {
+      await this.listenerClient.connect();
+      await this.listenerClient.query('LISTEN realtime_message');
+      this.isConnected = true;
+      this.reconnectAttempts = 0;
+
+      this.listenerClient.on('notification', (msg) => {
+        if (msg.channel === 'realtime_message' && msg.payload) {
+          void this.handlePGNotification(msg.payload);
+        }
+      });
+
+      this.listenerClient.on('error', (error) => {
+        logger.error('RealtimeManager connection error', { error: error.message });
+        this.handleDisconnect();
+      });
+
+      this.listenerClient.on('end', () => {
+        logger.warn('RealtimeManager connection ended');
+        this.handleDisconnect();
+      });
+
+      logger.info('RealtimeManager initialized and listening');
+    } catch (error) {
+      logger.error('Failed to initialize RealtimeManager', { error });
+      this.handleDisconnect();
+    }
+  }
+
+  /**
+   * Handle incoming pg_notify notification
+   * Payload is just the message_id (UUID string) to bypass 8KB limit
+   */
+  private async handlePGNotification(messageId: string): Promise<void> {
+    try {
+      // 1. Fetch message and channel in parallel
+      // channelId is guaranteed non-null for fresh messages (publish/insertMessage validate channel)
+      const message = await RealtimeMessageService.getInstance().getById(messageId);
+
+      if (!message || !message.channelId) {
+        logger.warn('Message not found or invalid for realtime notification', { messageId });
+        return;
+      }
+
+      // 2. Look up channel configuration (for enabled check and webhook URLs)
+      const channel = await RealtimeChannelService.getInstance().getById(message.channelId);
+
+      if (!channel?.enabled) {
+        logger.debug('Channel not found or disabled, skipping', { channelId: message.channelId });
+        return;
+      }
+
+      // 3. Publish to WebSocket and/or Webhooks
+      const result = await this.publishMessage(message, channel);
+
+      // 4. Update message record with delivery stats
+      await RealtimeMessageService.getInstance().updateDeliveryStats(messageId, result);
+
+      logger.debug('Realtime message published', {
+        messageId,
+        channelName: message.channelName,
+        eventName: message.eventName,
+        ...result,
+      });
+    } catch (error) {
+      logger.error('Failed to publish realtime message', { error, messageId });
+    }
+  }
+
+  /**
+   * Publish message to WebSocket clients and webhook URLs
+   */
+  private async publishMessage(
+    message: RealtimeMessage,
+    channel: RealtimeChannel
+  ): Promise<DeliveryResult> {
+    const result: DeliveryResult = {
+      wsAudienceCount: 0,
+      whAudienceCount: 0,
+      whDeliveredCount: 0,
+    };
+
+    // Publish to WebSocket clients
+    result.wsAudienceCount = this.publishToWebSocket(message);
+
+    // Publish to Webhook URLs if configured
+    if (channel.webhookUrls && channel.webhookUrls.length > 0) {
+      const webhookPayload: WebhookMessage = {
+        messageId: message.id,
+        channel: message.channelName,
+        eventName: message.eventName,
+        payload: message.payload,
+      };
+      const whResult = await this.publishToWebhooks(channel.webhookUrls, webhookPayload);
+      result.whAudienceCount = whResult.audienceCount;
+      result.whDeliveredCount = whResult.deliveredCount;
+    }
+
+    return result;
+  }
+
+  /**
+   * Publish message to WebSocket clients subscribed to the channel
+   * Returns the number of clients in the room (audience count)
+   */
+  private publishToWebSocket(message: RealtimeMessage): number {
+    const socketManager = SocketManager.getInstance();
+    const roomName = `realtime:${message.channelName}`;
+
+    const audienceCount = socketManager.getRoomSize(roomName);
+
+    if (audienceCount > 0) {
+      socketManager.broadcastToRoom(
+        roomName,
+        message.eventName,
+        message.payload,
+        message.senderType,
+        message.senderId ?? undefined,
+        message.id
+      );
+    }
+
+    return audienceCount;
+  }
+
+  /**
+   * Publish message to all configured webhook URLs
+   */
+  private async publishToWebhooks(
+    urls: string[],
+    message: WebhookMessage
+  ): Promise<{ audienceCount: number; deliveredCount: number }> {
+    const audienceCount = urls.length;
+    const results = await this.webhookSender.sendToAll(urls, message);
+    const deliveredCount = results.filter((r) => r.success).length;
+
+    return { audienceCount, deliveredCount };
+  }
+
+  /**
+   * Handle disconnection and attempt reconnection
+   */
+  private handleDisconnect(): void {
+    this.isConnected = false;
+
+    if (this.listenerClient) {
+      this.listenerClient.removeAllListeners();
+      this.listenerClient = null;
+    }
+
+    // Reconnect with exponential backoff
+    if (this.reconnectAttempts < this.maxReconnectAttempts) {
+      const delay = this.baseReconnectDelay * Math.pow(2, this.reconnectAttempts);
+      this.reconnectAttempts++;
+
+      if (!this.reconnectTimeout) {
+        this.reconnectTimeout = setTimeout(() => {
+          this.reconnectTimeout = null;
+          logger.info(
+            `Attempting to reconnect RealtimeManager (attempt ${this.reconnectAttempts})...`
+          );
+          void this.initialize();
+        }, delay);
+      }
+    } else {
+      logger.error('RealtimeManager max reconnect attempts reached');
+    }
+  }
+
+  /**
+   * Close the realtime manager connection
+   */
+  async close(): Promise<void> {
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+      this.reconnectTimeout = null;
+    }
+
+    if (this.listenerClient) {
+      this.listenerClient.removeAllListeners();
+      await this.listenerClient.end();
+      this.listenerClient = null;
+      this.isConnected = false;
+      logger.info('RealtimeManager closed');
+    }
+  }
+
+  /**
+   * Check if the manager is connected and healthy
+   */
+  isHealthy(): boolean {
+    return this.isConnected;
+  }
+}

package/backend/src/infra/realtime/webhook-sender.ts

@@ -0,0 +1,82 @@
+import axios, { AxiosError } from 'axios';
+import logger from '@/utils/logger.js';
+import type { WebhookMessage } from '@insforge/shared-schemas';
+
+export interface WebhookResult {
+  url: string;
+  success: boolean;
+  statusCode?: number;
+  error?: string;
+}
+
+/**
+ * WebhookSender - Handles HTTP delivery of realtime messages to webhook endpoints
+ */
+export class WebhookSender {
+  private readonly timeout = 10000; // 10 seconds
+  private readonly maxRetries = 2;
+
+  /**
+   * Send message to all webhook URLs in parallel
+   */
+  async sendToAll(urls: string[], message: WebhookMessage): Promise<WebhookResult[]> {
+    const promises = urls.map((url) => this.send(url, message));
+    return Promise.all(promises);
+  }
+
+  /**
+   * Send message to a single webhook URL with retry logic
+   */
+  private async send(url: string, message: WebhookMessage): Promise<WebhookResult> {
+    let lastError: string | undefined;
+
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      try {
+        const response = await axios.post(url, message.payload, {
+          timeout: this.timeout,
+          headers: {
+            'Content-Type': 'application/json',
+            'X-InsForge-Event': message.eventName,
+            'X-InsForge-Channel': message.channel,
+            'X-InsForge-Message-Id': message.messageId,
+          },
+        });
+
+        return {
+          url,
+          success: response.status >= 200 && response.status < 300,
+          statusCode: response.status,
+        };
+      } catch (error) {
+        const axiosError = error as AxiosError;
+        lastError = axiosError.message;
+
+        if (axiosError.response) {
+          // Server responded with error status - don't retry
+          return {
+            url,
+            success: false,
+            statusCode: axiosError.response.status,
+            error: `HTTP ${axiosError.response.status}`,
+          };
+        }
+
+        // Network error - retry with backoff
+        if (attempt < this.maxRetries) {
+          await this.delay(1000 * (attempt + 1)); // 1s, 2s
+        }
+      }
+    }
+    logger.warn('Webhook delivery failed after retries', { url, error: lastError });
+
+    return {
+      url,
+      success: false,
+      error: lastError,
+    };
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}