@realtimex/email-automator 2.10.9 → 2.11.0

package/api/server.ts

@@ -4,6 +4,9 @@ import path, { join } from 'path';
 import { fileURLToPath } from 'url';
 import { spawn } from 'child_process';
 import { existsSync } from 'fs';
+import axios from 'axios';
+import crypto from 'crypto';
+import { SDKService } from './src/services/SDKService.js';
 import { config, validateConfig } from './src/config/index.js';
 import { errorHandler } from './src/middleware/errorHandler.js';
 import { apiRateLimit } from './src/middleware/rateLimit.js';
@@ -21,6 +24,9 @@ if (!configValidation.valid) {
     logger.warn('Configuration warnings', { errors: configValidation.errors });
 }
 
+// Initialize RealTimeX SDK
+SDKService.initialize();
+
 const app = express();
 
 // Security headers
@@ -36,8 +42,8 @@ app.use((req, res, next) => {
 
 // CORS configuration
 app.use(cors({
-    origin: config.isProduction 
-        ? config.security.corsOrigins 
+    origin: config.isProduction
+        ? config.security.corsOrigins
         : true, // Allow all in development
     credentials: true,
     methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'],
@@ -67,6 +73,189 @@ app.use('/api', apiRateLimit);
 // API routes
 app.use('/api', routes);
 
+// --- SETUP & PROVISIONING (Golden Standard) ---
+
+// GET /api/setup/organizations - List Supabase organizations using access token
+app.get('/api/setup/organizations', async (req, res) => {
+    const authHeader = req.headers['authorization'];
+    if (!authHeader) {
+        return res.status(401).json({ error: 'Missing Authorization header' });
+    }
+
+    try {
+        const response = await axios.get('https://api.supabase.com/v1/organizations', {
+            headers: { 'Authorization': authHeader }
+        });
+        res.json(response.data);
+    } catch (error: any) {
+        logger.error('Failed to fetch organizations', error);
+        res.status(error.response?.status || 500).json({
+            error: error.response?.data?.message || 'Failed to fetch organizations'
+        });
+    }
+});
+
+// POST /api/setup/auto-provision - Create project and poll for readiness (SSE)
+app.post('/api/setup/auto-provision', async (req, res) => {
+    const { orgId, projectName: customProjectName, region: customRegion } = req.body;
+    const authHeader = req.headers['authorization'];
+
+    if (!orgId) {
+        return res.status(400).json({ error: 'Missing required parameter (orgId)' });
+    }
+
+    if (!authHeader) {
+        return res.status(401).json({ error: 'Missing Authorization header' });
+    }
+
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Connection', 'keep-alive');
+
+    const sendEvent = (type: string, data: any) => {
+        res.write(`data: ${JSON.stringify({ type, data })}\n\n`);
+    };
+
+    try {
+        const projectName = customProjectName || `Email-Automator-${crypto.randomBytes(2).toString('hex')}`;
+        const region = customRegion || 'us-east-1';
+
+        // Generate a secure DB password server-side
+        const dbPass = crypto.randomBytes(16).toString('base64')
+            .replace(/\+/g, 'a')
+            .replace(/\//g, 'b')
+            .replace(/=/g, 'c') + '1!Aa';
+
+        sendEvent('info', `🚀 Creating Supabase project: ${projectName} in ${region}...`);
+
+        // 1. Create Project
+        const createResponse = await axios.post('https://api.supabase.com/v1/projects', {
+            name: projectName,
+            organization_id: orgId,
+            region: region,
+            db_pass: dbPass
+        }, {
+            headers: { 'Authorization': authHeader },
+            timeout: 15000
+        });
+
+        const project = createResponse.data;
+        const projectRef = project.id;
+
+        sendEvent('info', `📦 Project created! ID: ${projectRef}. Waiting for it to go live...`);
+        sendEvent('project_id', projectRef);
+
+        // 2. Poll for Readiness
+        let isReady = false;
+        let attempts = 0;
+        const maxAttempts = 60; // 5 minutes
+
+        while (!isReady && attempts < maxAttempts) {
+            attempts++;
+            await new Promise(resolve => setTimeout(resolve, 5000));
+
+            try {
+                const statusResponse = await axios.get(`https://api.supabase.com/v1/projects/${projectRef}`, {
+                    headers: { 'Authorization': authHeader },
+                    timeout: 10000
+                });
+
+                const status = statusResponse.data.status;
+                sendEvent('info', `⏳ Status: ${status} (Attempt ${attempts}/${maxAttempts})`);
+
+                if (status === 'ACTIVE_HEALTHY' || status === 'ACTIVE') {
+                    isReady = true;
+                }
+            } catch (pollError: any) {
+                logger.warn('Polling error during provision', { error: pollError.message });
+            }
+        }
+
+        if (!isReady) {
+            throw new Error('Project provision timed out after 5 minutes.');
+        }
+
+        // 3. Get API Keys
+        sendEvent('info', '🔑 Retrieving API keys...');
+
+        let anonKey = '';
+        let keyAttempts = 0;
+        const maxKeyAttempts = 10;
+
+        while (!anonKey && keyAttempts < maxKeyAttempts) {
+            keyAttempts++;
+            if (keyAttempts > 1) {
+                sendEvent('info', `⏳ API keys not ready yet. Retrying (Attempt ${keyAttempts}/${maxKeyAttempts})...`);
+                await new Promise(resolve => setTimeout(resolve, 3000));
+            }
+
+            try {
+                const keysResponse = await axios.get(`https://api.supabase.com/v1/projects/${projectRef}/api-keys`, {
+                    headers: { 'Authorization': authHeader },
+                    timeout: 10000
+                });
+
+                const keys = keysResponse.data;
+                if (Array.isArray(keys)) {
+                    anonKey = keys.find((k: any) => k.name === 'anon')?.api_key;
+                    if (anonKey) {
+                        sendEvent('info', '✅ API keys retrieved successfully.');
+                    }
+                }
+            } catch (err: any) {
+                logger.warn('Key retrieval attempt failed', { error: err.message });
+            }
+        }
+
+        if (!anonKey) {
+            throw new Error('Could not find anonymous API key for the new project.');
+        }
+
+        const supabaseUrl = `https://${projectRef}.supabase.co`;
+
+        // 4. DNS Verification
+        sendEvent('info', '🌐 Waiting for DNS propagation...');
+        let dnsReady = false;
+        let dnsAttempts = 0;
+        const maxDnsAttempts = 20;
+
+        while (!dnsReady && dnsAttempts < maxDnsAttempts) {
+            dnsAttempts++;
+            try {
+                const pingResponse = await axios.get(`${supabaseUrl}/rest/v1/`, {
+                    timeout: 5000,
+                    validateStatus: () => true
+                });
+                if (pingResponse.status < 500) {
+                    dnsReady = true;
+                    sendEvent('info', '✨ DNS resolved! Project is fully accessible.');
+                }
+            } catch (pingError: any) {
+                if (dnsAttempts % 5 === 0) {
+                    sendEvent('info', '⏳ DNS still propagating... standby.');
+                }
+                await new Promise(resolve => setTimeout(resolve, 3000));
+            }
+        }
+
+        sendEvent('success', {
+            url: supabaseUrl,
+            anonKey: anonKey,
+            projectId: projectRef,
+            dbPass: dbPass
+        });
+
+        sendEvent('done', 'success');
+    } catch (error: any) {
+        const errorMsg = error.response?.data?.message || error.message || 'Auto-provisioning failed';
+        logger.error('Auto-provision failed', { error: errorMsg });
+        sendEvent('error', errorMsg);
+        sendEvent('done', 'failed');
+    } finally {
+        res.end();
+    }
+});
+
 // Robust resolution for static assets (dist folder)
 function getDistPath() {
     // 1. Check environment variable override (must contain index.html)
@@ -103,9 +292,9 @@ app.get(/.*/, (req, res, next) => {
     res.sendFile(path.join(distPath, 'index.html'), (err) => {
         if (err) {
             // If dist doesn't exist, return 404 for non-API routes
-            res.status(404).json({ 
-                success: false, 
-                error: { code: 'NOT_FOUND', message: 'Frontend not built or endpoint not found' } 
+            res.status(404).json({
+                success: false,
+                error: { code: 'NOT_FOUND', message: 'Frontend not built or endpoint not found' }
             });
         }
     });
@@ -131,7 +320,7 @@ const server = app.listen(config.port, () => {
         environment: config.nodeEnv,
         supabase: getServerSupabase() ? 'connected' : 'not configured',
     });
-    
+
     // Start background scheduler
     if (getServerSupabase()) {
         startScheduler();

package/api/src/middleware/validation.ts

@@ -82,8 +82,7 @@ export const schemas = {
     // Migration schemas
     migrate: z.object({
         projectRef: z.string().min(1, 'Project reference is required'),
-        dbPassword: z.string().optional(),
-        accessToken: z.string().optional(),
+        accessToken: z.string().min(1, 'Access token is required for automatic migration'),
     }),
 
     // Rule schemas - supports both single action (legacy) and actions array
@@ -116,9 +115,8 @@ export const schemas = {
 
     // Settings schemas
     updateSettings: z.object({
+        llm_provider: z.string().optional(),
         llm_model: z.string().optional(),
-        llm_base_url: z.string().url().optional().or(z.literal('')),
-        llm_api_key: z.string().optional(),
         auto_trash_spam: z.boolean().optional(),
         smart_drafts: z.boolean().optional(),
         sync_interval_minutes: z.number().min(1).max(60).optional(),

package/api/src/routes/actions.ts

@@ -152,16 +152,7 @@ router.post('/draft/:emailId',
             throw new NotFoundError('Email');
         }
 
-        // Get user settings for LLM config
-        const { data: settings } = await req.supabase!
-            .from('user_settings')
-            .select('llm_model, llm_base_url')
-            .eq('user_id', userId)
-            .single();
-
-        const intelligenceService = getIntelligenceService(
-            settings ? { model: settings.llm_model, baseUrl: settings.llm_base_url } : undefined
-        );
+        const intelligenceService = getIntelligenceService();
 
         const draft = await intelligenceService.generateDraftReply(
             {

package/api/src/routes/index.ts

@@ -7,6 +7,7 @@ import rulesRoutes from './rules.js';
 import settingsRoutes from './settings.js';
 import emailsRoutes from './emails.js';
 import migrateRoutes from './migrate.js';
+import sdkRoutes from './sdk.js';
 
 const router = Router();
 
@@ -18,5 +19,6 @@ router.use('/rules', rulesRoutes);
 router.use('/settings', settingsRoutes);
 router.use('/emails', emailsRoutes);
 router.use('/migrate', migrateRoutes);
+router.use('/sdk', sdkRoutes);
 
 export default router;

package/api/src/routes/migrate.ts

@@ -13,65 +13,71 @@ const logger = createLogger('MigrateRoutes');
 router.post('/',
     validateBody(schemas.migrate),
     asyncHandler(async (req, res) => {
-        const { projectRef, dbPassword, accessToken } = req.body;
+        const { projectRef, accessToken } = req.body;
 
         logger.info('Starting migration', { projectRef });
 
-        res.setHeader('Content-Type', 'text/plain');
-        res.setHeader('Transfer-Encoding', 'chunked');
+        res.setHeader('Content-Type', 'text/event-stream');
+        res.setHeader('Cache-Control', 'no-cache');
+        res.setHeader('Connection', 'keep-alive');
 
-        const sendLog = (message: string) => {
-            res.write(message + '\n');
+        const sendEvent = (type: string, data: string) => {
+            res.write(`data: ${JSON.stringify({ type, data })}\n\n`);
         };
 
         try {
-            sendLog('🔧 Starting migration...');
-            
+            sendEvent('info', '🔧 Starting migration...');
+
             const scriptPath = join(config.scriptsDir, 'migrate.sh');
 
             const env = {
                 ...process.env,
                 SUPABASE_PROJECT_ID: projectRef,
-                SUPABASE_DB_PASSWORD: dbPassword || '',
                 SUPABASE_ACCESS_TOKEN: accessToken || '',
-                SKIP_FUNCTIONS: '1',
+                SKIP_FUNCTIONS: '0',
             };
 
-            const child = spawn('bash', [scriptPath], { 
-                env, 
+            const child = spawn('bash', [scriptPath], {
+                env,
                 cwd: config.rootDir,
             });
 
             child.stdout.on('data', (data) => {
-                sendLog(data.toString().trim());
+                const lines = data.toString().split('\n').filter((l: string) => l.trim());
+                for (const line of lines) {
+                    sendEvent('stdout', line);
+                }
             });
 
             child.stderr.on('data', (data) => {
-                sendLog(`⚠️  ${data.toString().trim()}`);
+                const lines = data.toString().split('\n').filter((l: string) => l.trim());
+                for (const line of lines) {
+                    sendEvent('stderr', line);
+                }
             });
 
             child.on('close', (code) => {
                 if (code === 0) {
-                    sendLog('✅ Migration successful!');
-                    sendLog('RESULT: success');
+                    sendEvent('info', '✅ Migration completed successfully!');
+                    sendEvent('done', 'success');
                     logger.info('Migration completed successfully', { projectRef });
                 } else {
-                    sendLog(`❌ Migration failed with code ${code}`);
-                    sendLog('RESULT: failure');
+                    sendEvent('error', `❌ Migration failed with code ${code}`);
+                    sendEvent('done', 'failed');
                     logger.error('Migration failed', new Error(`Exit code: ${code}`), { projectRef });
                 }
                 res.end();
             });
 
             child.on('error', (error) => {
-                sendLog(`❌ Failed to run migration: ${error.message}`);
-                sendLog('RESULT: failure');
+                sendEvent('error', `❌ Failed to run migration: ${error.message}`);
+                sendEvent('done', 'failed');
                 logger.error('Migration spawn error', error);
                 res.end();
             });
         } catch (error: any) {
-            sendLog(`❌ Server error: ${error.message}`);
-            sendLog('RESULT: failure');
+            sendEvent('error', `❌ Server error: ${error.message}`);
+            sendEvent('done', 'failed');
             logger.error('Migration error', error);
             res.end();
         }

package/api/src/routes/sdk.ts

@@ -0,0 +1,55 @@
+import { Router, Request, Response } from 'express';
+import { SDKService } from '../services/SDKService.js';
+import { ProvidersResponse } from '@realtimex/sdk';
+
+const router = Router();
+
+/**
+ * GET /api/sdk/providers/chat
+ * Returns available chat providers and their models
+ */
+router.get('/providers/chat', async (req: Request, res: Response) => {
+    try {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            return res.json({ success: false, message: 'SDK not available', providers: [] });
+        }
+
+        const { providers } = await SDKService.withTimeout<ProvidersResponse>(
+            sdk.llm.chatProviders(),
+            30000,
+            'Chat providers fetch timed out'
+        );
+
+        res.json({ success: true, providers: providers || [] });
+    } catch (error: any) {
+        console.warn('[SDK API] Failed to fetch chat providers:', error.message);
+        res.json({ success: false, providers: [], message: error.message });
+    }
+});
+
+/**
+ * GET /api/sdk/providers/embed
+ * Returns available embedding providers and their models
+ */
+router.get('/providers/embed', async (req: Request, res: Response) => {
+    try {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            return res.json({ success: false, message: 'SDK not available', providers: [] });
+        }
+
+        const { providers } = await SDKService.withTimeout<ProvidersResponse>(
+            sdk.llm.embedProviders(),
+            30000,
+            'Embed providers fetch timed out'
+        );
+
+        res.json({ success: true, providers: providers || [] });
+    } catch (error: any) {
+        console.warn('[SDK API] Failed to fetch embed providers:', error.message);
+        res.json({ success: false, providers: [], message: error.message });
+    }
+});
+
+export default router;

package/api/src/routes/settings.ts

@@ -4,6 +4,7 @@ import { authMiddleware } from '../middleware/auth.js';
 import { apiRateLimit } from '../middleware/rateLimit.js';
 import { validateBody, schemas } from '../middleware/validation.js';
 import { createLogger } from '../utils/logger.js';
+import { SDKService } from '../services/SDKService.js';
 
 const router = Router();
 const logger = createLogger('SettingsRoutes');
@@ -28,13 +29,21 @@ router.get('/',
         const settingsData = settingsResult.data;
         const integrationsData = integrationsResult.data || [];
 
-        // Return defaults if no settings exist
-        const settings = settingsData || {
-            llm_model: null,
-            llm_base_url: null,
+        // Return defaults if no settings exist, or upgrade old defaults
+        let settings = settingsData || {
+            llm_provider: SDKService.DEFAULT_LLM_PROVIDER,
+            llm_model: 'gpt-4o-mini',
             sync_interval_minutes: 5,
         };
 
+        // Proactive upgrade for existing settings objects
+        if (settings.llm_model === 'gpt-4.1-mini' || !settings.llm_model) {
+            settings.llm_model = 'gpt-4o-mini';
+        }
+        if (!settings.llm_provider) {
+            settings.llm_provider = SDKService.DEFAULT_LLM_PROVIDER;
+        }
+
         // Merge integration credentials back into settings for frontend compatibility
         const googleIntegration = integrationsData.find((i: any) => i.provider === 'google');
         if (googleIntegration?.credentials) {
@@ -67,7 +76,7 @@ router.patch('/',
             microsoft_tenant_id,
             ...userSettingsUpdates
         } = req.body;
-        
+
         const userId = req.user!.id;
 
         // 1. Update user_settings
@@ -91,7 +100,7 @@ router.patch('/',
                 .eq('user_id', userId)
                 .eq('provider', 'google')
                 .single();
-                
+
             const credentials: any = {};
             if (google_client_id) credentials.client_id = google_client_id;
             if (google_client_secret) credentials.client_secret = google_client_secret;
@@ -110,19 +119,19 @@ router.patch('/',
 
         // 3. Handle Microsoft Integration
         if (microsoft_client_id || microsoft_client_secret || microsoft_tenant_id) {
-             const { data: existing } = await req.supabase!
+            const { data: existing } = await req.supabase!
                 .from('integrations')
                 .select('credentials')
                 .eq('user_id', userId)
                 .eq('provider', 'microsoft')
                 .single();
 
-             const credentials: any = {};
-             if (microsoft_client_id) credentials.client_id = microsoft_client_id;
-             if (microsoft_client_secret) credentials.client_secret = microsoft_client_secret;
-             if (microsoft_tenant_id) credentials.tenant_id = microsoft_tenant_id;
+            const credentials: any = {};
+            if (microsoft_client_id) credentials.client_id = microsoft_client_id;
+            if (microsoft_client_secret) credentials.client_secret = microsoft_client_secret;
+            if (microsoft_tenant_id) credentials.tenant_id = microsoft_tenant_id;
 
-             const newCredentials = { ...(existing?.credentials || {}), ...credentials };
+            const newCredentials = { ...(existing?.credentials || {}), ...credentials };
 
             await req.supabase!
                 .from('integrations')
@@ -156,14 +165,8 @@ router.post('/test-llm',
     apiRateLimit,
     authMiddleware,
     asyncHandler(async (req, res) => {
-        const { llm_model, llm_base_url, llm_api_key } = req.body;
-
         const { getIntelligenceService } = await import('../services/intelligence.js');
-        const intelligence = getIntelligenceService({
-            model: llm_model,
-            baseUrl: llm_base_url,
-            apiKey: llm_api_key,
-        });
+        const intelligence = getIntelligenceService();
 
         const result = await intelligence.testConnection();
         res.json(result);