@realtimex/realtimex-alchemy 1.0.50 → 1.0.52

package/dist/CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.52] - 2026-01-27
+
+### Changed
+- **Configuration**: Refactored LLM provider configuration to drop legacy fields and utilize new SDK defaults, simplifying setup.
+
+### Fixed
+- **Embeddings**: Updated `AlchemistService` to allow embedding generation without an explicit `embedding_model` setting, enabling dynamic model resolution via the SDK.
+
+## [1.0.51] - 2026-01-27
+
+### Added
+- **Vector Storage**: Migrated embedding storage from local SDK to Supabase `pgvector`. This enables cloud-native similarity search and persistent memory across devices.
+- **Database**: Added `alchemy_vectors` table and HNSW indexes for high-performance semantic retrieval.
+
+### Improved
+- **Performance**: Implemented server-side similarity search via `match_vectors` RPC function, reducing latency for RAG and deduplication.
+
 ## [1.0.50] - 2026-01-26
 
 ### Improved

package/dist/api/index.js

@@ -276,11 +276,8 @@ app.post('/api/browser-paths/validate', async (req, res) => {
 app.post('/api/test/analyze', async (req, res) => {
     const { text } = req.body;
     try {
-        // Fetch settings from database
-        let settings = {
-            llm_provider: 'realtimexai',
-            llm_model: 'gpt-4o'
-        };
+        // Fetch settings from database (empty object = use dynamic defaults from SDK)
+        let settings = {};
         if (SupabaseService.isConfigured()) {
             const supabase = SupabaseService.getServiceRoleClient();
             const { data: userData } = await supabase.rpc('get_any_user_id');
@@ -306,9 +303,10 @@ app.post('/api/test/analyze', async (req, res) => {
 app.post('/api/llm/test', async (req, res) => {
     const { llmProvider, llmModel } = req.body;
     try {
+        // Pass provided values (if any) - if not provided, resolveChatProvider will get defaults from SDK
         const settings = {
-            llm_provider: llmProvider || 'realtimexai',
-            llm_model: llmModel || 'gpt-4o'
+            llm_provider: llmProvider,
+            llm_model: llmModel
         };
         const result = await alchemist.testConnection(settings);
         res.json(result);

package/dist/api/services/AlchemistService.js

@@ -89,10 +89,9 @@ export class AlchemistService {
             skipped: 0,
             errors: 0
         };
-        console.log('[AlchemistService] LLM Config:', {
-            provider: settings.llm_provider || 'realtimexai',
-            model: settings.llm_model || 'gpt-4o'
-        });
+        // Resolve LLM provider dynamically
+        const llmConfig = await SDKService.resolveChatProvider(settings);
+        console.log('[AlchemistService] LLM Config:', llmConfig);
         for (const entry of allowedEntries) {
             // Emit: Reading
             await this.processingEvents.log({
@@ -189,7 +188,8 @@ export class AlchemistService {
                         }, supabase);
                         stats.signals++;
                         // 5. Generate Embedding & Check for Duplicates (non-blocking)
-                        if (settings.embedding_model && await embeddingService.isAvailable()) {
+                        // Note: embedding_model is resolved dynamically via SDK if not in settings
+                        if (await embeddingService.isAvailable()) {
                             await this.processEmbedding(insertedSignal, settings, userId, supabase).catch((err) => {
                                 console.error('[AlchemistService] Embedding processing failed:', err);
                             });
@@ -261,6 +261,8 @@ export class AlchemistService {
         if (!sdk) {
             throw new Error('RealTimeX SDK not available');
         }
+        // Resolve LLM provider dynamically from SDK
+        const { provider, model } = await SDKService.resolveChatProvider(settings);
         const prompt = `
         Act as "The Alchemist", a high-level intelligence analyst.
         Analyze the following article value based on the content and the User's Interests.
@@ -309,8 +311,8 @@ export class AlchemistService {
             { role: 'system', content: 'You are a precise analyzer. Return ONLY valid JSON, no other text.' },
             { role: 'user', content: prompt }
         ], {
-            provider: settings.llm_provider || 'realtimexai',
-            model: settings.llm_model || 'gpt-4o'
+            provider,
+            model
         });
         // SDK returns response.response?.content based on documentation
         const raw = response.response?.content || '{}';
@@ -329,16 +331,18 @@ export class AlchemistService {
                     message: 'RealTimeX SDK not available. Please run via RealTimeX Desktop.'
                 };
             }
+            // Resolve LLM provider dynamically from SDK
+            const { provider, model } = await SDKService.resolveChatProvider(settings);
             const response = await sdk.llm.chat([
                 { role: 'user', content: 'Say "OK"' }
             ], {
-                provider: settings.llm_provider || 'realtimexai',
-                model: settings.llm_model || 'gpt-4o'
+                provider,
+                model
             });
             return {
                 success: true,
                 message: `Connection successful!`,
-                model: settings.llm_model || 'gpt-4o'
+                model
             };
         }
         catch (error) {
@@ -392,6 +396,8 @@ export class AlchemistService {
     async processEmbedding(signal, settings, userId, supabase) {
         try {
             console.log('[AlchemistService] Generating embedding for signal:', signal.id);
+            // Resolve embedding provider dynamically
+            const { model: embeddingModel } = await SDKService.resolveEmbedProvider(settings);
             // Generate embedding
             const text = `${signal.title} ${signal.summary}`;
             const embedding = await embeddingService.generateEmbedding(text, settings);
@@ -410,20 +416,21 @@ export class AlchemistService {
                     .eq('id', signal.id);
                 return;
             }
-            // Store embedding in RealTimeX vector storage
+            // Store embedding in Supabase pgvector storage
             await embeddingService.storeSignalEmbedding(signal.id, embedding, {
                 title: signal.title,
                 summary: signal.summary,
                 url: signal.url,
                 category: signal.category,
-                userId
-            });
+                userId,
+                model: embeddingModel
+            }, supabase);
             // Update signal metadata
             await supabase
                 .from('signals')
                 .update({
                 has_embedding: true,
-                embedding_model: settings.embedding_model
+                embedding_model: embeddingModel
             })
                 .eq('id', signal.id);
             console.log('[AlchemistService] Embedding processed successfully for signal:', signal.id);

package/dist/api/services/ChatService.js

@@ -64,7 +64,7 @@ export class ChatService {
             let sources = [];
             // 3. Retrieve Context (if embedding checks out)
             if (queryEmbedding) {
-                const similar = await embeddingService.findSimilarSignals(queryEmbedding, userId, 0.55, // Lowered threshold for better recall
+                const similar = await embeddingService.findSimilarSignals(queryEmbedding, userId, supabase, 0.55, // Lowered threshold for better recall
                 10 // Increased Top K
                 );
                 console.log(`[ChatService] RAG Retrieval: Found ${similar.length} signals for query: "${content}"`);
@@ -106,9 +106,11 @@ Be concise, helpful, and professional.
                 { role: 'user', content: finalPrompt } // Current turn with RAG context
             ];
             console.log('[ChatService] Final Prompt being sent to LLM:', JSON.stringify(messages, null, 2));
+            // Resolve LLM provider dynamically from SDK
+            const { provider, model } = await SDKService.resolveChatProvider(settings);
             const response = await sdk.llm.chat(messages, {
-                provider: settings.llm_provider || 'realtimexai',
-                model: settings.llm_model || 'gpt-4o'
+                provider,
+                model
             });
             console.log('[ChatService] LLM Response:', JSON.stringify(response, null, 2));
             const aiContent = response.response?.content || "I'm sorry, I couldn't generate a response. The LLM returned empty content.";
@@ -161,12 +163,14 @@ Be concise, helpful, and professional.
                 const sdk = SDKService.getSDK();
                 if (!sdk)
                     return;
+                // Resolve LLM provider dynamically from SDK
+                const { provider, model } = await SDKService.resolveChatProvider(settings);
                 const response = await sdk.llm.chat([
                     { role: 'system', content: 'Generate a very short title (3-5 words) for this chat conversation. Return ONLY the title.' },
                     { role: 'user', content: `User: ${userMsg}\nAI: ${aiMsg}` }
                 ], {
-                    provider: settings.llm_provider || 'realtimexai',
-                    model: 'gpt-4o-mini'
+                    provider,
+                    model
                 });
                 const newTitle = response.response?.content?.replace(/['"]/g, '').trim();
                 if (newTitle) {

package/dist/api/services/DeduplicationService.js

@@ -18,7 +18,7 @@ export class DeduplicationService {
         try {
             // 1. Semantic Check (if embedding exists)
             if (embedding && embedding.length > 0) {
-                const similar = await embeddingService.findSimilarSignals(embedding, userId, this.SIMILARITY_THRESHOLD, 5 // Check top 5 matches
+                const similar = await embeddingService.findSimilarSignals(embedding, userId, supabase, this.SIMILARITY_THRESHOLD, 5 // Check top 5 matches
                 );
                 if (similar.length > 0) {
                     const bestMatch = similar[0];
@@ -154,6 +154,8 @@ export class DeduplicationService {
                 // Fallback: use longer summary
                 return existing.length >= newSummary.length ? existing : newSummary;
             }
+            // Resolve LLM provider dynamically from SDK
+            const { provider, model } = await SDKService.resolveChatProvider(settings);
             // Use LLM to intelligently merge summaries
             const response = await sdk.llm.chat([
                 {
@@ -165,8 +167,8 @@ export class DeduplicationService {
                     content: `Summary 1: ${existing}\nSummary 2: ${newSummary}\n\nMerged summary:`
                 }
             ], {
-                provider: settings.llm_provider || 'realtimexai',
-                model: settings.llm_model || 'gpt-4o-mini'
+                provider,
+                model
             });
             const merged = response.response?.content?.trim();
             return merged || existing;

package/dist/api/services/EmbeddingService.js

@@ -1,11 +1,11 @@
 import { SDKService } from './SDKService.js';
 /**
- * Embedding Service using RealTimeX SDK
- * Provides simplified interface for embedding generation and vector operations
+ * Embedding Service
+ * Uses RealTimeX SDK for embedding generation
+ * Uses Supabase pgvector for vector storage and similarity search
  * Gracefully degrades if SDK is not available
  */
 export class EmbeddingService {
-    WORKSPACE_ID = 'alchemy-signals';
     SIMILARITY_THRESHOLD = 0.85;
     /**
      * Generate embedding for a single text
@@ -20,8 +20,8 @@ export class EmbeddingService {
                 console.warn('[EmbeddingService] RealTimeX SDK not available');
                 return null;
             }
-            const provider = this.getProvider(settings);
-            const model = settings.embedding_model || 'text-embedding-3-small';
+            // Resolve embedding provider dynamically from SDK
+            const { provider, model } = await SDKService.resolveEmbedProvider(settings);
             const response = await sdk.llm.embed(text, {
                 provider,
                 model
@@ -46,8 +46,8 @@ export class EmbeddingService {
                 console.warn('[EmbeddingService] RealTimeX SDK not available');
                 return null;
             }
-            const provider = this.getProvider(settings);
-            const model = settings.embedding_model || 'text-embedding-3-small';
+            // Resolve embedding provider dynamically from SDK
+            const { provider, model } = await SDKService.resolveEmbedProvider(settings);
             const response = await sdk.llm.embed(texts, {
                 provider,
                 model
@@ -60,22 +60,40 @@ export class EmbeddingService {
         }
     }
     /**
-     * Store signal embedding in RealTimeX vector storage
+     * Store signal embedding in Supabase pgvector storage
      * @param signalId - Unique signal ID
      * @param embedding - Embedding vector
      * @param metadata - Signal metadata
+     * @param supabase - Supabase client
      */
-    async storeSignalEmbedding(signalId, embedding, metadata) {
+    async storeSignalEmbedding(signalId, embedding, metadata, supabase) {
         try {
-            const sdk = SDKService.getSDK();
-            if (!sdk) {
-                throw new Error('SDK not available');
+            // Format embedding as pgvector string
+            const embeddingStr = `[${embedding.join(',')}]`;
+            // Use model from metadata if provided, otherwise try to get default
+            let modelName = metadata.model || 'unknown';
+            if (!metadata.model) {
+                try {
+                    const { model } = await SDKService.resolveEmbedProvider({});
+                    modelName = model;
+                }
+                catch (e) {
+                    // Keep 'unknown' if we can't resolve
+                }
+            }
+            const { error } = await supabase
+                .from('alchemy_vectors')
+                .upsert({
+                signal_id: signalId,
+                user_id: metadata.userId,
+                embedding: embeddingStr,
+                model: modelName
+            }, {
+                onConflict: 'signal_id,model'
+            });
+            if (error) {
+                throw error;
             }
-            await sdk.llm.vectors.upsert([{
-                    id: signalId,
-                    vector: embedding,
-                    metadata
-                }], { workspaceId: this.WORKSPACE_ID });
             console.log('[EmbeddingService] Stored embedding for signal:', signalId);
         }
         catch (error) {
@@ -84,32 +102,39 @@ export class EmbeddingService {
         }
     }
     /**
-     * Find similar signals using semantic search
+     * Find similar signals using semantic search via Supabase pgvector
      * @param queryEmbedding - Query embedding vector
      * @param userId - User ID for filtering
+     * @param supabase - Supabase client
      * @param threshold - Similarity threshold (0-1)
      * @param limit - Max results
      * @returns Array of similar signals
      */
-    async findSimilarSignals(queryEmbedding, userId, threshold = this.SIMILARITY_THRESHOLD, limit = 10) {
+    async findSimilarSignals(queryEmbedding, userId, supabase, threshold = this.SIMILARITY_THRESHOLD, limit = 10) {
         try {
-            const sdk = SDKService.getSDK();
-            if (!sdk) {
+            // Format embedding as pgvector string
+            const embeddingStr = `[${queryEmbedding.join(',')}]`;
+            const { data, error } = await supabase.rpc('match_vectors', {
+                query_embedding: embeddingStr,
+                match_threshold: threshold,
+                match_count: limit,
+                target_user_id: userId
+            });
+            if (error) {
+                console.error('[EmbeddingService] Similarity search RPC error:', error.message);
                 return [];
             }
-            const response = await sdk.llm.vectors.query(queryEmbedding, {
-                topK: limit,
-                workspaceId: this.WORKSPACE_ID
-            });
-            // Filter by similarity threshold and user
-            const results = response.results || [];
-            return results
-                .filter((r) => r.metadata?.userId === userId)
-                .filter((r) => r.score >= threshold)
-                .map((r) => ({
-                id: r.id,
-                score: r.score,
-                metadata: r.metadata || {}
+            // Map results to expected format
+            return (data || []).map((r) => ({
+                id: r.signal_id,
+                score: r.similarity,
+                metadata: {
+                    title: r.title,
+                    summary: r.summary,
+                    url: r.url,
+                    category: r.category,
+                    userId
+                }
             }));
         }
         catch (error) {
@@ -120,39 +145,24 @@ export class EmbeddingService {
     /**
      * Delete all embeddings for a user
      * @param userId - User ID
+     * @param supabase - Supabase client
      */
-    async deleteUserEmbeddings(userId) {
+    async deleteUserEmbeddings(userId, supabase) {
         try {
-            // Note: Current SDK only supports deleteAll
-            // In future, we may need user-specific workspaces
-            console.warn('[EmbeddingService] User-specific deletion not yet supported');
+            const { error } = await supabase
+                .from('alchemy_vectors')
+                .delete()
+                .eq('user_id', userId);
+            if (error) {
+                throw error;
+            }
+            console.log('[EmbeddingService] Deleted all embeddings for user:', userId);
         }
         catch (error) {
             console.error('[EmbeddingService] Deletion failed:', error.message);
             throw error;
         }
     }
-    /**
-     * Determine provider from settings
-     * @param settings - Alchemy settings
-     * @returns Provider name
-     */
-    getProvider(settings) {
-        // Use embedding_provider if set
-        if (settings.embedding_provider) {
-            return settings.embedding_provider;
-        }
-        // Fallback: detect from base URL (legacy)
-        if (settings.embedding_base_url) {
-            const url = settings.embedding_base_url.toLowerCase();
-            if (url.includes('openai'))
-                return 'openai';
-            if (url.includes('google') || url.includes('gemini'))
-                return 'gemini';
-        }
-        // Default to realtimexai
-        return 'realtimexai';
-    }
     /**
      * Check if embedding service is available
      * @returns True if SDK is configured and available

package/dist/api/services/SDKService.js

@@ -107,6 +107,130 @@ export class SDKService {
         this.instance = null;
         this.initAttempted = false;
     }
+    // Cache for default providers (avoid repeated SDK calls)
+    static defaultChatProvider = null;
+    static defaultEmbedProvider = null;
+    /**
+     * Get default chat provider/model from SDK dynamically
+     * Caches result to avoid repeated SDK calls
+     */
+    static async getDefaultChatProvider() {
+        // Return cached if available
+        if (this.defaultChatProvider) {
+            return this.defaultChatProvider;
+        }
+        const sdk = this.getSDK();
+        if (!sdk) {
+            throw new Error('RealTimeX SDK not available. Cannot determine default LLM provider.');
+        }
+        try {
+            const { providers } = await this.withTimeout(sdk.llm.chatProviders(), 30000, 'Chat providers fetch timed out');
+            if (!providers || providers.length === 0) {
+                throw new Error('No LLM providers available. Please configure a provider in RealTimeX Desktop.');
+            }
+            // Find first provider with available models
+            for (const p of providers) {
+                if (p.models && p.models.length > 0) {
+                    this.defaultChatProvider = {
+                        provider: p.provider,
+                        model: p.models[0].id
+                    };
+                    console.log(`[SDKService] Default chat provider: ${this.defaultChatProvider.provider}/${this.defaultChatProvider.model}`);
+                    return this.defaultChatProvider;
+                }
+            }
+            throw new Error('No LLM models available. Please configure a model in RealTimeX Desktop.');
+        }
+        catch (error) {
+            console.error('[SDKService] Failed to get default chat provider:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Get default embedding provider/model from SDK dynamically
+     * Caches result to avoid repeated SDK calls
+     */
+    static async getDefaultEmbedProvider() {
+        // Return cached if available
+        if (this.defaultEmbedProvider) {
+            return this.defaultEmbedProvider;
+        }
+        const sdk = this.getSDK();
+        if (!sdk) {
+            throw new Error('RealTimeX SDK not available. Cannot determine default embedding provider.');
+        }
+        try {
+            const { providers } = await this.withTimeout(sdk.llm.embedProviders(), 30000, 'Embed providers fetch timed out');
+            if (!providers || providers.length === 0) {
+                throw new Error('No embedding providers available. Please configure a provider in RealTimeX Desktop.');
+            }
+            // Find first provider with available models
+            for (const p of providers) {
+                if (p.models && p.models.length > 0) {
+                    this.defaultEmbedProvider = {
+                        provider: p.provider,
+                        model: p.models[0].id
+                    };
+                    console.log(`[SDKService] Default embed provider: ${this.defaultEmbedProvider.provider}/${this.defaultEmbedProvider.model}`);
+                    return this.defaultEmbedProvider;
+                }
+            }
+            throw new Error('No embedding models available. Please configure a model in RealTimeX Desktop.');
+        }
+        catch (error) {
+            console.error('[SDKService] Failed to get default embed provider:', error.message);
+            throw error;
+        }
+    }
+    // Default provider/model configuration
+    // realtimexai routes through RealTimeX Desktop to user's configured providers
+    static DEFAULT_LLM_PROVIDER = 'realtimexai';
+    static DEFAULT_LLM_MODEL = 'gpt-4.1-mini';
+    static DEFAULT_EMBED_PROVIDER = 'realtimexai';
+    static DEFAULT_EMBED_MODEL = 'text-embedding-3-small';
+    /**
+     * Resolve LLM provider/model - use settings if available, otherwise use defaults
+     */
+    static async resolveChatProvider(settings) {
+        // If both provider and model are set in settings, use them
+        if (settings.llm_provider && settings.llm_model) {
+            return { provider: settings.llm_provider, model: settings.llm_model };
+        }
+        // Try to get from SDK discovery first
+        try {
+            return await this.getDefaultChatProvider();
+        }
+        catch (error) {
+            // Fallback to hardcoded defaults if SDK discovery fails
+            console.log(`[SDKService] Using default LLM: ${this.DEFAULT_LLM_PROVIDER}/${this.DEFAULT_LLM_MODEL}`);
+            return { provider: this.DEFAULT_LLM_PROVIDER, model: this.DEFAULT_LLM_MODEL };
+        }
+    }
+    /**
+     * Resolve embedding provider/model - use settings if available, otherwise use defaults
+     */
+    static async resolveEmbedProvider(settings) {
+        // If both provider and model are set in settings, use them
+        if (settings.embedding_provider && settings.embedding_model) {
+            return { provider: settings.embedding_provider, model: settings.embedding_model };
+        }
+        // Try to get from SDK discovery first
+        try {
+            return await this.getDefaultEmbedProvider();
+        }
+        catch (error) {
+            // Fallback to hardcoded defaults if SDK discovery fails
+            console.log(`[SDKService] Using default embedding: ${this.DEFAULT_EMBED_PROVIDER}/${this.DEFAULT_EMBED_MODEL}`);
+            return { provider: this.DEFAULT_EMBED_PROVIDER, model: this.DEFAULT_EMBED_MODEL };
+        }
+    }
+    /**
+     * Clear provider cache (useful when providers change)
+     */
+    static clearProviderCache() {
+        this.defaultChatProvider = null;
+        this.defaultEmbedProvider = null;
+    }
     /**
      * Helper to wrap a promise with a timeout
      */
@@ -147,7 +271,6 @@ export class SDKService {
             throw new Error('RealTimeX SDK not linked. Cannot get app data dir.');
         }
         try {
-            // @ts-ignore - SDK method for getting app data directory
             return await sdk.getAppDataDir();
         }
         catch (error) {

package/dist/api/services/TransmuteService.js

@@ -225,9 +225,11 @@ export class TransmuteService {
         if (!sdk) {
             throw new Error('RealTimeX SDK not available. Please ensure the desktop app is running.');
         }
-        // Use engine config for model/provider if available, else defaults
-        const provider = engine.config.llm_provider || 'realtimexai';
-        const model = engine.config.llm_model || 'gpt-4o-mini';
+        // Resolve LLM provider dynamically from SDK using engine config
+        const { provider, model } = await SDKService.resolveChatProvider({
+            llm_provider: engine.config.llm_provider,
+            llm_model: engine.config.llm_model
+        });
         const response = await sdk.llm.chat([
             { role: 'system', content: systemPrompt },
             { role: 'user', content: userPrompt }
@@ -273,12 +275,12 @@ export class TransmuteService {
                 continue;
             }
             console.log(`[Transmute] Bootstrapping missing category engine: ${title}`);
+            // Note: llm_provider and llm_model are intentionally omitted
+            // They will be resolved dynamically at runtime via SDKService.resolveChatProvider()
             const config = {
                 category,
                 execution_mode: 'desktop',
                 schedule: 'Daily',
-                llm_provider: 'realtimexai',
-                llm_model: 'gpt-4o',
                 max_signals: 30,
                 custom_prompt: `Create a comprehensive daily newsletter focused on ${category}. Highlight the most important developments, key insights, and actionable takeaways. Use a professional, insight-driven tone with clear structure: start with 'The Big Story' followed by 'Quick Hits' for other notable items.`
             };
@@ -354,12 +356,12 @@ export class TransmuteService {
                 continue;
             }
             console.log(`[Transmute] Creating dynamic tag engine for "${tag}" (${count} signals)`);
+            // Note: llm_provider and llm_model are intentionally omitted
+            // They will be resolved dynamically at runtime via SDKService.resolveChatProvider()
             const config = {
                 tag,
                 execution_mode: 'desktop',
-                schedule: 'Daily',
-                llm_provider: 'realtimexai',
-                llm_model: 'gpt-4o'
+                schedule: 'Daily'
             };
             const { error } = await supabase
                 .from('engines')