@realtimex/realtimex-alchemy 1.0.23 → 1.0.25

package/dist/api/lib/realtimex-sdk.js

@@ -0,0 +1,163 @@
+import axios from 'axios';
+/**
+ * RealTimeX SDK Client
+ * Provides access to RealTimeX Desktop APIs for LLM, embeddings, and vector storage
+ */
+export class RealTimeXSDK {
+    client;
+    appId;
+    baseUrl;
+    constructor(baseUrl = 'http://localhost:3001') {
+        this.baseUrl = baseUrl;
+        // RTX_APP_ID is injected by RealTimeX when starting the app
+        this.appId = process.env.RTX_APP_ID || '';
+        this.client = axios.create({
+            baseURL: this.baseUrl,
+            headers: {
+                'X-App-Id': this.appId,
+                'Content-Type': 'application/json'
+            },
+            timeout: 30000 // 30 second timeout
+        });
+    }
+    /**
+     * Generate embeddings for text inputs
+     * @param input - Single string or array of strings to embed
+     * @param provider - Provider name (realtimexai, openai, gemini)
+     * @param model - Model name (e.g., text-embedding-3-small)
+     * @returns Array of embedding vectors
+     */
+    async generateEmbedding(input, provider = 'realtimexai', model = 'text-embedding-3-small') {
+        try {
+            const response = await this.client.post('/sdk/llm/embed', {
+                input: Array.isArray(input) ? input : [input],
+                provider,
+                model
+            });
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Embedding generation failed');
+            }
+            return response.data.embeddings;
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] Embedding error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Store vectors in RealTimeX managed vector storage
+     * @param vectors - Array of vectors with metadata
+     * @param workspaceId - Workspace/namespace for isolation
+     */
+    async upsertVectors(vectors, workspaceId = 'default') {
+        try {
+            const response = await this.client.post('/sdk/llm/vectors/upsert', {
+                vectors,
+                workspaceId
+            });
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Vector upsert failed');
+            }
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] Vector upsert error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Query vectors for semantic similarity search
+     * @param queryVector - Query embedding vector
+     * @param topK - Number of results to return
+     * @param workspaceId - Workspace/namespace to search in
+     * @param filter - Optional metadata filter
+     * @returns Array of similar vectors with scores
+     */
+    async queryVectors(queryVector, topK = 10, workspaceId = 'default', filter) {
+        try {
+            const response = await this.client.post('/sdk/llm/vectors/query', {
+                vector: queryVector,
+                topK,
+                workspaceId,
+                filter
+            });
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Vector query failed');
+            }
+            return response.data.results || [];
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] Vector query error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Delete vectors from storage
+     * @param workspaceId - Workspace to delete from
+     * @param deleteAll - If true, deletes all vectors in workspace
+     */
+    async deleteVectors(workspaceId, deleteAll = false) {
+        try {
+            const response = await this.client.post('/sdk/llm/vectors/delete', {
+                workspaceId,
+                deleteAll
+            });
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Vector deletion failed');
+            }
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] Vector delete error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * List available LLM and embedding providers
+     * @returns Provider configuration
+     */
+    async getProviders() {
+        try {
+            const response = await this.client.get('/sdk/llm/providers');
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Failed to fetch providers');
+            }
+            return response.data;
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] Get providers error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * List all vector storage workspaces
+     * @returns Array of workspace IDs
+     */
+    async listVectorWorkspaces() {
+        try {
+            const response = await this.client.get('/sdk/llm/vectors/workspaces');
+            if (!response.data.success) {
+                throw new Error(response.data.error || 'Failed to list workspaces');
+            }
+            return response.data.workspaces || [];
+        }
+        catch (error) {
+            console.error('[RealTimeXSDK] List workspaces error:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Check if SDK is properly configured
+     * @returns True if app ID is set
+     */
+    isConfigured() {
+        return !!this.appId;
+    }
+    /**
+     * Get current app ID
+     * @returns App ID or empty string
+     */
+    getAppId() {
+        return this.appId;
+    }
+}
+// Export singleton instance
+export const realtimeXSDK = new RealTimeXSDK();

package/dist/api/services/AlchemistService.js

@@ -1,6 +1,8 @@
 import OpenAI from 'openai';
 import { ProcessingEventService } from './ProcessingEventService.js';
 import { RouterService } from './RouterService.js';
+import { embeddingService } from './EmbeddingService.js';
+import { deduplicationService } from './DeduplicationService.js';
 export class AlchemistService {
     processingEvents;
     router;
@@ -86,7 +88,9 @@ export class AlchemistService {
                     }, supabase);
                     // 4. Save Signal
                     console.log('[AlchemistService] Saving signal to database...');
-                    const { error: insertError } = await supabase.from('signals').insert([{
+                    const { data: insertedSignal, error: insertError } = await supabase
+                        .from('signals')
+                        .insert([{
                             user_id: userId,
                             url: entry.url,
                             title: entry.title,
@@ -96,7 +100,9 @@ export class AlchemistService {
                             entities: response.entities,
                             tags: response.tags,
                             content: content
-                        }]);
+                        }])
+                        .select()
+                        .single();
                     if (insertError) {
                         console.error('[AlchemistService] Insert error:', insertError);
                         stats.errors++;
@@ -104,6 +110,12 @@ export class AlchemistService {
                     else {
                         console.log('[AlchemistService] Signal saved successfully');
                         stats.signals++;
+                        // 5. Generate Embedding & Check for Duplicates (non-blocking)
+                        if (settings.embedding_model && await embeddingService.isAvailable()) {
+                            this.processEmbedding(insertedSignal, settings, userId, supabase).catch((err) => {
+                                console.error('[AlchemistService] Embedding processing failed:', err);
+                            });
+                        }
                     }
                 }
                 else {
@@ -238,4 +250,52 @@ export class AlchemistService {
             return { score: 0, summary: 'Failed to parse', category: 'Error', entities: [], tags: [], relevant: false };
         }
     }
+    /**
+     * Process embedding generation and deduplication for a signal
+     * This runs asynchronously and doesn't block the main mining pipeline
+     */
+    async processEmbedding(signal, settings, userId, supabase) {
+        try {
+            console.log('[AlchemistService] Generating embedding for signal:', signal.id);
+            // Generate embedding
+            const text = `${signal.title} ${signal.summary}`;
+            const embedding = await embeddingService.generateEmbedding(text, settings);
+            if (!embedding) {
+                console.warn('[AlchemistService] Embedding generation returned null, skipping');
+                return;
+            }
+            // Check for duplicates
+            const dedupeResult = await deduplicationService.checkAndMergeDuplicate(signal, embedding, userId, supabase);
+            if (dedupeResult.isDuplicate) {
+                console.log(`[AlchemistService] Signal is duplicate, merged into: ${dedupeResult.mergedSignalId}`);
+                // Delete the newly inserted signal since it's a duplicate
+                await supabase
+                    .from('signals')
+                    .delete()
+                    .eq('id', signal.id);
+                return;
+            }
+            // Store embedding in RealTimeX vector storage
+            await embeddingService.storeSignalEmbedding(signal.id, embedding, {
+                title: signal.title,
+                summary: signal.summary,
+                url: signal.url,
+                category: signal.category,
+                userId
+            });
+            // Update signal metadata
+            await supabase
+                .from('signals')
+                .update({
+                has_embedding: true,
+                embedding_model: settings.embedding_model
+            })
+                .eq('id', signal.id);
+            console.log('[AlchemistService] Embedding processed successfully for signal:', signal.id);
+        }
+        catch (error) {
+            console.error('[AlchemistService] Embedding processing error:', error.message);
+            // Don't throw - we don't want to fail the entire mining process
+        }
+    }
 }

package/dist/api/services/DeduplicationService.js

@@ -0,0 +1,146 @@
+import { embeddingService } from './EmbeddingService.js';
+/**
+ * Deduplication Service
+ * Handles smart signal merging based on semantic similarity
+ */
+export class DeduplicationService {
+    SIMILARITY_THRESHOLD = 0.85;
+    /**
+     * Check if signal is a duplicate and merge if necessary
+     * @param signal - New signal to check
+     * @param embedding - Signal embedding vector
+     * @param userId - User ID
+     * @param supabase - Supabase client
+     * @returns Deduplication result
+     */
+    async checkAndMergeDuplicate(signal, embedding, userId, supabase) {
+        try {
+            // Find similar signals
+            const similar = await embeddingService.findSimilarSignals(embedding, userId, this.SIMILARITY_THRESHOLD, 5 // Check top 5 matches
+            );
+            if (similar.length === 0) {
+                return { isDuplicate: false };
+            }
+            // Get the most similar signal
+            const bestMatch = similar[0];
+            console.log(`[Deduplication] Found similar signal: ${bestMatch.id} (score: ${bestMatch.score})`);
+            // Merge signals
+            const mergedId = await this.mergeSignals(bestMatch.id, signal, userId, supabase);
+            return {
+                isDuplicate: true,
+                mergedSignalId: mergedId,
+                similarityScore: bestMatch.score
+            };
+        }
+        catch (error) {
+            console.error('[Deduplication] Error:', error.message);
+            return { isDuplicate: false };
+        }
+    }
+    /**
+     * Merge new signal into existing signal
+     * @param existingSignalId - ID of existing signal
+     * @param newSignal - New signal to merge
+     * @param userId - User ID
+     * @param supabase - Supabase client
+     * @returns Merged signal ID
+     */
+    async mergeSignals(existingSignalId, newSignal, userId, supabase) {
+        // Fetch existing signal
+        const { data: existing, error } = await supabase
+            .from('signals')
+            .select('*')
+            .eq('id', existingSignalId)
+            .eq('user_id', userId)
+            .single();
+        if (error || !existing) {
+            console.error('[Deduplication] Failed to fetch existing signal:', error);
+            return existingSignalId;
+        }
+        // Calculate boosted score
+        const mentionCount = (existing.mention_count || 1) + 1;
+        const scoreBoost = Math.min(mentionCount * 0.1, 0.5); // Max 50% boost
+        const newScore = Math.min(existing.score + scoreBoost, 10); // Cap at 10
+        // Combine summaries (use longer one)
+        const combinedSummary = this.combineSummaries(existing.summary, newSignal.summary);
+        // Track source URLs in metadata
+        const existingUrls = existing.metadata?.source_urls || [existing.url];
+        const sourceUrls = [...new Set([...existingUrls, newSignal.url])]; // Deduplicate URLs
+        // Update existing signal
+        const { error: updateError } = await supabase
+            .from('signals')
+            .update({
+            score: newScore,
+            summary: combinedSummary,
+            mention_count: mentionCount,
+            metadata: {
+                ...existing.metadata,
+                source_urls: sourceUrls,
+                last_seen: new Date().toISOString(),
+                duplicate_count: mentionCount - 1
+            },
+            updated_at: new Date().toISOString()
+        })
+            .eq('id', existingSignalId);
+        if (updateError) {
+            console.error('[Deduplication] Failed to update signal:', updateError);
+        }
+        else {
+            console.log(`[Deduplication] Merged signal ${newSignal.id} into ${existingSignalId} (mentions: ${mentionCount}, score: ${newScore})`);
+        }
+        return existingSignalId;
+    }
+    /**
+     * Combine two summaries intelligently
+     * @param existing - Existing summary
+     * @param newSummary - New summary
+     * @returns Combined summary
+     */
+    combineSummaries(existing, newSummary) {
+        // Simple strategy: use the longer summary
+        // TODO: In future, use LLM to intelligently merge summaries
+        if (existing.length >= newSummary.length) {
+            return existing;
+        }
+        return newSummary;
+    }
+    /**
+     * Get deduplication statistics for a user
+     * @param userId - User ID
+     * @param supabase - Supabase client
+     * @returns Statistics object
+     */
+    async getStats(userId, supabase) {
+        try {
+            // Count total signals
+            const { count: totalSignals } = await supabase
+                .from('signals')
+                .select('*', { count: 'exact', head: true })
+                .eq('user_id', userId);
+            // Count merged signals (mention_count > 1)
+            const { count: mergedSignals } = await supabase
+                .from('signals')
+                .select('*', { count: 'exact', head: true })
+                .eq('user_id', userId)
+                .gt('mention_count', 1);
+            const deduplicationRate = totalSignals
+                ? (mergedSignals || 0) / totalSignals
+                : 0;
+            return {
+                totalSignals: totalSignals || 0,
+                mergedSignals: mergedSignals || 0,
+                deduplicationRate
+            };
+        }
+        catch (error) {
+            console.error('[Deduplication] Failed to get stats:', error.message);
+            return {
+                totalSignals: 0,
+                mergedSignals: 0,
+                deduplicationRate: 0
+            };
+        }
+    }
+}
+// Export singleton instance
+export const deduplicationService = new DeduplicationService();

package/dist/api/services/EmbeddingService.js

@@ -0,0 +1,169 @@
+/**
+ * Embedding Service using RealTimeX SDK
+ * Provides simplified interface for embedding generation and vector operations
+ * Gracefully degrades if SDK is not available
+ */
+export class EmbeddingService {
+    WORKSPACE_ID = 'alchemy-signals';
+    SIMILARITY_THRESHOLD = 0.85;
+    sdk = null;
+    sdkLoadAttempted = false;
+    /**
+     * Lazy load SDK - only loads when first needed
+     */
+    async loadSDK() {
+        if (this.sdkLoadAttempted) {
+            return this.sdk;
+        }
+        this.sdkLoadAttempted = true;
+        try {
+            const sdkModule = await import('../lib/realtimex-sdk.js');
+            this.sdk = sdkModule.realtimeXSDK;
+            console.log('[EmbeddingService] RealTimeX SDK loaded successfully');
+        }
+        catch (error) {
+            console.warn('[EmbeddingService] RealTimeX SDK not available - embedding features disabled');
+            this.sdk = null;
+        }
+        return this.sdk;
+    }
+    /**
+     * Generate embedding for a single text
+     * @param text - Text to embed
+     * @param settings - Alchemy settings with embedding configuration
+     * @returns Embedding vector or null if failed
+     */
+    async generateEmbedding(text, settings) {
+        try {
+            const sdk = await this.loadSDK();
+            if (!sdk || !sdk.isConfigured()) {
+                console.warn('[EmbeddingService] RealTimeX SDK not available');
+                return null;
+            }
+            const provider = this.getProvider(settings);
+            const model = settings.embedding_model || 'text-embedding-3-small';
+            const embeddings = await sdk.generateEmbedding(text, provider, model);
+            return embeddings[0] || null;
+        }
+        catch (error) {
+            console.error('[EmbeddingService] Generation failed:', error.message);
+            return null;
+        }
+    }
+    /**
+     * Generate embeddings for multiple texts (batch)
+     * @param texts - Array of texts to embed
+     * @param settings - Alchemy settings
+     * @returns Array of embedding vectors or null if failed
+     */
+    async generateEmbeddings(texts, settings) {
+        try {
+            const sdk = await this.loadSDK();
+            if (!sdk || !sdk.isConfigured()) {
+                console.warn('[EmbeddingService] RealTimeX SDK not configured');
+                return null;
+            }
+            const provider = this.getProvider(settings);
+            const model = settings.embedding_model || 'text-embedding-3-small';
+            return await sdk.generateEmbedding(texts, provider, model);
+        }
+        catch (error) {
+            console.error('[EmbeddingService] Batch generation failed:', error.message);
+            return null;
+        }
+    }
+    /**
+     * Store signal embedding in RealTimeX vector storage
+     * @param signalId - Unique signal ID
+     * @param embedding - Embedding vector
+     * @param metadata - Signal metadata
+     */
+    async storeSignalEmbedding(signalId, embedding, metadata) {
+        try {
+            const sdk = await this.loadSDK();
+            if (!sdk) {
+                throw new Error('SDK not available');
+            }
+            await sdk.upsertVectors([{
+                    id: signalId,
+                    vector: embedding,
+                    metadata
+                }], this.WORKSPACE_ID);
+            console.log('[EmbeddingService] Stored embedding for signal:', signalId);
+        }
+        catch (error) {
+            console.error('[EmbeddingService] Storage failed:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Find similar signals using semantic search
+     * @param queryEmbedding - Query embedding vector
+     * @param userId - User ID for filtering
+     * @param threshold - Similarity threshold (0-1)
+     * @param limit - Max results
+     * @returns Array of similar signals
+     */
+    async findSimilarSignals(queryEmbedding, userId, threshold = this.SIMILARITY_THRESHOLD, limit = 10) {
+        try {
+            const sdk = await this.loadSDK();
+            if (!sdk) {
+                return [];
+            }
+            const results = await sdk.queryVectors(queryEmbedding, limit, this.WORKSPACE_ID, { userId } // Filter by user
+            );
+            // Filter by similarity threshold
+            return results.filter((r) => r.score >= threshold);
+        }
+        catch (error) {
+            console.error('[EmbeddingService] Similarity search failed:', error.message);
+            return [];
+        }
+    }
+    /**
+     * Delete all embeddings for a user
+     * @param userId - User ID
+     */
+    async deleteUserEmbeddings(userId) {
+        try {
+            // Note: Current SDK only supports deleteAll
+            // In future, we may need user-specific workspaces
+            console.warn('[EmbeddingService] User-specific deletion not yet supported');
+        }
+        catch (error) {
+            console.error('[EmbeddingService] Deletion failed:', error.message);
+            throw error;
+        }
+    }
+    /**
+     * Determine provider from settings
+     * @param settings - Alchemy settings
+     * @returns Provider name
+     */
+    getProvider(settings) {
+        // If embedding_base_url is not set, use realtimexai (default)
+        if (!settings.embedding_base_url) {
+            return 'realtimexai';
+        }
+        // Detect provider from base URL
+        const url = settings.embedding_base_url.toLowerCase();
+        if (url.includes('openai')) {
+            return 'openai';
+        }
+        else if (url.includes('google') || url.includes('gemini')) {
+            return 'gemini';
+        }
+        // Default to realtimexai
+        return 'realtimexai';
+    }
+    /**
+     * Check if embedding service is available
+     * @returns True if SDK is configured and available
+     */
+    async isAvailable() {
+        const sdk = await this.loadSDK();
+        return sdk !== null && sdk.isConfigured();
+    }
+}
+// Export singleton instance
+export const embeddingService = new EmbeddingService();