@realtimex/realtimex-alchemy 1.0.39 → 1.0.41

package/dist/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.41] - 2026-01-24
+
+### Added
+- **Active Learning**: Implemented "Boost-to-Embed" workflow. Boosting a signal in the UI now triggers the backend to generate a vector embedding for that signal, actively improving the AI's future context retrieval.
+- **Deduplication**: Enhanced deduplication engine with Title-based heuristics to catch duplicates across different URLs (e.g., tracking links) and added explicit exact-URL matching.
+
+### Improved
+- **Data Retention**: The Alchemist engine now persists *all* mined signals, including low-scoring ones (marked as auto-dismissed). This allows users to review and rescue missed opportunities via the new System Logs filters.
+- **RAG**: Tuned Chat retrieval parameters (Lowered threshold to 0.55, Increased limit to 10) to significantly improve the recall of relevant historical context during conversations.
+- **UI**: Connected the System Logs "Boost" action to the new API endpoint, ensuring immediate feedback loops for the AI.
+
+## [1.0.40] - 2026-01-24
+
+### Improved
+- **UI**: Standardized the styling of disabled input fields in Account Settings (Email field) to match the new `bg-black/5` convention, ensuring consistent visual language and readability across the app.
+
 ## [1.0.39] - 2026-01-24
 
 ### Fixed

package/dist/api/index.js

@@ -357,6 +357,62 @@ app.post('/api/chat/message', async (req, res) => {
         res.status(500).json({ success: false, message: e.message });
     }
 });
+// Active Learning: Boost Signal & Trigger Embedding
+app.post('/api/signals/:id/boost', async (req, res) => {
+    try {
+        const userId = req.headers['x-user-id'];
+        const signalId = req.params.id;
+        const { value, settings: bodySettings } = req.body; // value = true/false
+        console.log(`[API] Boost Request for ${signalId}: Value=${value}, User=${userId ? 'Present' : 'Missing'}`);
+        if (!userId) {
+            console.warn('[API] Boost failed: Missing x-user-id header');
+            return res.status(401).json({ error: 'Unauthorized: Missing User ID' });
+        }
+        const supabase = getAuthenticatedSupabase(req);
+        // 1. Update Signal in DB
+        const { data: signal, error } = await supabase
+            .from('signals')
+            .update({
+            is_boosted: value,
+            // If boosting, also un-dismiss it if it was dismissed
+            ...(value ? { is_dismissed: false } : {})
+        })
+            .eq('id', signalId)
+            .select()
+            .single();
+        if (error)
+            throw error;
+        // 2. If Boosting, Check/Generate Embedding
+        if (value === true && signal) {
+            // Fetch settings if not provided
+            let settings = bodySettings;
+            if (!settings || Object.keys(settings).length === 0) {
+                const { data: dbSettings } = await supabase
+                    .from('alchemy_settings')
+                    .select('*')
+                    .eq('user_id', userId)
+                    .single();
+                settings = dbSettings || {};
+            }
+            // Check if embedding works
+            if (await SDKService.isAvailable()) {
+                console.log(`[API] Triggering Active Learning embedding for boosted signal: ${signal.title}`);
+                // Public method to process embedding (exposed via casting or refactor)
+                // Since processEmbedding is private, we can use a small hack or public wrapper.
+                // For now, let's expose specific functionality or reuse the logic.
+                // BETTER: Add a public method to AlchemistService 'promoteToRelevant(signal, settings)'
+                // Calling the private method via 'any' cast for expediency, 
+                // but cleaner architecture would be AlchemistService.promoteSignal()
+                await alchemist.processEmbedding(signal, settings, userId, supabase);
+            }
+        }
+        res.json({ success: true, signal });
+    }
+    catch (e) {
+        console.error('[API] Boost failed:', e);
+        res.status(500).json({ success: false, message: e.message });
+    }
+});
 // Unified Static Assets Serving
 const staticPath = process.env.ELECTRON_STATIC_PATH || path.join(__dirname, '..', '..', 'dist');
 if (fs.existsSync(staticPath)) {

package/dist/api/services/AlchemistService.js

@@ -130,40 +130,43 @@ export class AlchemistService {
                 // 3. LLM Analysis
                 const response = await this.analyzeContent(content, entry.url, settings, learningContext);
                 const duration = Date.now() - startAnalysis;
-                if (response.relevant) {
-                    // Emit: Signal Found
-                    await this.processingEvents.log({
-                        eventType: 'action',
-                        agentState: 'Signal',
-                        message: `Found signal: ${response.summary} (${response.score}%)`,
-                        level: 'info',
-                        metadata: response,
-                        durationMs: duration,
-                        userId
-                    }, supabase);
-                    // 4. Save Signal
-                    console.log('[AlchemistService] Saving signal to database...');
-                    const { data: insertedSignal, error: insertError } = await supabase
-                        .from('signals')
-                        .insert([{
-                            user_id: userId,
-                            url: entry.url,
-                            title: entry.title,
-                            score: response.score,
-                            summary: response.summary,
-                            category: response.category,
-                            entities: response.entities,
-                            tags: response.tags,
-                            content: content
-                        }])
-                        .select()
-                        .single();
-                    if (insertError) {
-                        console.error('[AlchemistService] Insert error:', insertError);
-                        stats.errors++;
-                    }
-                    else {
-                        console.log('[AlchemistService] Signal saved successfully');
+                // 4. Save Signal (ALWAYS save for Active Learning - Low scores = candidates for boost)
+                console.log(`[AlchemistService] Saving signal (${response.score}%)...`);
+                const { data: insertedSignal, error: insertError } = await supabase
+                    .from('signals')
+                    .insert([{
+                        user_id: userId,
+                        url: entry.url,
+                        title: entry.title,
+                        score: response.score,
+                        summary: response.summary,
+                        category: response.category,
+                        entities: response.entities,
+                        tags: response.tags,
+                        content: content,
+                        // Mark as dismissed if low score so it doesn't clutter main feed, 
+                        // but is available in "Low" filter
+                        is_dismissed: response.score < 50
+                    }])
+                    .select()
+                    .single();
+                if (insertError) {
+                    console.error('[AlchemistService] Insert error:', insertError);
+                    stats.errors++;
+                }
+                else {
+                    console.log('[AlchemistService] Signal saved successfully');
+                    if (response.relevant) {
+                        // High/Medium Score: Emit Signal Found & Auto-Embed
+                        await this.processingEvents.log({
+                            eventType: 'action',
+                            agentState: 'Signal',
+                            message: `Found signal: ${response.summary} (${response.score}%)`,
+                            level: 'info',
+                            metadata: response,
+                            durationMs: duration,
+                            userId
+                        }, supabase);
                         stats.signals++;
                         // 5. Generate Embedding & Check for Duplicates (non-blocking)
                         if (settings.embedding_model && await embeddingService.isAvailable()) {
@@ -172,18 +175,24 @@ export class AlchemistService {
                             });
                         }
                     }
-                }
-                else {
-                    // Emit: Skipped
-                    await this.processingEvents.log({
-                        eventType: 'info',
-                        agentState: 'Skipped',
-                        message: `Irrelevant content (${response.score}%): ${entry.title}`,
-                        level: 'debug',
-                        durationMs: duration,
-                        userId
-                    }, supabase);
-                    stats.skipped++;
+                    else {
+                        // Low Score: Emit Skipped (but it IS saved in DB now)
+                        // Trigger metadata-based deduplication (no embedding) to merge tracking links/redirects
+                        this.processDeduplicationOnly(insertedSignal, settings, userId, supabase).catch((err) => {
+                            console.error('[AlchemistService] Deduplication check failed:', err);
+                        });
+                        await this.processingEvents.log({
+                            eventType: 'info',
+                            agentState: 'Skipped',
+                            message: `Low signal stored for review (${response.score}%): ${entry.title}`,
+                            level: 'debug',
+                            durationMs: duration,
+                            userId
+                        }, supabase);
+                        // We count it as 'skipped' for the summary stats even though it's physically in the DB,
+                        // because it's not a "Found Signal" in the user's main feed context.
+                        stats.skipped++;
+                    }
                 }
             }
             catch (error) {
@@ -316,6 +325,23 @@ export class AlchemistService {
             return { score: 0, summary: 'Failed to parse', category: 'Error', entities: [], tags: [], relevant: false };
         }
     }
+    /**
+     * Process deduplication without generating embedding (Metadata only)
+     * Used for low-score signals to merge duplicates based on Title/URL
+     */
+    async processDeduplicationOnly(signal, settings, userId, supabase) {
+        // Check for duplicates using null embedding (forces metadata check)
+        const dedupeResult = await deduplicationService.checkAndMergeDuplicate(signal, null, // No embedding
+        userId, supabase, settings);
+        if (dedupeResult.isDuplicate) {
+            console.log(`[AlchemistService] Low-score signal is duplicate, merged into: ${dedupeResult.mergedSignalId}`);
+            // Delete the newly inserted signal since it's a duplicate
+            await supabase
+                .from('signals')
+                .delete()
+                .eq('id', signal.id);
+        }
+    }
     /**
      * Process embedding generation and deduplication for a signal
      * This runs asynchronously and doesn't block the main mining pipeline

package/dist/api/services/ChatService.js

@@ -64,9 +64,10 @@ export class ChatService {
             let sources = [];
             // 3. Retrieve Context (if embedding checks out)
             if (queryEmbedding) {
-                const similar = await embeddingService.findSimilarSignals(queryEmbedding, userId, 0.75, // Threshold
-                5 // Top K
+                const similar = await embeddingService.findSimilarSignals(queryEmbedding, userId, 0.55, // Lowered threshold for better recall
+                10 // Increased Top K
                 );
+                console.log(`[ChatService] RAG Retrieval: Found ${similar.length} signals for query: "${content}"`);
                 if (similar.length > 0) {
                     sources = similar.map(s => ({
                         id: s.id,

package/dist/api/services/DeduplicationService.js

@@ -16,22 +16,50 @@ export class DeduplicationService {
      */
     async checkAndMergeDuplicate(signal, embedding, userId, supabase, settings) {
         try {
-            // Find similar signals
-            const similar = await embeddingService.findSimilarSignals(embedding, userId, this.SIMILARITY_THRESHOLD, 5 // Check top 5 matches
-            );
-            if (similar.length === 0) {
-                return { isDuplicate: false };
+            // 1. Semantic Check (if embedding exists)
+            if (embedding && embedding.length > 0) {
+                const similar = await embeddingService.findSimilarSignals(embedding, userId, this.SIMILARITY_THRESHOLD, 5 // Check top 5 matches
+                );
+                if (similar.length > 0) {
+                    const bestMatch = similar[0];
+                    console.log(`[Deduplication] Found semantic duplicate: ${bestMatch.id} (score: ${bestMatch.score})`);
+                    const mergedId = await this.mergeSignals(bestMatch.id, signal, userId, supabase, settings);
+                    return { isDuplicate: true, mergedSignalId: mergedId, similarityScore: bestMatch.score };
+                }
             }
-            // Get the most similar signal
-            const bestMatch = similar[0];
-            console.log(`[Deduplication] Found similar signal: ${bestMatch.id} (score: ${bestMatch.score})`);
-            // Merge signals
-            const mergedId = await this.mergeSignals(bestMatch.id, signal, userId, supabase, settings);
-            return {
-                isDuplicate: true,
-                mergedSignalId: mergedId,
-                similarityScore: bestMatch.score
-            };
+            // 2. Title Match Check (Metadata Heuristic)
+            // Useful for redirected URLs or tracking links where content is same but URL differs
+            if (signal.title && signal.title.length > 10) {
+                const { data: titleMatch } = await supabase
+                    .from('signals')
+                    .select('id, score, title')
+                    .eq('user_id', userId)
+                    .ilike('title', signal.title.trim()) // Case-insensitive match
+                    .neq('id', signal.id || '00000000-0000-0000-0000-000000000000') // Don't match self
+                    .order('created_at', { ascending: false })
+                    .limit(1)
+                    .maybeSingle();
+                if (titleMatch) {
+                    console.log(`[Deduplication] Found title match: ${titleMatch.id} ("${titleMatch.title}")`);
+                    const mergedId = await this.mergeSignals(titleMatch.id, signal, userId, supabase, settings);
+                    return { isDuplicate: true, mergedSignalId: mergedId, similarityScore: 0.95 }; // High confidence
+                }
+            }
+            // 3. Exact URL Check (Fallback for signals without embeddings)
+            // Even if semantic check failed (or skipped), we shouldn't save the exact same URL twice.
+            const { data: existingUrlMatch } = await supabase
+                .from('signals')
+                .select('id, score')
+                .eq('user_id', userId)
+                .eq('url', signal.url)
+                .neq('id', signal.id || '00000000-0000-0000-0000-000000000000') // Don't match self
+                .maybeSingle();
+            if (existingUrlMatch) {
+                console.log(`[Deduplication] Found exact URL match: ${existingUrlMatch.id}`);
+                const mergedId = await this.mergeSignals(existingUrlMatch.id, signal, userId, supabase, settings);
+                return { isDuplicate: true, mergedSignalId: mergedId, similarityScore: 1.0 };
+            }
+            return { isDuplicate: false };
         }
         catch (error) {
             console.error('[Deduplication] Error:', error.message);