@realtimex/realtimex-alchemy 1.0.58 → 1.0.60

package/dist/CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.60] - 2026-01-28
+
+### Added
+- **Content Intelligence**: Implemented "Alchemist's Distillation" – The AI now actively cleanses mined content during analysis, stripping navigation, ads, and "read more" noise while preserving the core text verbatim.
+- **Data Quality**: Signals now store the AI-refined version of content (`refined_content`) instead of raw HTML scrapes, significantly improving the quality of future RAG retrieval.
+
+### Improved
+- **Context Window**: Increased the analysis safety ceiling from 10k to **200k characters** (~50k tokens). This ensures deep-dive articles, long-form essays, and technical documentation are analyzed in full without truncation.
+
+## [1.0.59] - 2026-01-28
+
+### Added
+- **Localization**: Completed full localization for **Authentication** and **Setup Wizard** flows.
+- **UI**: Integrated `LanguageSwitcher` directly into the Login and Setup screens, allowing users to choose their preferred language before onboarding.
+
+### Improved
+- **UX**: Enhanced error messages and helper text in the Setup Wizard with localized strings.
+
 ## [1.0.58] - 2026-01-28
 
 ### Documentation

package/dist/api/services/AlchemistService.js

@@ -120,8 +120,12 @@ export class AlchemistService {
                             content = `Page Title: ${entry.title} (Login/paywall required - content not accessible)`;
                         }
                         else {
-                            // Truncate to avoid token limits (keep ~8000 chars)
-                            const truncated = cleaned.length > 10000 ? cleaned.substring(0, 10000) + '...' : cleaned;
+                            // Safety ceiling to prevent pathological cases (e.g., entire doc sites scraped as one page)
+                            // 200k chars ≈ 50k tokens - only triggers on edge cases, not normal articles
+                            const MAX_SAFE_CHARS = 200000;
+                            const truncated = cleaned.length > MAX_SAFE_CHARS
+                                ? cleaned.substring(0, MAX_SAFE_CHARS) + '\n\n[Content truncated - exceeds 200k chars]'
+                                : cleaned;
                             content = `Page Title: ${entry.title}\nContent: ${truncated}`;
                         }
                     }
@@ -142,11 +146,25 @@ export class AlchemistService {
                     level: 'info',
                     userId
                 }, supabase);
-                // 3. LLM Analysis
+                // 3. LLM Analysis (+ Content Cleaning)
                 const response = await this.analyzeContent(content, finalUrl, settings, learningContext);
                 const duration = Date.now() - startAnalysis;
                 // 4. Save Signal (ALWAYS save for Active Learning - Low scores = candidates for boost)
                 console.log(`[AlchemistService] Saving signal (${response.score}%)...`);
+                // DECISION: Use "refined_content" from LLM if available and valid, otherwise fallback to our cleaned version
+                let finalContentToSave = content;
+                if (!isGatedContent && response.refined_content && response.refined_content.length > 50) {
+                    // Use the LLM's pristine version
+                    finalContentToSave = response.refined_content;
+                    // Re-prefix title if lost (optional, but good for context)
+                    if (!finalContentToSave.startsWith('Page Title:')) {
+                        finalContentToSave = `Page Title: ${entry.title}\n\n${finalContentToSave}`;
+                    }
+                }
+                else {
+                    // Fallback: Use original content (already has 200k safety ceiling applied above)
+                    finalContentToSave = content;
+                }
                 const { data: insertedSignal, error: insertError } = await supabase
                     .from('signals')
                     .insert([{
@@ -158,13 +176,14 @@ export class AlchemistService {
                         category: response.category,
                         entities: response.entities,
                         tags: (response.tags || []).map(t => t.toLowerCase().trim()),
-                        content: content,
+                        content: finalContentToSave,
                         // Mark as dismissed if low score OR gated content
                         is_dismissed: response.score < 50 || isGatedContent,
                         metadata: {
                             original_source_url: entry.url,
                             resolved_at: new Date().toISOString(),
-                            is_gated: isGatedContent
+                            is_gated: isGatedContent,
+                            ai_cleaned: !!(response.refined_content && response.refined_content.length > 50)
                         }
                     }])
                     .select()
@@ -281,7 +300,7 @@ export class AlchemistService {
            - "Page not found", error pages, access denied
            - App store pages, download prompts
            - Empty or placeholder content
-           For these, return: score=0, category="Other", summary="[Login wall/Navigation page/etc]", tags=[], entities=[]
+           For these, return: score=0, category="Other", summary="[Login wall/Navigation page/etc]", tags=[], entities=[], refined_content=""
 
         2. SCORING GUIDE:
            - High (80-100): Original research, data, insights, technical depth. MATCHES USER INTERESTS.
@@ -297,6 +316,13 @@ export class AlchemistService {
            "machine learning", "startups", "regulations", "cybersecurity", "investing"
            NEVER include: "login", "navigation", "authentication", "menu", "footer", "social media", "facebook", "meta"
 
+        5. CONTENT REFINEMENT (The "Alchemist's Distillation"):
+           - You must CLEAN the input content to remove noise.
+           - Remove: "Read more" links, social media footers, "Subscribe" prompts, navigation elements, ads.
+           - KEEP: The core article text VERBATIM. Do not rewrite sentences. Do not fix grammar. Only remove noise lines.
+           - If the content is already clean, return it as is.
+           - This "refined_content" will be stored as the permanent record.
+
         Return STRICT JSON:
         {
             "score": number (0-100),
@@ -304,7 +330,8 @@ export class AlchemistService {
             "summary": string (1-sentence concise gist, or "[Junk page]" if score=0),
             "entities": string[] (people, companies, products mentioned),
             "tags": string[] (3-5 TOPIC tags only, no platform/UI terms),
-            "relevant": boolean (true if score > 50)
+            "relevant": boolean (true if score > 50),
+            "refined_content": string (The noise-free, verbatim article text)
         }
         `;
         const response = await sdk.llm.chat([