bulltrackers-module 1.0.68 → 1.0.70

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -6,8 +6,14 @@
 
 const { FieldPath } = require('@google-cloud/firestore');
 // Import sub-pipes/utils from their new locations
-// --- MODIFIED: Added loadDailyInsights ---
-const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights } = require('../utils/data_loader.js');
+// --- MODIFIED: Added loadDailySocialPostInsights ---
+const { 
+    getPortfolioPartRefs, 
+    loadFullDayMap, 
+    loadDataByRefs, 
+    loadDailyInsights,
+    loadDailySocialPostInsights // <-- NEW
+} = require('../utils/data_loader.js');
 const {
     historicalCalculations, dailyCalculations, HISTORICAL_CALC_NAMES,
     withRetry,
@@ -34,13 +40,22 @@ async function runComputationOrchestrator(config, dependencies) {
     const masterDailyList = Object.entries(dailyCalculations).flatMap(([cat, calcs]) =>
         Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
     );
-    // --- NEW: Add insights calculations to the master lists ---
+    // --- (EXISTING) insights calculations ---
     const insightsCalculations = require('aiden-shared-calculations-unified').calculations.insights || {};
     const masterInsightsList = Object.keys(insightsCalculations).map(name => ({ category: 'insights', calcName: name }));
 
-    const masterFullList = [...masterHistoricalList, ...masterDailyList, ...masterInsightsList];
+    // --- NEW: Add social post calculations to the master lists ---
+    const socialPostCalculations = require('aiden-shared-calculations-unified').calculations.socialPosts || {};
+    const masterSocialPostList = Object.keys(socialPostCalculations).map(name => ({ category: 'socialPosts', calcName: name }));
     // --- END NEW ---
 
+    const masterFullList = [
+        ...masterHistoricalList, 
+        ...masterDailyList, 
+        ...masterInsightsList,
+        ...masterSocialPostList // <-- NEW
+    ];
+
     // Pass dependencies to sub-pipe
     const firstDate = await getFirstDateFromSourceData(config, dependencies);
     const startDateUTC = firstDate
@@ -54,18 +69,25 @@ async function runComputationOrchestrator(config, dependencies) {
     const existingDateIds = new Set(insightDocs.docs.map(d => d.id).filter(id => /^\d{4}-\d{2}-\d{2}$/.test(id)));
 
     // --- PASS 1 ---
-    // --- MODIFIED: Include insights calcs in missing list ---
+    // --- MODIFIED: Include insights & social calcs in missing list ---
     const missingDates = allExpectedDates.filter(dateStr => !existingDateIds.has(dateStr));
-    const pass1Jobs = missingDates.map(date => ({ date, missing: [...masterDailyList, ...masterInsightsList] }));
+    const pass1Jobs = missingDates.map(date => ({ 
+        date, 
+        missing: [...masterDailyList, ...masterInsightsList, ...masterSocialPostList] // <-- NEW
+    }));
     // --- END MODIFIED ---
 
-    logger.log('INFO', `[Orchestrator] Pass 1: Found ${pass1Jobs.length} missing dates to process (daily & insights calcs).`);
+    logger.log('INFO', `[Orchestrator] Pass 1: Found ${pass1Jobs.length} missing dates to process (daily, insights & social calcs).`);
     // Pass dependencies to sub-pipe
-    // --- MODIFIED: Combine daily and insights calcs for Pass 1 run ---
-    const pass1SourcePackage = { ...dailyCalculations, insights: insightsCalculations };
+    // --- MODIFIED: Combine daily, insights, and social calcs for Pass 1 run ---
+    const pass1SourcePackage = { 
+        ...dailyCalculations, 
+        insights: insightsCalculations,
+        socialPosts: socialPostCalculations // <-- NEW
+    };
     const pass1Results = await processJobsInParallel(
         pass1Jobs,
-        (date, missing) => runUnifiedComputation(date, missing, 'Pass 1 (Daily & Insights)', pass1SourcePackage, config, dependencies),
+        (date, missing) => runUnifiedComputation(date, missing, 'Pass 1 (Daily, Insights & Social)', pass1SourcePackage, config, dependencies),
         'Pass 1',
         config
     );
@@ -94,9 +116,10 @@ async function runComputationOrchestrator(config, dependencies) {
         pass2Jobs,
         async (date, missing) => {
             const historicalMissing = missing.filter(c => HISTORICAL_CALC_NAMES.has(c.calcName));
-            // --- MODIFIED: Include insights in daily missing ---
-            const dailyMissing = missing.filter(c => !HISTORICAL_CALC_NAMES.has(c.calcName) && c.category !== 'insights');
+            // --- MODIFIED: Include insights & social in daily missing ---
+            const dailyMissing = missing.filter(c => !HISTORICAL_CALC_NAMES.has(c.calcName) && c.category !== 'insights' && c.category !== 'socialPosts');
             const insightsMissing = missing.filter(c => c.category === 'insights');
+            const socialPostsMissing = missing.filter(c => c.category === 'socialPosts'); // <-- NEW
             // --- END MODIFIED ---
             const results = [];
 
@@ -105,12 +128,16 @@ async function runComputationOrchestrator(config, dependencies) {
                 const histResult = await runUnifiedComputation(date, historicalMissing, 'Pass 2 (Historical)', historicalCalculations, config, dependencies);
                 results.push(histResult);
             }
-             // --- MODIFIED: Run daily and insights separately or combined ---
-            const dailyAndInsightsMissing = [...dailyMissing, ...insightsMissing];
-            if (dailyAndInsightsMissing.length > 0) {
-                 logger.log('INFO', `[Pass 2] Running ${dailyAndInsightsMissing.length} daily/insights calcs for ${date.toISOString().slice(0, 10)}.`);
-                 const dailyAndInsightsSourcePackage = { ...dailyCalculations, insights: insightsCalculations };
-                const dailyResult = await runUnifiedComputation(date, dailyAndInsightsMissing, 'Pass 2 (Daily/Insights)', dailyAndInsightsSourcePackage, config, dependencies);
+             // --- MODIFIED: Run daily, insights, and social ---
+            const dailyAndInsightsAndSocialMissing = [...dailyMissing, ...insightsMissing, ...socialPostsMissing]; // <-- NEW
+            if (dailyAndInsightsAndSocialMissing.length > 0) {
+                 logger.log('INFO', `[Pass 2] Running ${dailyAndInsightsAndSocialMissing.length} daily/insights/social calcs for ${date.toISOString().slice(0, 10)}.`);
+                 const dailyAndInsightsSourcePackage = { 
+                     ...dailyCalculations, 
+                     insights: insightsCalculations,
+                     socialPosts: socialPostCalculations // <-- NEW
+                 };
+                const dailyResult = await runUnifiedComputation(date, dailyAndInsightsAndSocialMissing, 'Pass 2 (Daily/Insights/Social)', dailyAndInsightsSourcePackage, config, dependencies);
                 results.push(dailyResult);
             }
             // --- END MODIFIED ---
@@ -149,16 +176,23 @@ function initializeCalculators(calculationsToRun, sourcePackage, logger) {
 
 /**
  * Internal sub-pipe: Streams data and calls process() on calculators.
- * --- MODIFIED: Added todayInsights and yesterdayInsights ---
+ * --- MODIFIED: Added today/yesterday social post insights ---
  */
-async function streamAndProcess(dateStr, todayRefs, state, passName, config, dependencies, yesterdayPortfolios = {}, todayInsights = null, yesterdayInsights = null) {
+async function streamAndProcess(
+    dateStr, todayRefs, state, passName, config, dependencies, 
+    yesterdayPortfolios = {}, 
+    todayInsights = null, 
+    yesterdayInsights = null,
+    todaySocialPostInsights = null,  // <-- NEW
+    yesterdaySocialPostInsights = null // <-- NEW
+) {
     const { db, logger } = dependencies;
     logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
 
     const { instrumentToTicker, instrumentToSector } = await unifiedUtils.loadInstrumentMappings();
     const context = { instrumentMappings: instrumentToTicker, sectorMapping: instrumentToSector };
     const batchSize = config.partRefBatchSize || 10;
-    let isFirstUser = true; // Flag for insights calculations
+    let isFirstUser = true; // Flag for insights/social calculations
 
     for (let i = 0; i < todayRefs.length; i += batchSize) {
         const batchRefs = todayRefs.slice(i, i + batchSize);
@@ -178,14 +212,21 @@ async function streamAndProcess(dateStr, todayRefs, state, passName, config, dep
 
                 const [category, calcName] = key.split('/');
                 let processArgs;
-
-                // --- MODIFIED: Handle insights calculations ---
-                if (category === 'insights') {
-                    // Only process insights once per day, using the first user as a trigger
+                const allContextArgs = [
+                    context, 
+                    todayInsights, 
+                    yesterdayInsights, 
+                    todaySocialPostInsights,  // <-- NEW
+                    yesterdaySocialPostInsights // <-- NEW
+                ];
+
+                // --- MODIFIED: Handle insights & social calculations ---
+                if (category === 'insights' || category === 'socialPosts') {
+                    // Only process these once per day, using the first user as a trigger
                     if (isFirstUser) {
-                         processArgs = [null, null, null, context, todayInsights, yesterdayInsights];
+                         processArgs = [null, null, null, ...allContextArgs];
                     } else {
-                        continue; // Skip insights processing for subsequent users
+                        continue; // Skip processing for subsequent users
                     }
                 } else if (HISTORICAL_CALC_NAMES.has(calcName)) {
                      // --- MODIFIED: Handle missing yesterday's portfolio gracefully ---
@@ -195,14 +236,14 @@ async function streamAndProcess(dateStr, todayRefs, state, passName, config, dep
                          // logger.log('TRACE', `Skipping historical calc ${key} for user ${uid} due to missing yesterday portfolio.`);
                          continue;
                     }
-                    processArgs = [p, pYesterday, uid, context, todayInsights, yesterdayInsights]; // Pass insights too
+                    processArgs = [p, pYesterday, uid, ...allContextArgs]; // Pass all context
                 } else {
                     // Standard daily calculation
                     if ((userType === 'normal' && category === 'speculators') ||
                         (userType === 'speculator' && !['speculators', 'sanity'].includes(category))) {
                         continue;
                     }
-                    processArgs = [p, null, uid, context, todayInsights, yesterdayInsights]; // Pass null for yesterdayPortfolio, add insights
+                    processArgs = [p, null, uid, ...allContextArgs]; // Pass null for yesterdayPortfolio, add all context
                 }
                 // --- END MODIFIED ---
 
@@ -219,7 +260,7 @@ async function streamAndProcess(dateStr, todayRefs, state, passName, config, dep
 
 /**
  * Internal sub-pipe: Runs computations for a single date.
- * --- MODIFIED: Load and pass insights data ---
+ * --- MODIFIED: Load and pass social post insights data ---
  */
 async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, sourcePackage, config, dependencies) {
     const { db, logger } = dependencies;
@@ -227,33 +268,35 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
     logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
 
     try {
-        // --- NEW: Load today's insights ---
+        // --- (EXISTING) Load today's instrument insights ---
         const todayInsightsData = await loadDailyInsights(config, dependencies, dateStr);
+        // --- NEW: Load today's social post insights ---
+        const todaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, dateStr);
         // --- END NEW ---
 
         // Pass dependencies to sub-pipe
         const todayRefs = await getPortfolioPartRefs(config, dependencies, dateStr);
-         // --- MODIFIED: Check if *any* data exists (portfolio OR insights) ---
-        if (todayRefs.length === 0 && !todayInsightsData) {
-            logger.log('WARN', `[${passName}] No portfolio data OR insights data found for ${dateStr}. Skipping.`);
+         // --- MODIFIED: Check if *any* data exists (portfolio OR insights OR social) ---
+        if (todayRefs.length === 0 && !todayInsightsData && !todaySocialPostInsightsData) {
+            logger.log('WARN', `[${passName}] No portfolio, instrument insights, OR social post data found for ${dateStr}. Skipping.`);
             return { success: true, date: dateStr, message: "No source data for today." };
         }
         // --- END MODIFIED ---
 
 
         let yesterdayPortfolios = {};
-        let yesterdayInsightsData = null; // NEW: Variable for yesterday's insights
-        // --- MODIFIED: Check if *any* calc requires yesterday data (portfolio OR insights) ---
+        let yesterdayInsightsData = null; 
+        let yesterdaySocialPostInsightsData = null; // <-- NEW
+        
+        // --- MODIFIED: Check if *any* calc requires yesterday data ---
         const requiresYesterdayPortfolio = calculationsToRun.some(c => sourcePackage[c.category]?.[c.calcName]?.prototype?.process.length >= 3);
-        const requiresYesterdayInsights = calculationsToRun.some(c => c.category === 'insights'); // Insights calcs always need yesterday
+        const requiresYesterdayInsights = calculationsToRun.some(c => c.category === 'insights');
+        const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.category === 'socialPosts'); // <-- NEW
         // --- END MODIFIED ---
 
-         if (requiresYesterdayPortfolio || requiresYesterdayInsights) {
-            const prev = new Date(dateToProcess);
-            prev.setUTCDate(prev.getUTCDate() - 1);
-            const prevStr = prev.toISOString().slice(0, 10);
-
-             // --- MODIFIED: Load yesterday's insights if needed ---
+         if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) { // <-- NEW
+            
+            // --- (EXISTING) Load yesterday's instrument insights ---
             if(requiresYesterdayInsights) {
                 let daysAgo = 1;
                 const maxLookback = 30; // Or from config
@@ -263,38 +306,77 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                     prev.setUTCDate(prev.getUTCDate() - daysAgo);
                     const prevStr = prev.toISOString().slice(0, 10);
                     
-                    // Keep trying to load data until we find some
                     yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
                     
                     if (yesterdayInsightsData) {
-                        logger.log('INFO', `[${passName}] Found 'yesterday' insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                        logger.log('INFO', `[${passName}] Found 'yesterday' instrument insights data from ${daysAgo} day(s) ago (${prevStr}).`);
                     } else {
-                        daysAgo++; // Try the previous day
+                        daysAgo++;
                     }
                 }
-                
                 if (!yesterdayInsightsData) {
-                    logger.log('WARN', `[${passName}] Could not find any 'yesterday' insights data within a ${maxLookback} day lookback.`);
+                    logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
                 }
             }
-            // --- END MODIFIED ---
+            // --- END (EXISTING) ---
+            
+            // --- NEW: Load yesterday's social post insights ---
+            if(requiresYesterdaySocialPosts) {
+                let daysAgo = 1;
+                const maxLookback = 30; 
+                
+                while (!yesterdaySocialPostInsightsData && daysAgo <= maxLookback) {
+                    const prev = new Date(dateToProcess);
+                    prev.setUTCDate(prev.getUTCDate() - daysAgo);
+                    const prevStr = prev.toISOString().slice(0, 10);
+                    
+                    yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
+                    
+                    if (yesterdaySocialPostInsightsData) {
+                        logger.log('INFO', `[${passName}] Found 'yesterday' social post insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                    } else {
+                        daysAgo++;
+                    }
+                }
+                if (!yesterdaySocialPostInsightsData) {
+                    logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
+                }
+            }
+            // --- END NEW ---
 
+            // --- (EXISTING) Load yesterday's portfolio data ---
             if (requiresYesterdayPortfolio) {
+                const prev = new Date(dateToProcess);
+                prev.setUTCDate(prev.getUTCDate() - 1);
+                const prevStr = prev.toISOString().slice(0, 10);
                 const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
+                
                 if (yesterdayRefs.length > 0) {
                     yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
                      logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
                 } else {
                     logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
-                    // Ensure calculations that strictly need yesterday's portfolio don't run or handle the missing data
                 }
             }
+            // --- END (EXISTING) ---
         }
 
 
         const state = initializeCalculators(calculationsToRun, sourcePackage, logger);
         // Pass dependencies AND insights data to sub-pipe
-        await streamAndProcess(dateStr, todayRefs, state, passName, config, dependencies, yesterdayPortfolios, todayInsightsData, yesterdayInsightsData);
+        await streamAndProcess(
+            dateStr, 
+            todayRefs, 
+            state, 
+            passName, 
+            config, 
+            dependencies, 
+            yesterdayPortfolios, 
+            todayInsightsData, 
+            yesterdayInsightsData,
+            todaySocialPostInsightsData,    // <-- NEW
+            yesterdaySocialPostInsightsData // <-- NEW
+        );
 
         let successCount = 0;
         const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection); // Use db

package/functions/computation-system/utils/data_loader.js

@@ -3,6 +3,7 @@
  * REFACTORED: Now stateless and receive dependencies.
  */
 const { withRetry } = require('aiden-shared-calculations-unified').utils;
+const { FieldPath } = require('@google-cloud/firestore'); // <<< --- ADD FieldPath
 
 /**
  * Sub-pipe: pipe.computationSystem.dataLoader.getPortfolioPartRefs
@@ -99,7 +100,7 @@ async function loadFullDayMap(config, dependencies, partRefs) {
 }
 
 /**
- * --- NEW ---
+ * --- (EXISTING) ---
  * Sub-pipe: pipe.computationSystem.dataLoader.loadDailyInsights
  * Fetches the daily instrument insights document for a specific date.
  * @param {object} config - The computation system configuration object.
@@ -126,11 +127,50 @@ async function loadDailyInsights(config, dependencies, dateString) {
         return null; // Return null on error to allow computations to proceed partially if possible
     }
 }
+
+/**
+ * --- NEW ---
+ * Sub-pipe: pipe.computationSystem.dataLoader.loadDailySocialPostInsights
+ * Fetches all analyzed social post documents for a specific date.
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @returns {Promise<object|null>} An object map of { [postId]: postData } or null.
+ */
+async function loadDailySocialPostInsights(config, dependencies, dateString) {
+    const { db, logger } = dependencies;
+    const socialInsightsCollectionName = config.socialInsightsCollectionName || 'daily_social_insights';
+    logger.log('INFO', `Loading social post insights for date: ${dateString} from ${socialInsightsCollectionName}`);
+    
+    try {
+        const postsCollectionRef = db.collection(socialInsightsCollectionName).doc(dateString).collection('posts');
+        const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
+
+        if (querySnapshot.empty) {
+            logger.log('WARN', `No social post insights found for ${dateString}.`);
+            return null;
+        }
+        
+        const postsMap = {};
+        querySnapshot.forEach(doc => {
+            postsMap[doc.id] = doc.data();
+        });
+        
+        logger.log('TRACE', `Successfully loaded ${Object.keys(postsMap).length} social post insights for ${dateString}.`);
+        return postsMap;
+
+    } catch (error) {
+        logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
+        return null;
+    }
+}
 // --- END NEW ---
 
+
 module.exports = {
     getPortfolioPartRefs,
     loadDataByRefs,
     loadFullDayMap,
-    loadDailyInsights, // Export the new function
+    loadDailyInsights,
+    loadDailySocialPostInsights, // Export the new function
 };

package/functions/social-orchestrator/helpers/orchestrator_helpers.js

@@ -0,0 +1,62 @@
+/**
+ * @fileoverview Main pipe: pipe.maintenance.runSocialOrchestrator
+ * This function is triggered by a schedule. It fans out the work by
+ * publishing one Pub/Sub message for each target ticker.
+ */
+
+/**
+ * Main pipe (Orchestrator): pipe.maintenance.runSocialOrchestrator
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, firestoreUtils, pubsubUtils.
+ * @returns {Promise<object>} Summary of the orchestration.
+ */
+exports.runSocialOrchestrator = async (config, dependencies) => {
+    const { logger, pubsubUtils } = dependencies;
+    logger.log('INFO', '[SocialOrchestrator] Starting social post fetching orchestration...');
+
+    // Validate configuration
+    if (!config.targetTickerIds || !Array.isArray(config.targetTickerIds) || !config.socialFetchTaskTopicName || !config.socialFetchTimeWindowHours) {
+        logger.log('ERROR', '[SocialOrchestrator] Missing required configuration: targetTickerIds (array), socialFetchTaskTopicName, or socialFetchTimeWindowHours.');
+        throw new Error('Missing required configuration for Social Orchestrator.');
+    }
+
+    try {
+        const tasks = [];
+        const { targetTickerIds, socialFetchTimeWindowHours } = config;
+        
+        // Calculate the 'since' timestamp
+        const sinceTimestamp = new Date();
+        sinceTimestamp.setHours(sinceTimestamp.getHours() - socialFetchTimeWindowHours);
+        
+        // Add a 15-minute buffer to ensure overlap
+        sinceTimestamp.setMinutes(sinceTimestamp.getMinutes() - 15); 
+        
+        const sinceISO = sinceTimestamp.toISOString();
+
+        for (const tickerId of targetTickerIds) {
+            tasks.push({
+                tickerId: String(tickerId), // Ensure it's a string
+                since: sinceISO
+            });
+        }
+
+        if (tasks.length === 0) {
+            logger.log('WARN', '[SocialOrchestrator] No target tickers found to process.');
+            return { success: true, message: "No target tickers configured." };
+        }
+
+        // Use pubsubUtils to batch publish all ticker tasks
+        await pubsubUtils.batchPublishTasks(dependencies, {
+            topicName: config.socialFetchTaskTopicName,
+            tasks: tasks,
+            taskType: 'social-fetch-task'
+        });
+
+        logger.log('SUCCESS', `[SocialOrchestrator] Successfully published ${tasks.length} social fetch tasks for window >= ${sinceISO}.`);
+        return { success: true, tasksQueued: tasks.length };
+
+    } catch (error) {
+        logger.log('ERROR', '[SocialOrchestrator] Fatal error during orchestration.', { errorMessage: error.message, errorStack: error.stack });
+        throw error;
+    }
+};

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -0,0 +1,243 @@
+/**
+ * @fileoverview Main pipe: pipe.maintenance.handleSocialTask
+ * This function is triggered by Pub/Sub for a single ticker.
+ * It fetches posts, deduplicates, analyzes, and stores them.
+ */
+
+const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+
+/**
+ * --- NEW HELPER FUNCTION ---
+ * Calls Gemini to classify sentiment.
+ * @param {object} dependencies - Contains logger and geminiModel.
+ * @param {string} snippet - The text snippet to analyze.
+ * @returns {Promise<string>} 'Bullish', 'Bearish', or 'Neutral'.
+ */
+async function getSentimentFromGemini(dependencies, snippet) {
+    const { logger, geminiModel } = dependencies;
+
+    if (!geminiModel) {
+        logger.log('WARN', '[getSentimentFromGemini] Gemini model not found in dependencies.');
+        return 'Neutral';
+    }
+
+    const prompt = `You are a financial sentiment analyzer for social media posts.
+Classify the following post as 'Bullish', 'Bearish', or 'Neutral'.
+Respond with only one of those three words.
+
+Post: "${snippet}"`;
+
+    try {
+        const request = {
+            contents: [{ parts: [{ text: prompt }] }],
+            generationConfig: {
+                // Set low temperature for classification
+                temperature: 0.1, 
+                topP: 0.1,
+                // Set max output tokens to be safe
+                maxOutputTokens: 10 
+            }
+        };
+
+        const result = await geminiModel.generateContent(request);
+        const response = result.response;
+        const text = response.candidates[0].content.parts[0].text.trim();
+
+        // Robust parsing: Find the keyword, case-insensitive.
+        const match = text.match(/(Bullish|Bearish|Neutral)/i);
+        if (match && match[0]) {
+            // Capitalize first letter, e.g., "bullish" -> "Bullish"
+            const sentiment = match[0].charAt(0).toUpperCase() + match[0].slice(1).toLowerCase();
+            return sentiment;
+        }
+
+        logger.log('WARN', `[getSentimentFromGemini] Unexpected response from AI: "${text}". Defaulting to Neutral.`);
+        return 'Neutral';
+
+    } catch (error) {
+        logger.log('ERROR', '[getSentimentFromGemini] Error calling Gemini API.', { 
+            errorMessage: error.message,
+            errorStack: error.stack
+        });
+        // Default to 'Neutral' on API error to avoid halting the pipeline
+        return 'Neutral';
+    }
+}
+
+
+/**
+ * Main pipe (Task Handler): pipe.maintenance.handleSocialTask
+ * @param {object} message - The Pub/Sub message.
+ * @param {object} context - The message context.
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, geminiModel.
+ * @returns {Promise<void>}
+ */
+exports.handleSocialTask = async (message, context, config, dependencies) => {
+    // --- MODIFIED: Added geminiModel to dependencies ---
+    const { db, logger, headerManager, proxyManager } = dependencies;
+
+    let task;
+    try {
+        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
+    } catch (e) {
+        logger.log('ERROR', '[SocialTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
+        return; // Acknowledge the message
+    }
+
+    const { tickerId, since } = task;
+    const sinceDate = new Date(since);
+    const taskId = `social-${tickerId}-${context.eventId || Date.now()}`;
+    logger.log('INFO', `[SocialTask/${taskId}] Processing ticker ${tickerId} for posts since ${since}.`);
+
+    // --- Config validation ---
+    if (!config.socialApiBaseUrl || !config.socialInsightsCollectionName || !config.processedPostsCollectionName) {
+        logger.log('ERROR', `[SocialTask/${taskId}] Missing required configuration.`);
+        throw new Error('Missing required configuration for Social Task.');
+    }
+
+    const processedPostsRef = db.collection(config.processedPostsCollectionName);
+    const today = new Date().toISOString().slice(0, 10);
+    const insightsCollectionRef = db.collection(config.socialInsightsCollectionName).doc(today).collection('posts');
+
+    let offset = 0;
+    const take = 10; // hardcode to 10
+    let keepFetching = true;
+    let postsProcessed = 0;
+    let postsSaved = 0;
+    const processedInThisRun = new Set(); // Local dedupe
+
+    try {
+        while (keepFetching) {
+            const url = `${config.socialApiBaseUrl}${tickerId}?take=${take}&offset=${offset}&reactionsPageSize=20`;
+            logger.log('TRACE', `[SocialTask/${taskId}] Fetching: ${url}`);
+            
+            const selectedHeader = await headerManager.selectHeader();
+            let wasSuccess = false;
+            let response;
+            
+            try {
+                response = await proxyManager.fetch(url, { headers: selectedHeader.header });
+                if (!response.ok) {
+                    throw new Error(`API error ${response.status}`);
+                }
+                wasSuccess = true;
+            } catch (fetchError) {
+                logger.log('WARN', `[SocialTask/${taskId}] Fetch failed for offset ${offset}.`, { err: fetchError.message });
+                keepFetching = false; // Stop on fetch error
+                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, false);
+                continue;
+            } finally {
+                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+            }
+
+            const page = await response.json();
+            const discussions = page?.discussions;
+
+            if (!Array.isArray(discussions) || discussions.length === 0) {
+                logger.log('INFO', `[SocialTask/${taskId}] No more posts found at offset ${offset}. Stopping.`);
+                keepFetching = false;
+                continue;
+            }
+
+            const postIds = discussions.map(d => d.post.id).filter(Boolean);
+            if (postIds.length === 0) {
+                offset += take;
+                continue;
+            }
+
+            // --- Deduplication Check ---
+            const existingDocs = await processedPostsRef.where(FieldPath.documentId(), 'in', postIds).get();
+            const existingIds = new Set(existingDocs.docs.map(d => d.id));
+
+            const batch = db.batch();
+            let newPostsInBatch = 0;
+
+            for (const discussion of discussions) {
+                const post = discussion?.post;
+                if (!post || !post.id || !post.message?.text) continue;
+
+                // Stop pagination if we've reached posts older than our window
+                const postCreatedDate = new Date(post.created);
+                if (postCreatedDate < sinceDate) {
+                    keepFetching = false;
+                    continue; 
+                }
+
+                // Skip if already processed
+                if (existingIds.has(post.id) || processedInThisRun.has(post.id)) {
+                    continue;
+                }
+                
+                // Filter language (accept 'en' and 'en-gb')
+                const lang = post.message.languageCode || 'unknown';
+                if (lang !== 'en' && lang !== 'en-gb') {
+                    continue; // Skip non-English posts
+                }
+
+                // --- Process the new post ---
+                postsProcessed++;
+                processedInThisRun.add(post.id);
+                const text = post.message.text;
+
+                // 1. Truncate for AI
+                const MAX_CHARS = 500; // ~125 tokens, very cheap
+                let snippet = text;
+                if (text.length > (MAX_CHARS * 2)) {
+                  // Create a "summary" snippet
+                  snippet = text.substring(0, MAX_CHARS) + " ... " + text.substring(text.length - MAX_CHARS);
+                } else if (text.length > MAX_CHARS) {
+                  // Just truncate
+                  snippet = text.substring(0, MAX_CHARS);
+                }
+
+                // 2. AI Sentiment Analysis
+                // Pass full dependencies object
+                const sentiment = await getSentimentFromGemini(dependencies, snippet);
+
+                // 3. Prepare data for storage
+                const postData = {
+                    sentiment: sentiment,
+                    textSnippet: snippet,
+                    language: lang,
+                    tickers: post.tags.map(t => t.market?.symbolName).filter(Boolean),
+                    postOwnerId: post.owner?.id,
+                    createdAt: post.created,
+                    fetchedAt: FieldValue.serverTimestamp()
+                };
+                
+                // 4. Add to batch for `daily_social_insights`
+                const insightDocRef = insightsCollectionRef.doc(post.id);
+                batch.set(insightDocRef, postData);
+
+                // 5. Add to batch for `processed_social_posts` (dedupe collection)
+                const dedupeDocRef = processedPostsRef.doc(post.id);
+                batch.set(dedupeDocRef, { processedAt: FieldValue.serverTimestamp() });
+                
+                newPostsInBatch++;
+            }
+
+            if (newPostsInBatch > 0) {
+                await batch.commit();
+                postsSaved += newPostsInBatch;
+                logger.log('INFO', `[SocialTask/${taskId}] Saved ${newPostsInBatch} new posts from offset ${offset}.`);
+            }
+
+            // Continue to next page
+            offset += take;
+        }
+
+        logger.log('SUCCESS', `[SocialTask/${taskId}] Run complete. Processed ${postsProcessed} new posts, saved ${postsSaved}.`);
+
+    } catch (error) {
+        logger.log('ERROR', `[SocialTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
+        throw error;
+    } finally {
+        // Always flush header performance
+        try {
+            await headerManager.flushPerformanceUpdates();
+        } catch (flushError) {
+            logger.log('ERROR', `[SocialTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message });
+        }
+    }
+};

package/functions/user-activity-sampler/helpers/sampler_helpers.js

@@ -5,6 +5,36 @@
  * - handleSampleBlockTask: Processes a single block with parallel fetching.
  */
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit'); // npm install p-limit
+
+/**
+ * Helper: delay
+ */
+function delay(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Helper: fetch with retry and exponential backoff on 429
+ */
+async function fetchWithRetry(fetchFn, maxRetries = 5, logger, taskId) {
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const response = await fetchFn();
+            if (response.status !== 429) return response;
+
+            const backoff = 1000 * Math.pow(2, attempt); // 1s, 2s, 4s, 8s...
+            logger.log('WARN', `[SamplerTask/${taskId}] Received 429. Retrying in ${backoff}ms (attempt ${attempt + 1}/${maxRetries})`);
+            await delay(backoff);
+        } catch (err) {
+            logger.log('ERROR', `[SamplerTask/${taskId}] Fetch failed on attempt ${attempt + 1}.`, { errorMessage: err.message });
+            if (attempt === maxRetries) throw err;
+            const backoff = 500 * Math.pow(2, attempt); // slightly faster backoff for network errors
+            await delay(backoff);
+        }
+    }
+    throw new Error('Max retries reached for fetchWithRetry');
+}
 
 /**
  * Main pipe (Orchestrator): pipe.maintenance.runUserActivitySamplerOrchestrator
@@ -139,25 +169,17 @@ async function fetchSampleBatch(blockId, config, dependencies, processedInThisRu
 }
 
 /**
- * Main pipe (Task Handler): pipe.maintenance.handleSampleBlockTask
- * This function is triggered by Pub/Sub for a single block.
- * It runs the sampling in parallel to finish within timeout.
- *
- * @param {object} message - The Pub/Sub message.
- * @param {object} context - The message context.
- * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
- * @returns {Promise<void>}
+ * Updated Task Handler: handleSampleBlockTask
  */
 exports.handleSampleBlockTask = async (message, context, config, dependencies) => {
-    const { db, logger, headerManager } = dependencies;
-    
+    const { db, logger, headerManager, proxyManager } = dependencies;
+
     let task;
     try {
         task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
     } catch (e) {
         logger.log('ERROR', '[SamplerTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return; // Acknowledge the message to prevent retries
+        return;
     }
 
     const { blockId } = task;
@@ -165,7 +187,7 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
     const today = new Date().toISOString().slice(0, 10);
     logger.log('INFO', `[SamplerTask/${taskId}] Processing block ${blockId}...`);
 
-    // --- Config validation for this task ---
+    // Config validation
     if (!config.rankingsApiUrl || !config.targetPublicUsersPerBlock || !config.apiBatchSize || !config.outputCollectionName || !config.parallelRequests) {
         logger.log('ERROR', `[SamplerTask/${taskId}] Missing required configuration for task execution.`);
         throw new Error('Missing required configuration for Sampler Task.');
@@ -174,27 +196,37 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
     const processedInThisRun = new Set();
     let N_public_sampled_block = 0;
     let N_sampled_total_block = 0;
-    const public_users_data = []; // Store { CID, LastActivity }
-
+    const public_users_data = [];
     const MAX_ATTEMPTS = config.maxSamplingAttemptsPerBlock || 1000;
     let totalBatchesAttempted = 0;
-    const CONCURRENT_REQUESTS = config.parallelRequests;
+
+    const limit = pLimit(config.parallelRequests || 3);
 
     try {
-        // --- Start Parallel Loop ---
         while (N_public_sampled_block < config.targetPublicUsersPerBlock && totalBatchesAttempted < MAX_ATTEMPTS) {
+            const numRequests = Math.min(config.parallelRequests, MAX_ATTEMPTS - totalBatchesAttempted);
             const promises = [];
-            const numRequests = Math.min(CONCURRENT_REQUESTS, MAX_ATTEMPTS - totalBatchesAttempted);
 
-            logger.log('TRACE', `[SamplerTask/${taskId}] Starting parallel batch of ${numRequests} requests...`);
             for (let i = 0; i < numRequests; i++) {
-                promises.push(fetchSampleBatch(blockId, config, dependencies, processedInThisRun));
+                promises.push(limit(async () => {
+                    // Wrap fetchSampleBatch to include fetchWithRetry
+                    async function fetchBatchWithRetry() {
+                        return await fetchSampleBatch(blockId, config, {
+                            ...dependencies,
+                            proxyManager: {
+                                fetch: (url, options) =>
+                                    fetchWithRetry(() => proxyManager.fetch(url, options), 5, logger, taskId)
+                            }
+                        }, processedInThisRun);
+                    }
+                    return await fetchBatchWithRetry();
+                }));
             }
 
             const results = await Promise.allSettled(promises);
             totalBatchesAttempted += numRequests;
 
-            // Process results from the parallel batch
+            // Process results
             for (const result of results) {
                 if (result.status === 'fulfilled' && result.value.success) {
                     const batchResult = result.value;
@@ -202,27 +234,21 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
                     N_public_sampled_block += batchResult.publicUsers.length;
                     public_users_data.push(...batchResult.publicUsers);
                 } else if (result.status === 'fulfilled' && !result.value.success) {
-                    // Failed API call, but we still count the CIDs we tried to sample
                     N_sampled_total_block += result.value.cidsSent;
                 } else {
-                    // Promise rejected (unexpected error)
                     logger.log('WARN', `[SamplerTask/${taskId}] A sample fetch promise was rejected.`, { reason: result.reason });
                 }
             }
-            
-            logger.log('INFO', `[SamplerTask/${taskId}] Batch complete. Total public sampled: ${N_public_sampled_block}/${config.targetPublicUsersPerBlock}`);
 
-            // --- REMOVED artificial delay ---
+            logger.log('INFO', `[SamplerTask/${taskId}] Batch complete. Total public sampled: ${N_public_sampled_block}/${config.targetPublicUsersPerBlock}`);
         }
-        // --- End Parallel Loop ---
 
         if (totalBatchesAttempted >= MAX_ATTEMPTS) {
             logger.log('WARN', `[SamplerTask/${taskId}] Reached max sampling attempts (${MAX_ATTEMPTS}). Proceeding with ${N_public_sampled_block} users.`);
         }
 
-        // --- Calculate and Store Results (same as before) ---
+        // --- Calculate and Store Results ---
         const f_private_block = N_sampled_total_block > 0 ? 1 - (N_public_sampled_block / N_sampled_total_block) : 0;
-
         const counts = { A1: 0, A2: 0, A3: 0, A4: 0 };
         const now = new Date();
         const oneDayAgo = new Date(now.getTime() - (24 * 60 * 60 * 1000));
@@ -230,9 +256,7 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
         const threeMonthsAgo = new Date(now.getTime() - (90 * 24 * 60 * 60 * 1000));
 
         public_users_data.forEach(user => {
-            if (!user.LastActivity) {
-                counts.A4++; return;
-            }
+            if (!user.LastActivity) { counts.A4++; return; }
             try {
                 const lastActivityDate = new Date(user.LastActivity);
                 if (isNaN(lastActivityDate)) { counts.A4++; return; }
@@ -240,25 +264,20 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
                 else if (lastActivityDate >= oneWeekAgo) counts.A2++;
                 else if (lastActivityDate >= threeMonthsAgo) counts.A3++;
                 else counts.A4++;
-            } catch (e) {
-                logger.log('WARN', `[SamplerTask/${taskId}] Error parsing LastActivity date '${user.LastActivity}' for user ${user.CID}. Counting as A4.`);
+            } catch {
                 counts.A4++;
             }
         });
 
         const fractions = {};
-        for (const category in counts) {
-            fractions[category] = N_public_sampled_block > 0 ? (counts[category] / N_public_sampled_block) : 0;
-        }
+        for (const category in counts) fractions[category] = N_public_sampled_block > 0 ? (counts[category] / N_public_sampled_block) : 0;
 
         const N_block = 1000000;
         const estimatedCounts = {};
-        for (const category in fractions) {
-            estimatedCounts[category] = Math.round(fractions[category] * N_block);
-        }
+        for (const category in fractions) estimatedCounts[category] = Math.round(fractions[category] * N_block);
 
         const result = {
-            blockId: blockId,
+            blockId,
             sampledDate: today,
             f_private: f_private_block,
             publicSampleSize: N_public_sampled_block,
@@ -272,15 +291,12 @@ exports.handleSampleBlockTask = async (message, context, config, dependencies) =
 
         const docRef = db.collection(config.outputCollectionName).doc(`${blockId}_${today}`);
         await docRef.set(result);
-        
         logger.log('SUCCESS', `[SamplerTask/${taskId}] Stored results for block ${blockId}. Sampled: ${N_public_sampled_block}. Private fraction: ${f_private_block.toFixed(3)}.`);
 
     } catch (error) {
         logger.log('ERROR', `[SamplerTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
-        // Re-throw the error to signal failure to Cloud Functions, which will trigger a retry.
         throw error;
     } finally {
-        // Always flush header performance at the end of the task, even on failure.
         try {
             await headerManager.flushPerformanceUpdates();
             logger.log('INFO', `[SamplerTask/${taskId}] Header performance flushed.`);

package/index.js

@@ -5,6 +5,7 @@
  */
 
 // --- Core Utilities (Classes and Stateless Helpers) ---
+// ... (no changes here) ...
 const core = {
     IntelligentHeaderManager: require('./functions/core/utils/intelligent_header_manager').IntelligentHeaderManager,
     IntelligentProxyManager: require('./functions/core/utils/intelligent_proxy_manager').IntelligentProxyManager,
@@ -14,6 +15,7 @@ const core = {
 };
 
 // --- Pipe 1: Orchestrator ---
+// ... (no changes here) ...
 const orchestrator = {
     // Main Pipes (Entry points for Cloud Functions)
     runDiscoveryOrchestrator: require('./functions/orchestrator/index').runDiscoveryOrchestrator,
@@ -30,6 +32,7 @@ const orchestrator = {
 };
 
 // --- Pipe 2: Dispatcher ---
+// ... (no changes here) ...
 const dispatcher = {
     // Main Pipe
     handleRequest: require('./functions/dispatcher/index').handleRequest,
@@ -39,6 +42,7 @@ const dispatcher = {
 };
 
 // --- Pipe 3: Task Engine ---
+// ... (no changes here) ...
 const taskEngine = {
     // Main Pipe
     handleRequest: require('./functions/task-engine/handler_creator').handleRequest,
@@ -50,6 +54,7 @@ const taskEngine = {
 };
 
 // --- Pipe 4: Computation System ---
+// ... (no changes here) ...
 const computationSystem = {
     // Main Pipe
     runOrchestration: require('./functions/computation-system/helpers/orchestration_helpers').runComputationOrchestrator,
@@ -60,6 +65,7 @@ const computationSystem = {
 };
 
 // --- Pipe 5: API ---
+// ... (no changes here) ...
 const api = {
     // Main Pipe
     createApiApp: require('./functions/generic-api/index').createApiApp,
@@ -80,9 +86,15 @@ const maintenance = {
     runUserActivitySamplerOrchestrator: require('./functions/user-activity-sampler/helpers/sampler_helpers').runUserActivitySamplerOrchestrator, 
     handleSampleBlockTask: require('./functions/user-activity-sampler/helpers/sampler_helpers').handleSampleBlockTask, 
     // --- END UPDATE ---
+    
+    // --- NEW SOCIAL SENTIMENT FUNCTIONS ---
+    runSocialOrchestrator: require('./functions/social-orchestrator/helpers/orchestrator_helpers').runSocialOrchestrator,
+    handleSocialTask: require('./functions/social-task-handler/helpers/handler_helpers').handleSocialTask,
+    // --- END NEW ---
 };
 
 // --- Pipe 7: Proxy ---
+// ... (no changes here) ...
 const proxy = {
     handlePost: require('./functions/appscript-api/index').handlePost,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.68",
+  "version": "1.0.70",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -16,7 +16,9 @@
     "functions/fetch-insights/",
     "functions/etoro-price-fetcher/",
     "functions/appscript-api/",
-    "functions/user-activity-sampler/"
+    "functions/user-activity-sampler/",
+    "functions/social-orchestrator/",
+    "functions/social-task-handler/"
   ],
   "keywords": [
     "bulltrackers",
@@ -32,7 +34,8 @@
     "aiden-shared-calculations-unified": "1.0.0",
     "@google-cloud/pubsub": "latest",
     "express": "^4.19.2",
-    "cors": "^2.8.5"
+    "cors": "^2.8.5",
+    "p-limit": "latest"
   },
   "engines": {
     "node": ">=20"