bulltrackers-module 1.0.474 → 1.0.476

package/functions/computation-system/WorkflowOrchestrator.js

@@ -168,12 +168,17 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
     return report;
 }
 
-async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null, dependencyResultHashes = {}) {
+async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null, dependencyResultHashes = {}, metadata = {}) {
     const { logger } = dependencies;
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const calcManifest = manifestMap.get(normalizeName(targetComputation));
     
     if (!calcManifest) throw new Error(`Calc '${targetComputation}' not found.`);
+    
+    // Merge runtime metadata (like targetCid) into the manifest
+    // This allows the Executor to access it via 'calcInstance.manifest'
+    Object.assign(calcManifest, metadata);
+    
     calcManifest.dependencyResultHashes = dependencyResultHashes;
 
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);

package/functions/computation-system/executors/StandardExecutor.js

@@ -235,6 +235,14 @@ class StandardExecutor {
         let chunkFailures = 0;
 
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
+            // --- OPTIMIZATION: TARGET SPECIFIC USER ---
+            // If the request contains a targetCid, skip all other users immediately
+            if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) {
+                if (stats) stats.skippedUsers++;
+                continue;
+            }
+            // ------------------------------------------
+            
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory = historyData ? historyData[userId] : null;
             

package/functions/computation-system/helpers/computation_worker.js

@@ -93,7 +93,7 @@ function startMemoryHeartbeat(db, ledgerPath, workerId, computationName, traceId
 /**
  * STRICT IDEMPOTENCY GATE
  */
-async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId) {
+async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId, onDemand = false) {
     const docRef = db.doc(ledgerPath);
 
     try {
@@ -103,25 +103,27 @@ async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerI
             if (doc.exists) {
                 const data = doc.data();
                 
-                // [BUG] This blocks legitimate re-runs initiated by the Dispatcher
-                /* if (['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
-                    return { shouldRun: false, reason: `Task already in terminal state: ${data.status}` };
-                }
-                */
-
-                // [FIX] Only block if it's the EXACT SAME Dispatch ID (Duplicate Delivery)
-                if (['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
-                    if (data.dispatchId === dispatchId) {
-                        return { shouldRun: false, reason: `Task already in terminal state: ${data.status}` };
+                // [ON-DEMAND OVERRIDE] If this is an on-demand request, force re-run even if completed
+                if (onDemand && ['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
+                    // Force overwrite - on-demand requests should always run
+                    // Log will be handled by caller
+                    // Continue to claim lease below
+                } else {
+                    // [FIX] Only block if it's the EXACT SAME Dispatch ID (Duplicate Delivery)
+                    if (['COMPLETED', 'FAILED', 'CRASH'].includes(data.status)) {
+                        if (data.dispatchId === dispatchId) {
+                            return { shouldRun: false, reason: `Task already in terminal state: ${data.status}` };
+                        }
+                        // If dispatchId differs, we allow the overwrite (Re-Run).
+                        // The Dispatcher is the authority; if it sent a message, we run it.
                     }
-                    // If dispatchId differs, we allow the overwrite (Re-Run).
-                    // The Dispatcher is the authority; if it sent a message, we run it.
                 }
 
                 if (data.status === 'IN_PROGRESS' && data.dispatchId === dispatchId) {
                     return { shouldRun: false, reason: 'Duplicate delivery: Task already IN_PROGRESS with same ID.' };
                 }
-                if (data.status === 'IN_PROGRESS') {
+                if (data.status === 'IN_PROGRESS' && !onDemand) {
+                    // On-demand can break locks if needed, but regular requests should wait
                     return { shouldRun: false, reason: 'Collision: Task currently IN_PROGRESS by another worker.' };
                 }
             }
@@ -130,7 +132,8 @@ async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerI
                 status: 'IN_PROGRESS',
                 workerId: workerId,
                 dispatchId: dispatchId || 'unknown',
-                startedAt: new Date()
+                startedAt: new Date(),
+                onDemand: onDemand || false
             };
             
             t.set(docRef, lease, { merge: true });
@@ -151,8 +154,9 @@ async function handleComputationTask(message, config, dependencies) {
 
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') return;
     
-    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, resources, traceContext } = data;
+    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, resources, traceContext, metadata } = data;
     const resourceTier = resources || 'standard';
+    const onDemand = metadata?.onDemand === true || false; // Extract on-demand flag
     const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computation}`;
     const workerId = process.env.K_REVISION || os.hostname();
 
@@ -174,12 +178,16 @@ async function handleComputationTask(message, config, dependencies) {
     const runDeps = { ...dependencies, logger };
     const db = dependencies.db;
 
-    // --- STEP 1: IDEMPOTENCY CHECK ---
-    const gate = await checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId);
+    // --- STEP 1: IDEMPOTENCY CHECK (with on-demand override) ---
+    const gate = await checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId, onDemand);
     if (!gate.shouldRun) {
         logger.log('WARN', `[Worker] 🛑 Idempotency Gate: Skipping ${computation}. Reason: ${gate.reason}`);
         return; 
     }
+    
+    if (onDemand) {
+        logger.log('INFO', `[Worker] 🔄 On-demand request: Forcing re-run of ${computation} for ${date}`);
+    }
 
     logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}] [ID: ${dispatchId}]`);
 
@@ -195,7 +203,8 @@ async function handleComputationTask(message, config, dependencies) {
         
         const result = await executeDispatchTask(
             date, pass, computation, config, runDeps, 
-            manifest, previousCategory, dependencyResultHashes
+            manifest, previousCategory, dependencyResultHashes,
+            metadata // Pass metadata (including targetCid) to orchestrator
         );
 
         heartbeats.stop();

package/functions/computation-system/helpers/on_demand_helpers.js

@@ -0,0 +1,151 @@
+/**
+ * @fileoverview Helpers for on-demand computation requests
+ * Handles dependency chain resolution and ordered triggering
+ */
+
+const { getManifest } = require('../topology/ManifestLoader');
+const { normalizeName } = require('../utils/utils');
+
+// Import calculations package (matching computation_worker.js pattern)
+let calculationPackage;
+try { 
+    calculationPackage = require('aiden-shared-calculations-unified'); 
+} catch (e) { 
+    throw new Error(`Failed to load calculations package: ${e.message}`); 
+}
+const calculations = calculationPackage.calculations;
+
+/**
+ * Resolves all dependencies for a given computation and returns them grouped by pass
+ * @param {string} computationName - The target computation name
+ * @param {Array} manifest - The computation manifest
+ * @returns {Array<{pass: number, computations: Array<string>}>} Dependencies grouped by pass
+ */
+function resolveDependencyChain(computationName, manifest) {
+    const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
+    const normalizedTarget = normalizeName(computationName);
+    
+    const targetCalc = manifestMap.get(normalizedTarget);
+    if (!targetCalc) {
+        throw new Error(`Computation ${computationName} not found in manifest`);
+    }
+    
+    // Build adjacency list (dependencies map)
+    const adjacency = new Map();
+    for (const calc of manifest) {
+        const normName = normalizeName(calc.name);
+        adjacency.set(normName, (calc.dependencies || []).map(d => normalizeName(d)));
+    }
+    
+    // Get all dependencies recursively
+    const required = new Set([normalizedTarget]);
+    const queue = [normalizedTarget];
+    
+    while (queue.length > 0) {
+        const calcName = queue.shift();
+        const dependencies = adjacency.get(calcName) || [];
+        for (const dep of dependencies) {
+            if (!required.has(dep)) {
+                required.add(dep);
+                queue.push(dep);
+            }
+        }
+    }
+    
+    // Group by pass
+    const byPass = new Map();
+    for (const calcName of required) {
+        const calc = manifestMap.get(calcName);
+        if (calc) {
+            const pass = calc.pass || 1;
+            if (!byPass.has(pass)) {
+                byPass.set(pass, []);
+            }
+            byPass.get(pass).push(calc.name); // Use original name, not normalized
+        }
+    }
+    
+    // Convert to sorted array
+    const passes = Array.from(byPass.entries())
+        .sort((a, b) => a[0] - b[0])
+        .map(([pass, computations]) => ({ pass, computations }));
+    
+    return passes;
+}
+
+/**
+ * Triggers computations for a given date, handling dependencies in order
+ * @param {string} targetComputation - The target computation to run
+ * @param {string} date - The date string (YYYY-MM-DD)
+ * @param {object} dependencies - Contains pubsub, logger, etc.
+ * @param {object} config - Computation system config
+ * @param {object} metadata - Additional metadata for the computation request
+ * @returns {Promise<Array>} Array of triggered computation messages
+ */
+async function triggerComputationWithDependencies(targetComputation, date, dependencies, config, metadata = {}) {
+    const { pubsub, logger } = dependencies;
+    const computationTopic = config.computationTopicStandard || 'computation-tasks';
+    const topic = pubsub.topic(computationTopic);
+    const crypto = require('crypto');
+    
+    // Get manifest to resolve dependencies
+    const manifest = getManifest(config.activeProductLines || [], calculations);
+    
+    // Resolve dependency chain
+    const dependencyPasses = resolveDependencyChain(targetComputation, manifest);
+    
+    logger.log('INFO', `[On-Demand] Resolved dependency chain for ${targetComputation}:`, {
+        totalPasses: dependencyPasses.length,
+        passes: dependencyPasses.map(p => `Pass ${p.pass}: ${p.computations.length} computations`)
+    });
+    
+    const triggeredMessages = [];
+    
+    // Trigger each pass in order
+    for (const passGroup of dependencyPasses) {
+        const { pass, computations } = passGroup;
+        
+        // Trigger all computations in this pass
+        for (const computation of computations) {
+            const dispatchId = crypto.randomUUID();
+            const isTarget = normalizeName(computation) === normalizeName(targetComputation);
+            
+            const computationMessage = {
+                action: 'RUN_COMPUTATION_DATE',
+                computation: computation,
+                date: date,
+                pass: String(pass),
+                dispatchId: dispatchId,
+                triggerReason: metadata.triggerReason || 'on_demand',
+                resources: metadata.resources || 'standard',
+                metadata: {
+                    ...metadata,
+                    onDemand: true,
+                    isTargetComputation: isTarget,
+                    targetCid: metadata.targetCid || null // Pass through targetCid for optimization
+                },
+                traceContext: {
+                    traceId: crypto.randomBytes(16).toString('hex'),
+                    spanId: crypto.randomBytes(8).toString('hex'),
+                    sampled: true
+                }
+            };
+            
+            await topic.publishMessage({
+                data: Buffer.from(JSON.stringify(computationMessage))
+            });
+            
+            triggeredMessages.push(computationMessage);
+            
+            logger.log('INFO', `[On-Demand] Triggered ${computation} (Pass ${pass})${isTarget ? ' [TARGET]' : ' [DEPENDENCY]'}`);
+        }
+    }
+    
+    return triggeredMessages;
+}
+
+module.exports = {
+    resolveDependencyChain,
+    triggerComputationWithDependencies
+};
+

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,6 +1,7 @@
-# Data Feeder Pipeline (V3.1 - Syntax Fixed)
+# Data Feeder Pipeline (V3.2 - Automatic Root Data Indexing)
 # Starts at 22:00 UTC via Cloud Scheduler.
-# Fixes: Split assign/call steps and corrected assign syntax.
+# UPDATED: Removed intermediate root data indexer calls - each fetcher now automatically triggers indexing.
+# Only the global verification run at midnight remains.
 
 main:
   params: [input]
@@ -45,17 +46,7 @@ main:
     - wait_10_after_price:
         call: sys.sleep
         args: { seconds: 600 } # 10 Minutes
-
-    # FIX 1: Split assign and call
-    - prepare_index_price:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_price:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Price fetcher now automatically triggers root data indexer after completion
 
     - wait_10_before_insights:
         call: sys.sleep
@@ -78,17 +69,7 @@ main:
     - wait_10_after_insights:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 2: Split assign and call
-    - prepare_index_insights:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_insights:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Insights fetcher now automatically triggers root data indexer after completion
 
     # ==========================================
     # PHASE 2: WAIT FOR MIDNIGHT
@@ -124,17 +105,7 @@ main:
     - wait_10_after_rankings:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 3: Split assign and call
-    - prepare_index_rankings:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_rankings:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Popular investor rankings fetcher now automatically triggers root data indexer after completion
 
     - phase_0000_social:
         try:
@@ -158,7 +129,7 @@ main:
         call: http.post
         args:
           url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          # No targetDate = Global Run
+          # No targetDate = Global verification run (all intermediate indexing is now automatic)
           auth: { type: OIDC }
 
     # ==========================================
@@ -194,17 +165,7 @@ main:
               - wait_10_in_loop:
                   call: sys.sleep
                   args: { seconds: 600 }
-              
-              # FIX 4: Split assign and call
-              - prepare_index_loop:
-                   assign:
-                     - today: '${text.split(time.format(sys.now()), "T")[0]}'
-              - index_today_in_loop:
-                   call: http.post
-                   args:
-                     url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-                     body: { targetDate: '${today}' }
-                     auth: { type: OIDC }
+                  # NOTE: Social tasks are handled by task engine, which automatically triggers root data indexer after batch completion
               
               # FIX 5: Correct assign syntax (must be a list)
               - increment_loop:

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -94,6 +94,30 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         
         logger.log('INFO', `[PriceFetcherHelpers] Wrote price date tracking document for ${today} with ${priceDatesArray.length} dates (from August 2025 onwards)`);
         
+        // Update root data indexer for today's date after price data is stored
+        try {
+            const { runRootDataIndexer } = require('../../root-data-indexer/index');
+            const rootDataIndexerConfig = config.rootDataIndexer || {
+                availabilityCollection: 'system_root_data_index',
+                earliestDate: '2025-08-01',
+                collections: {
+                    prices: priceCollectionName
+                }
+            };
+            
+            const indexerConfig = {
+                ...rootDataIndexerConfig,
+                targetDate: today // Index only today's date for speed
+            };
+            
+            logger.log('INFO', `[PriceFetcherHelpers] Triggering root data indexer for date ${today} after price data storage...`);
+            await runRootDataIndexer(indexerConfig, dependencies);
+            logger.log('INFO', `[PriceFetcherHelpers] Root data indexer completed for date ${today}`);
+        } catch (indexerError) {
+            logger.log('ERROR', `[PriceFetcherHelpers] Failed to run root data indexer for ${today}`, indexerError);
+            // Continue - price data is stored, indexer failure is non-critical
+        }
+        
         const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
         logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
         return { success: true, message: successMessage, instrumentsProcessed: results.length };

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -118,6 +118,30 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
 
         await docRef.set(firestorePayload);
         
+        // Update root data indexer for today's date after insights data is stored
+        try {
+            const { runRootDataIndexer } = require('../../root-data-indexer/index');
+            const rootDataIndexerConfig = config.rootDataIndexer || {
+                availabilityCollection: 'system_root_data_index',
+                earliestDate: '2025-08-01',
+                collections: {
+                    insights: config.insightsCollectionName
+                }
+            };
+            
+            const indexerConfig = {
+                ...rootDataIndexerConfig,
+                targetDate: today // Index only today's date for speed
+            };
+            
+            logger.log('INFO', `[FetchInsightsHelpers] Triggering root data indexer for date ${today} after insights data storage...`);
+            await runRootDataIndexer(indexerConfig, dependencies);
+            logger.log('INFO', `[FetchInsightsHelpers] Root data indexer completed for date ${today}`);
+        } catch (indexerError) {
+            logger.log('ERROR', `[FetchInsightsHelpers] Failed to run root data indexer for ${today}`, indexerError);
+            // Continue - insights data is stored, indexer failure is non-critical
+        }
+        
         const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`; 
         logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, { documentId: today, instrumentCount: insightsData.length }); 
         return { success: true, message: successMsg, instrumentCount: insightsData.length };

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -113,6 +113,37 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
             });
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${rankingsCollectionName}/${today}`);
+            
+            // Update root data indexer for today's date after rankings data is stored
+            try {
+                const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                // Access rootDataIndexer from config (passed from index.js) or use defaults
+                // Using bracket notation to avoid TypeScript errors
+                let rootDataIndexerConfig;
+                if (config && typeof config === 'object' && config['rootDataIndexer']) {
+                    rootDataIndexerConfig = config['rootDataIndexer'];
+                } else {
+                    rootDataIndexerConfig = {
+                        availabilityCollection: 'system_root_data_index',
+                        earliestDate: '2025-08-01',
+                        collections: {
+                            piRankings: rankingsCollectionName
+                        }
+                    };
+                }
+                
+                const indexerConfig = Object.assign({}, rootDataIndexerConfig, {
+                    targetDate: today // Index only today's date for speed
+                });
+                
+                logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
+                await runRootDataIndexer(indexerConfig, dependencies);
+                logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed for date ${today}`);
+            } catch (indexerError) {
+                logger.log('ERROR', `[PopularInvestorFetch] Failed to run root data indexer for ${today}`, indexerError);
+                // Continue - rankings data is stored, indexer failure is non-critical
+            }
+            
             return { success: true, count: data.TotalRows };
 
         } catch (dbError) {

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -149,6 +149,7 @@ async function requestPiFetch(req, res, dependencies, config) {
             actualRequestedBy: Number(userCid), // Track actual developer CID
             metadata: {
                 onDemand: true,
+                targetCid: piCidNum, // Target specific user for optimization
                 requestedAt: now.toISOString(),
                 isImpersonating: isImpersonating || false
             }

package/functions/generic-api/user-api/helpers/user_sync_helpers.js

@@ -163,6 +163,7 @@ async function requestUserSync(req, res, dependencies, config) {
             effectiveRequestedBy: effectiveCid,
             metadata: {
                 onDemand: true,
+                targetCid: targetCidNum, // Target specific user for optimization
                 requestedAt: now.toISOString(),
                 isImpersonating: isImpersonating || false
             }

package/functions/generic-api/user-api/helpers/verification_helpers.js

@@ -163,31 +163,55 @@ async function finalizeVerification(req, res, dependencies, config) {
         }, { merge: true });
 
         // 3. Trigger Downstream Systems via Pub/Sub
+        // Send unified request to task engine that handles both portfolio/history AND social data
         const pubsubUtils = new PubSubUtils(dependencies);
-
-        // TRIGGER 1: Portfolio (Only if Public)
+        
+        // Create a unified on-demand request that includes both portfolio and social
+        // The task engine will process both, update root data, then trigger computations
+        const unifiedTask = {
+            type: 'ON_DEMAND_USER_UPDATE', // Matches Task Engine handler
+            data: {
+                cid: realCID,
+                username: profileData.username,
+                source: 'user_signup', // Mark as signup to ensure computations are triggered
+                includeSocial: true, // Flag to include social data fetch
+                since: new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString() // Last 7 days for social
+            },
+            metadata: {
+                onDemand: true,
+                targetCid: realCID, // Optimization: only process this user
+                isNewUser: true, // Flag to add to daily update queue
+                requestedAt: new Date().toISOString()
+            }
+        };
+        
+        // Only trigger if user is public (has portfolio data)
         if (!isOptOut) {
-            const portfolioTask = {
-                type: 'ON_DEMAND_USER_UPDATE', // Matches Task Engine handler
+            await pubsubUtils.publish(pubsubTopicUserFetch, unifiedTask);
+            logger.log('INFO', `[Verification] Triggered unified data fetch (portfolio + social) for ${username} (${realCID})`);
+        } else {
+            // For private users, still fetch social data but no portfolio
+            const socialOnlyTask = {
+                type: 'ON_DEMAND_USER_UPDATE',
                 data: {
                     cid: realCID,
-                    username: profileData.username
+                    username: profileData.username,
+                    source: 'user_signup',
+                    includeSocial: true,
+                    portfolioOnly: false, // Skip portfolio for private users
+                    since: new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString()
+                },
+                metadata: {
+                    onDemand: true,
+                    targetCid: realCID,
+                    isNewUser: true,
+                    requestedAt: new Date().toISOString()
                 }
             };
-            await pubsubUtils.publish(pubsubTopicUserFetch, portfolioTask);
-            logger.log('INFO', `[Verification] Triggered Portfolio Fetch for ${username}`);
+            await pubsubUtils.publish(pubsubTopicUserFetch, socialOnlyTask);
+            logger.log('INFO', `[Verification] Triggered social-only fetch for private user ${username} (${realCID})`);
         }
 
-        // TRIGGER 2: Social Posts (Always, assuming social is visible or standard handling applies)
-        const socialTask = {
-            type: 'SIGNED_IN_USER', // Matches Social Task Handler
-            id: String(realCID),
-            username: profileData.username,
-            since: new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString() // Last 7 days initial fetch
-        };
-        await pubsubUtils.publish(pubsubTopicSocialFetch, socialTask);
-        logger.log('INFO', `[Verification] Triggered Social Fetch for ${username}`);
-
         return res.status(200).json({ 
             success: true, 
             message: "Account verified successfully. Data ingestion started.",

package/functions/social-orchestrator/helpers/orchestrator_helpers.js

@@ -78,15 +78,56 @@ exports.runSocialOrchestrator = async (config, dependencies) => {
             return { success: true, message: "No tasks to run." };
         }
 
-        // Use batch publish (assuming pubsubUtils handles batching logic internally or via array)
+        // Publish to task engine (not social task handler)
+        // The task engine will now handle social posts along with portfolio/history
+        const taskEngineTopicName = config.taskEngineTopicName || process.env.PUBSUB_TOPIC_USER_FETCH || 'etoro-user-fetch-topic';
+        
+        // Convert social tasks to task engine format
+        const taskEngineTasks = tasks.map(task => ({
+            type: task.type === 'INSTRUMENT' ? 'SOCIAL_INSTRUMENT_FETCH' : 
+                  task.type === 'POPULAR_INVESTOR' ? 'SOCIAL_PI_FETCH' : 
+                  'SOCIAL_SIGNED_IN_USER_FETCH',
+            data: {
+                id: task.id,
+                username: task.username,
+                since: task.since,
+                type: task.type
+            }
+        }));
+        
+        // Initialize batch counter for social tasks
+        const userTasksCount = tasks.filter(t => t.type === 'POPULAR_INVESTOR' || t.type === 'SIGNED_IN_USER').length;
+        let socialCounterRef = null;
+        
+        if (userTasksCount > 0) {
+            try {
+                const counterId = `social-${today}-${Date.now()}`;
+                socialCounterRef = db.collection('social_batch_counters').doc(counterId);
+                await socialCounterRef.set({
+                    totalTasks: userTasksCount,
+                    remainingTasks: userTasksCount,
+                    startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    date: today
+                });
+                logger.log('INFO', `[SocialOrchestrator] Initialized social batch counter: ${userTasksCount} user tasks for ${today}`);
+            } catch (counterError) {
+                logger.log('WARN', `[SocialOrchestrator] Failed to initialize social batch counter`, counterError);
+            }
+        }
+        
+        // Use batch publish to task engine
         await pubsubUtils.batchPublishTasks(dependencies, { 
-            topicName: socialFetchTaskTopicName, 
-            tasks: tasks, 
-            taskType: 'social-fetch-task' 
+            topicName: taskEngineTopicName, 
+            tasks: taskEngineTasks, 
+            taskType: 'social-fetch-task',
+            metadata: {
+                batchCounterRef: socialCounterRef?.path || null,
+                targetDate: today
+            }
         });
 
-        logger.log('SUCCESS', `[SocialOrchestrator] Published ${tasks.length} total social fetch tasks.`);
-        return { success: true, tasksQueued: tasks.length };
+        logger.log('SUCCESS', `[SocialOrchestrator] Published ${taskEngineTasks.length} total social fetch tasks to task engine.`);
+        return { success: true, tasksQueued: taskEngineTasks.length, batchCounterRef: socialCounterRef?.path || null };
 
     } catch (error) {
         logger.log('ERROR', '[SocialOrchestrator] Fatal error.', error);

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -162,7 +162,7 @@ async function getAdvancedAnalysisFromGemini(dependencies, snippet) {
 /**
  * Main pipe (Task Handler): pipe.maintenance.handleSocialTask
  */
-exports.handleSocialTask = async (message, context, config, dependencies) => {
+async function handleSocialTask(message, context, config, dependencies) {
     const { db, logger, headerManager, proxyManager } = dependencies;
     let task;
     try { 
@@ -383,4 +383,6 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
     } finally {
         await headerManager.flushPerformanceUpdates();
     }
-};
+}
+
+module.exports = { handleSocialTask, getGcidForUser };