bulltrackers-module 1.0.130 → 1.0.132

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,9 +6,11 @@
  * It reads its pass number from the config and executes only those calculations.
  * This file contains the high-level "manual" of steps. The "how-to" logic
  * is extracted into 'computation_system_utils.js'.
+ * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
  */
 const { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings, getFirstDateFromSourceData } = require('../utils/utils.js');
+// --- MODIFIED: Import getEarliestDataDates ---
+const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
@@ -17,7 +19,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    const firstDate = await getFirstDateFromSourceData(config, dependencies);
+    
+    // --- MODIFIED: Call new date function ---
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const firstDate = earliestDates.absoluteEarliest; // Use the absolute earliest for the loop
+    // --- END MODIFICATION ---
+    
     const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
 
@@ -30,7 +37,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
     for (const dateStr of allExpectedDates) {
         const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies); if (!rootData) continue;
+            // --- MODIFIED: Pass earliestDates map to checkRootDataAvailability ---
+            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates); if (!rootData) continue;
+            // --- END MODIFICATION ---
+            
             const fetchedDeps = await fetchDependenciesForPass(dateStr, calcsInThisPass, computationManifest, config, dependencies);
             const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, fetchedDeps, passToRun, dateStr, logger);
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
@@ -44,4 +54,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -5,32 +5,95 @@ const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 /** Stage 1: Group manifest by pass number */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/** Stage 2: Check root data dependencies for a calc */
-function checkRootDependencies(calcManifest, rootDataStatus) { if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) return true;
-    for (const dep of calcManifest.rootDataDependencies) if ((dep==='portfolio'&&!rootDataStatus.hasPortfolio)||(dep==='insights'&&!rootDataStatus.hasInsights)||(dep==='social'&&!rootDataStatus.hasSocial)||(dep==='history'&&!rootDataStatus.hasHistory)) return false;
-    return true;
+/** * --- MODIFIED: Returns detailed missing dependencies for logging ---
+ * Stage 2: Check root data dependencies for a calc 
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    const missing = [];
+    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
+        return { canRun: true, missing };
+    }
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+    }
+    return { canRun: missing.length === 0, missing };
 }
 
-/** Stage 3: Check root data availability for a date */
-// --- START FIX ---
-// The signature is changed from '{ logger, ...deps }' to 'dependencies'
-// to ensure the full object is passed down.
-async function checkRootDataAvailability(dateStr, config, dependencies) {
-    const { logger } = dependencies; // Destructure logger for local use
+/** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
+ * Stage 3: Check root data availability for a date 
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
+    const { logger } = dependencies;
     logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
+
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+    let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
+
     try {
-        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
-            getPortfolioPartRefs(config, dependencies, dateStr),        // Pass full 'dependencies'
-            loadDailyInsights(config, dependencies, dateStr),         // Pass full 'dependencies'
-            loadDailySocialPostInsights(config, dependencies, dateStr), // Pass full 'dependencies'
-            getHistoryPartRefs(config, dependencies, dateStr)           // Pass full 'dependencies'
-        ]);
-        const hasPortfolio = !!(portfolioRefs?.length), hasInsights = !!insightsData, hasSocial = !!socialData, hasHistory = !!(historyRefs?.length);
-        if (!(hasPortfolio||hasInsights||hasSocial||hasHistory)) { logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null; }
-        return { portfolioRefs: portfolioRefs||[], insightsData: insightsData||null, socialData: socialData||null, historyRefs: historyRefs||[], status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
-    } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
+        const tasks = [];
+
+        if (dateToProcess >= earliestDates.portfolio) {
+            tasks.push(
+                getPortfolioPartRefs(config, dependencies, dateStr).then(res => {
+                    portfolioRefs = res;
+                    hasPortfolio = !!(res?.length);
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.insights) {
+            tasks.push(
+                loadDailyInsights(config, dependencies, dateStr).then(res => {
+                    insightsData = res;
+                    hasInsights = !!res;
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.social) {
+            tasks.push(
+                loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {
+                    socialData = res;
+                    hasSocial = !!res;
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.history) {
+            tasks.push(
+                getHistoryPartRefs(config, dependencies, dateStr).then(res => {
+                    historyRefs = res;
+                    hasHistory = !!(res?.length);
+                })
+            );
+        }
+
+        await Promise.all(tasks);
+
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
+            logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`);
+            return null;
+        }
+
+        return {
+            portfolioRefs,
+            insightsData,
+            socialData,
+            historyRefs,
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
+        };
+
+    } catch (err) {
+        logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message });
+        return null;
+    }
 }
-// --- END FIX ---
+
+
 
 /** Stage 4: Fetch computed dependencies from Firestore */
 async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
@@ -50,14 +113,47 @@ async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, conf
     return fetched;
 }
 
-/** Stage 5: Filter calculations based on available root data and dependencies */
+/** * --- MODIFIED: Added detailed logging ---
+ * Stage 5: Filter calculations based on available root data and dependencies 
+ */
 function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, fetchedDeps, passToRun, dateStr, logger) {
     const skipped = new Set();
-    const standardCalcsToRun = standardCalcs.filter(c => checkRootDependencies(c, rootDataStatus) || (logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} missing root data`), skipped.add(c.name), false));
-    const metaCalcsToRun = metaCalcs.filter(c => checkRootDependencies(c, rootDataStatus) && (c.dependencies||[]).every(d=>fetchedDeps[normalizeName(d)]) || (logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} missing dep`), skipped.add(c.name), false));
+
+    // Filter Standard Calcs
+    const standardCalcsToRun = standardCalcs.filter(c => {
+        const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
+        if (canRun) return true;
+        
+        logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
+        skipped.add(c.name);
+        return false;
+    });
+
+    // Filter Meta Calcs
+    const metaCalcsToRun = metaCalcs.filter(c => {
+        // 1. Check root data
+        const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
+        if (!canRun) {
+            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        
+        // 2. Check computed dependencies
+        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !fetchedDeps[d]);
+        if (missingDeps.length > 0) {
+            logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        
+        return true; // All checks passed
+    });
+    
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
+
 /** Stage 6: Initialize calculator instances */
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 
@@ -79,20 +175,159 @@ async function streamAndProcess(dateStr, todayRefs, state, passName, config, dep
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
-    const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    const fullRoot=await loadHistoricalData(date, calcs, config, deps, rootData);
-    const state=initializeCalculators(calcs, logger);
+    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
+    const state = initializeCalculators(calcs, logger);
     await streamAndProcess(dStr, fullRoot.portfolioRefs, state, passName, config, deps, fullRoot);
-    let success=0; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.getResult!=='function') continue; try{ const result=await Promise.resolve(calc.getResult()); if(result&&Object.keys(result).length){ /* Commit logic omitted for brevity */ success++; } } catch(e){logger.log('ERROR',`getResult failed ${name} for ${dStr}`,{err:e.message,stack:e.stack});} }
-    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+    
+    // --- START: FULL COMMIT LOGIC ---
+    let success = 0;
+    const standardWrites = [];
+    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
+
+    for (const name in state) {
+        const calc = state[name];
+        if (!calc || typeof calc.getResult !== 'function') continue;
+        
+        try {
+            const result = await Promise.resolve(calc.getResult());
+            if (result && Object.keys(result).length > 0) {
+                
+                // Separate sharded data from standard data
+                const standardResult = {};
+                for (const key in result) {
+                    if (key.startsWith('sharded_')) {
+                        // This is sharded data, e.g., sharded_user_profitability
+                        // The value is expected to be: { "collection_name": { "doc1": {...}, "doc2": {...} } }
+                        const shardedData = result[key];
+                        for (const collectionName in shardedData) {
+                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+                            // Merge doc data (e.g., combining data for "user_profitability_shard_1")
+                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
+                        }
+                    } else {
+                        // This is a standard, single-doc result
+                        standardResult[key] = result[key];
+                    }
+                }
+
+                // Add standard result to the batch
+                if (Object.keys(standardResult).length > 0) {
+                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                        .collection(config.computationsSubcollection).doc(name);
+                    
+                    standardWrites.push({ ref: docRef, data: standardResult });
+                }
+                success++; // Mark as success even if only sharded data was produced
+            }
+        } catch (e) {
+            logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
+        }
+    }
+    
+    // Commit standard (non-sharded) writes in chunks
+    if (standardWrites.length > 0) {
+        await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
+    }
+
+    // Commit all sharded writes
+    for (const collectionName in shardedWrites) {
+        const docs = shardedWrites[collectionName];
+        const shardedDocWrites = [];
+        for (const docId in docs) {
+            // This assumes docId is the full path for sharded docs, or just the doc ID
+            // Based on user_profitability_tracker, it's just the doc ID.
+            const docRef = deps.db.collection(collectionName).doc(docId);
+            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        }
+        if (shardedDocWrites.length > 0) {
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+        }
+    }
+    // --- END: FULL COMMIT LOGIC ---
+
+    logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
 /** Stage 10: Run meta computations */
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) { const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
+    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    let success=0; for(const mCalc of calcs){ const name=normalizeName(mCalc.name), Cl=mCalc.class; if(typeof Cl!=='function'){ logger.log('ERROR',`Invalid class ${name}`); continue; } const inst=new Cl(); try{ const result=await Promise.resolve(inst.process(dStr,{...deps,rootData},config,fetchedDeps)); if(result&&Object.keys(result).length){ /* Commit logic omitted */ success++; } } catch(e){logger.log('ERROR',`Meta-calc failed ${name} for ${dStr}`,{err:e.message,stack:e.stack}); } }
-    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+
+    // --- START: FULL COMMIT LOGIC ---
+    let success = 0;
+    const standardWrites = [];
+    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
+
+    for (const mCalc of calcs) {
+        const name = normalizeName(mCalc.name), Cl = mCalc.class;
+        if (typeof Cl !== 'function') {
+            logger.log('ERROR', `Invalid class ${name}`);
+            continue;
+        }
+        const inst = new Cl();
+        try {
+            // Pass the full dependencies object to process()
+            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
+            
+            if (result && Object.keys(result).length > 0) {
+                
+                // Separate sharded data from standard data
+                const standardResult = {};
+                for (const key in result) {
+                    if (key.startsWith('sharded_')) {
+                        const shardedData = result[key];
+                        for (const collectionName in shardedData) {
+                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
+                        }
+                    } else {
+                        standardResult[key] = result[key];
+                    }
+                }
+
+                // Add standard result to the batch
+                if (Object.keys(standardResult).length > 0) {
+                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                        .collection(config.resultsSubcollection).doc(mCalc.category)
+                        .collection(config.computationsSubcollection).doc(name);
+                    
+                    standardWrites.push({ ref: docRef, data: standardResult });
+                }
+                success++;
+            }
+        } catch (e) {
+            logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
+        }
+    }
+
+    // Commit standard (non-sharded) writes in chunks
+    if (standardWrites.length > 0) {
+        await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
+    }
+
+    // Commit all sharded writes
+    for (const collectionName in shardedWrites) {
+        const docs = shardedWrites[collectionName];
+        const shardedDocWrites = [];
+        for (const docId in docs) {
+            // Special case for stateful meta-calcs that write to a specific path
+            const docRef = docId.includes('/')
+                ? deps.db.doc(docId) // docId is a full path
+                : deps.db.collection(collectionName).doc(docId); // docId is just an ID
+            
+            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        }
+        if (shardedDocWrites.length > 0) {
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+        }
+    }
+    // --- END: FULL COMMIT LOGIC ---
+    
+    logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
+
 module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/functions/computation-system/utils/utils.js

@@ -2,8 +2,10 @@
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
  * DYNAMIC: Categorization logic is removed, replaced by manifest.
+ * --- MODIFIED: getFirstDateFromSourceData is now getEarliestDataDates
+ * and queries all data sources to build an availability map. ---
  */
-/** --- Computation System Sub-Pipes & Utils (Stateless) --- */
+/** --- Computation System Sub-Pipes & Utils (Stateless, Dependency-Injection) --- */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
@@ -46,13 +48,46 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/** Stage 4: Get the earliest date in a collection */
+/**
+ * --- NEW HELPER ---
+ * Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates.
+ */
+async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+
+    try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`);
+            return null;
+        }
+        const query = db.collection(collectionName)
+            .where(FieldPath.documentId(), '>=', '2000-01-01')
+            .orderBy(FieldPath.documentId(), 'asc')
+            .limit(1);
+        
+        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
+        
+        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) {
+            return new Date(snapshot.docs[0].id + 'T00:00:00Z');
+        }
+    } catch (e) {
+        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
+    }
+    return null;
+}
+
+/** Stage 4: Get the earliest date in a sharded collection */
 async function getFirstDateFromCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
 
     let earliestDate = null;
     try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`);
+            return null;
+        }
         const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
         if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
 
@@ -75,26 +110,69 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     return earliestDate;
 }
 
-/** Stage 5: Determine the earliest date from source data across both user types */
-async function getFirstDateFromSourceData(config, deps) {
+/** * --- MODIFIED FUNCTION ---
+ * Stage 5: Determine the earliest date from *all* source data.
+ */
+async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
-    logger.log('INFO', 'Querying for earliest date from source portfolio data...');
-
-    const investorDate = await getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection);
-    const speculatorDate = await getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection);
-
-    let earliestDate;
-    if (investorDate && speculatorDate) earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-    else earliestDate = investorDate || speculatorDate;
-
-    if (earliestDate) {
-        logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-        return earliestDate;
-    } else {
-        const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-        logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
-        return fallbackDate;
-    }
+    logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
+
+    // These return null on error or if empty
+    const [
+        investorDate,
+        speculatorDate,
+        investorHistoryDate,
+        speculatorHistoryDate,
+        insightsDate,
+        socialDate
+    ] = await Promise.all([
+        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
+        getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
+        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
+    ]);
+    
+    // Helper to find the minimum (earliest) of a set of dates
+    const getMinDate = (...dates) => {
+        const validDates = dates.filter(Boolean); // Filter out nulls
+        if (validDates.length === 0) return null;
+        return new Date(Math.min(...validDates));
+    };
+
+    const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate); // Already a single date
+    const earliestSocialDate = getMinDate(socialDate); // Already a single date
+
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate,
+        earliestHistoryDate,
+        earliestInsightsDate,
+        earliestSocialDate
+    );
+
+    // Fallback date
+    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+
+    const result = {
+        portfolio: earliestPortfolioDate || new Date('2999-12-31'), // Use a 'far future' date if null
+        history: earliestHistoryDate || new Date('2999-12-31'),
+        insights: earliestInsightsDate || new Date('2999-12-31'),
+        social: earliestSocialDate || new Date('2999-12-31'),
+        absoluteEarliest: absoluteEarliest || fallbackDate // Use fallback for the main loop
+    };
+
+    logger.log('INFO', 'Earliest data availability map built:', {
+        portfolio: result.portfolio.toISOString().slice(0, 10),
+        history: result.history.toISOString().slice(0, 10),
+        insights: result.insights.toISOString().slice(0, 10),
+        social: result.social.toISOString().slice(0, 10),
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
+    });
+
+    return result;
 }
 
 module.exports = {
@@ -103,5 +181,6 @@ module.exports = {
     normalizeName,
     commitBatchInChunks,
     getExpectedDateStrings,
-    getFirstDateFromSourceData,
-};
+    // getFirstDateFromSourceData, // This is replaced
+    getEarliestDataDates, // <-- EXPORT NEW FUNCTION
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.130",
+  "version": "1.0.132",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [