bulltrackers-module 1.0.525 → 1.0.527

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,7 +1,7 @@
 /**
  * @fileoverview Checks availability of root data via the Root Data Index.
  * REFACTORED: Fully supports granular flags for PI, Signed-In Users, Rankings, and Verification.
- * FIXED: Added safety fallback for unknown userTypes to prevent impossible runs.
+ * FIXED: Removed permissive fallbacks to enforce strict userType availability rules.
  * UPDATED: Added strict social data checking for Popular Investors and Signed-In Users.
  */
 const { normalizeName } = require('../utils/utils');
@@ -29,7 +29,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
             else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
             else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
             else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
-            else if (rootDataStatus.hasPortfolio) isAvailable = true; // Fallback
+            // REMOVED: Unconditional fallback that bypassed strict userType checks
             
             if (!isAvailable) {
                 if (userType === 'speculator') missing.push('speculatorPortfolio');
@@ -47,7 +47,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
             else if (userType === 'popular_investor' && rootDataStatus.piHistory) isAvailable = true;
             else if (userType === 'signed_in_user' && rootDataStatus.signedInUserHistory) isAvailable = true;
             else if (userType === 'all' && rootDataStatus.hasHistory) isAvailable = true;
-            else if (rootDataStatus.hasHistory) isAvailable = true; // Fallback
+            // REMOVED: Unconditional fallback that bypassed strict userType checks
             
             if (!isAvailable) {
                 if (userType === 'speculator') missing.push('speculatorHistory');
@@ -85,14 +85,17 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         }
         // [UPDATED] Strict Social Data Checking
         else if (dep === 'social') {
-            if (userType === 'popular_investor' && rootDataStatus.hasPISocial) {
-                isAvailable = true;
-                available.push('social');
-            } else if (userType === 'signed_in_user' && rootDataStatus.hasSignedInSocial) {
-                isAvailable = true;
-                available.push('social');
-            } else if (rootDataStatus.hasSocial) {
-                isAvailable = true;
+            // Strictly check based on userType to prevent fallback to generic social data
+            if (userType === 'popular_investor') {
+                if (rootDataStatus.hasPISocial) isAvailable = true;
+            } else if (userType === 'signed_in_user') {
+                if (rootDataStatus.hasSignedInSocial) isAvailable = true;
+            } else {
+                // For 'all', 'normal', 'speculator' or undefined, accept generic social data
+                if (rootDataStatus.hasSocial) isAvailable = true;
+            }
+
+            if (isAvailable) {
                 available.push('social');
             } else {
                 if (userType === 'popular_investor') missing.push('piSocial');
@@ -144,10 +147,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     }
     
     // [FIX] Enforce canHaveMissingRoots logic:
-    // - If canHaveMissingRoots: false, ALL required rootdata must be available
-    // - If canHaveMissingRoots: true, AT LEAST ONE required rootdata must be available (not zero)
     if (canHaveMissingRoots) {
-        // Must have at least one available rootdata type
         const canRun = available.length > 0;
         return { canRun, missing, available };
     }
@@ -246,7 +246,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
                     piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
                     signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
                     hasPISocial: false, hasSignedInSocial: false,
-                    // New Root Data Types
                     piRatings: false, piPageViews: false, watchlistMembership: false, piAlertHistory: false
                 }
             };

package/functions/computation-system/executors/StandardExecutor.js

@@ -17,12 +17,22 @@ class StandardExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
         const dStr   = date.toISOString().slice(0, 10);
         const logger = deps.logger;
+
+        // [FIX] Determine required user types for this batch of calculations
+        const requiredUserTypes = new Set();
+        calcs.forEach(c => {
+            const type = (c.userType || 'ALL').toUpperCase();
+            requiredUserTypes.add(type);
+        });
+        // If any calc requires 'ALL' (or has no userType), we fetch everything
+        const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
             const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr = prev.toISOString().slice(0, 10);
-            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
+            // [FIX] Pass userTypeArray to filter yesterday's data loading
+            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
         }
 
         const state = {};
@@ -36,10 +46,11 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite);
+        return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, userTypeArray);
     }
 
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite) {
+    // [UPDATED] Accepts requiredUserTypes param
+    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
         const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
@@ -80,11 +91,17 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+        // [FIX] Pass requiredUserTypes to streamPortfolioData
+        // Note: portfolioRefs (for today) might be null if not pre-fetched, allowing streamPortfolioData to fetch filtered refs
+        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs, requiredUserTypes);
+        
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
+        // Yesterday's refs were already filtered in run(), so we pass them directly
         const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+        
         const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+        // [FIX] Pass requiredUserTypes to streamHistoryData
+        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs, requiredUserTypes) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         let usersSinceLastFlush = 0;
@@ -139,7 +156,8 @@ class StandardExecutor {
 
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
-
+    
+    // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
         // ... (No changes to flushBuffer)
         const transformedState = {};

package/functions/computation-system/utils/data_loader.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
+ * FIXED: Added strict userType filtering to prevent fetching unnecessary data (e.g. Normal users for PI calcs).
  * --- NEW: Updated to read PI/Signed-In data from SHARDS (Parts) instead of individual docs. ---
  * --- NEW: Logic to merge Overall and Deep PI data from corresponding shards. ---
  * --- UPDATED: Added loaders for Rankings and Verification data. ---
@@ -22,80 +23,87 @@ function tryDecompress(data) {
 
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
-/** Stage 1: Get portfolio part document references for a given date */
-async function getPortfolioPartRefs(config, deps, dateString) {
+/** * Stage 1: Get portfolio part document references for a given date 
+ * [UPDATED] Accepts requiredUserTypes to filter collections.
+ */
+async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes = null) {
     const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
-    const { getCollectionPath } = collectionRegistry || {};
     
-    logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
+    // Normalize required types. If null/empty or contains 'ALL', fetch everything.
+    const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
+    const fetchAll = !types || types.size === 0 || types.has('ALL');
+
+    logger.log('INFO', `Getting portfolio part references for date: ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
     
     const allPartRefs = [];
     
     // NEW STRUCTURE: Read from date-based collections (per-user documents)
-    // Structure: Collection/{date}/{cid}/{cid} where {date} is document, {cid} is subcollection, {cid} is document
     try {
-        // Signed-In User Portfolios: SignedInUserPortfolioData/{date}/{cid}/{cid}
-        const signedInPortCollectionName = 'SignedInUserPortfolioData';
-        const signedInPortDateDoc = db.collection(signedInPortCollectionName).doc(dateString);
-        const signedInPortSubcollections = await withRetry(
-            () => signedInPortDateDoc.listCollections(),
-            `listSignedInPortfolios(${dateString})`
-        );
-        
-        signedInPortSubcollections.forEach(subcol => {
-            // Each subcollection is a CID, the document ID is also the CID
-            const cid = subcol.id;
-            const cidDocRef = subcol.doc(cid);
-            allPartRefs.push({ 
-                ref: cidDocRef, 
-                type: 'SIGNED_IN_USER',
-                cid: cid,
-                collectionType: 'NEW_STRUCTURE'
+        // Signed-In User Portfolios
+        if (fetchAll || types.has('SIGNED_IN_USER')) {
+            const signedInPortCollectionName = 'SignedInUserPortfolioData';
+            const signedInPortDateDoc = db.collection(signedInPortCollectionName).doc(dateString);
+            const signedInPortSubcollections = await withRetry(
+                () => signedInPortDateDoc.listCollections(),
+                `listSignedInPortfolios(${dateString})`
+            );
+            
+            signedInPortSubcollections.forEach(subcol => {
+                const cid = subcol.id;
+                allPartRefs.push({ 
+                    ref: subcol.doc(cid), 
+                    type: 'SIGNED_IN_USER',
+                    cid: cid,
+                    collectionType: 'NEW_STRUCTURE'
+                });
             });
-        });
-        
-        // Popular Investor Portfolios: PopularInvestorPortfolioData/{date}/{cid}/{cid}
-        const piPortCollectionName = 'PopularInvestorPortfolioData';
-        const piPortDateDoc = db.collection(piPortCollectionName).doc(dateString);
-        const piPortSubcollections = await withRetry(
-            () => piPortDateDoc.listCollections(),
-            `listPIPortfolios(${dateString})`
-        );
+        }
         
-        piPortSubcollections.forEach(subcol => {
-            const cid = subcol.id;
-            const cidDocRef = subcol.doc(cid);
-            allPartRefs.push({ 
-                ref: cidDocRef, 
-                type: 'POPULAR_INVESTOR',
-                cid: cid,
-                collectionType: 'NEW_STRUCTURE'
+        // Popular Investor Portfolios
+        if (fetchAll || types.has('POPULAR_INVESTOR')) {
+            const piPortCollectionName = 'PopularInvestorPortfolioData';
+            const piPortDateDoc = db.collection(piPortCollectionName).doc(dateString);
+            const piPortSubcollections = await withRetry(
+                () => piPortDateDoc.listCollections(),
+                `listPIPortfolios(${dateString})`
+            );
+            
+            piPortSubcollections.forEach(subcol => {
+                const cid = subcol.id;
+                allPartRefs.push({ 
+                    ref: subcol.doc(cid), 
+                    type: 'POPULAR_INVESTOR',
+                    cid: cid,
+                    collectionType: 'NEW_STRUCTURE'
+                });
             });
-        });
-        
-        logger.log('INFO', `Found ${allPartRefs.length} portfolio refs from new structure for ${dateString}`);
+        }
     } catch (newStructError) {
         logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
     }
     
-    // LEGACY STRUCTURE: Read from block-based collections (for backward compatibility)
-    const collectionsToQuery = [
-        config.normalUserPortfolioCollection, 
-        config.speculatorPortfolioCollection,
-        config.piPortfolioCollection,        // Legacy: PI Overall
-        config.signedInUsersCollection       // Legacy: Signed-In Users
-    ].filter(Boolean);
+    // LEGACY STRUCTURE: Read from block-based collections
+    const collectionsToQuery = [];
+    
+    if ((fetchAll || types.has('NORMAL')) && config.normalUserPortfolioCollection) 
+        collectionsToQuery.push({ name: config.normalUserPortfolioCollection, type: 'NORMAL' });
+        
+    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorPortfolioCollection) 
+        collectionsToQuery.push({ name: config.speculatorPortfolioCollection, type: 'SPECULATOR' });
+        
+    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piPortfolioCollection) 
+        collectionsToQuery.push({ name: config.piPortfolioCollection, type: 'POPULAR_INVESTOR' });
+        
+    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInUsersCollection) 
+        collectionsToQuery.push({ name: config.signedInUsersCollection, type: 'SIGNED_IN_USER' });
 
-    for (const collectionName of collectionsToQuery) {
+    for (const { name: collectionName, type: collectionType } of collectionsToQuery) {
         try {
-            // Assume standard structure: Collection -> Block(e.g. 19M) -> snapshots -> date -> parts
             const blockDocsQuery = db.collection(collectionName);
             const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
             
-            if (!blockDocRefs.length) { 
-                continue; 
-            }
+            if (!blockDocRefs.length) continue; 
 
             const partsPromises = blockDocRefs.map(blockDocRef => { 
                 const partsCollectionRef = blockDocRef
@@ -108,14 +116,9 @@ async function getPortfolioPartRefs(config, deps, dateString) {
             const partDocArrays = await Promise.all(partsPromises);
             
             partDocArrays.forEach(partDocs => { 
-                // Tag them so loadDataByRefs knows how to handle them
-                let type = 'PART';
-                if (collectionName === config.piPortfolioCollection) type = 'POPULAR_INVESTOR';
-                if (collectionName === config.signedInUsersCollection) type = 'SIGNED_IN_USER';
-                
                 allPartRefs.push(...partDocs.map(ref => ({ 
                     ref, 
-                    type,
+                    type: collectionType,
                     collectionType: 'LEGACY'
                 }))); 
             });
@@ -144,9 +147,6 @@ async function loadDataByRefs(config, deps, refObjects) {
         
         const snapshots = await withRetry(() => db.getAll(...refs), `getAll(batch ${Math.floor(i/batchSize)})`);
         
-        // --- Special Handling for PI Deep Data ---
-        // If we loaded a PI Overall Shard, we try to load the matching PI Deep Shard.
-        // Logic: Same Block, Same Date, Same Part ID.
         const deepFetchPromises = [];
 
         for (let j = 0; j < snapshots.length; j++) {
@@ -158,18 +158,13 @@ async function loadDataByRefs(config, deps, refObjects) {
             const rawData = doc.data();
             let chunkData;
             
-            // NEW STRUCTURE: Single user document per CID
             if (meta.collectionType === 'NEW_STRUCTURE') {
                 const cid = meta.cid || doc.id;
-                // Data is stored directly in the document, not as a map
                 const userData = tryDecompress(rawData);
-                // Convert to map format: { cid: data }
                 chunkData = { [cid]: userData };
                 
-                // Tag user type
                 if (meta.type === 'POPULAR_INVESTOR') {
                     chunkData[cid]._userType = 'POPULAR_INVESTOR';
-                    // Check for deep positions in the same document
                     if (chunkData[cid].deepPositions) {
                         chunkData[cid].DeepPositions = chunkData[cid].deepPositions;
                     }
@@ -179,40 +174,28 @@ async function loadDataByRefs(config, deps, refObjects) {
                 
                 deepFetchPromises.push(Promise.resolve(chunkData));
             } else {
-                // LEGACY STRUCTURE: Sharded parts with multiple users per document
-                chunkData = tryDecompress(rawData); // Map: { userId: data }
+                chunkData = tryDecompress(rawData); 
 
                 if (meta.type === 'POPULAR_INVESTOR' && config.piDeepPortfolioCollection) {
-                    // Construct Deep Path
-                    // Current: pi_portfolios_overall/19M/snapshots/{date}/parts/{part_X}
-                    // Target:  pi_portfolios_deep/19M/snapshots/{date}/parts/{part_X}
-                    
-                    const pathSegments = doc.ref.path.split('/'); // [col, block, snap, date, parts, partId]
-                    // Replace collection name with deep collection name
+                    const pathSegments = doc.ref.path.split('/'); 
                     const deepCollection = config.piDeepPortfolioCollection;
                     const deepPath = `${deepCollection}/${pathSegments[1]}/${pathSegments[2]}/${pathSegments[3]}/${pathSegments[4]}/${pathSegments[5]}`;
                     
-                    // Fetch deeply
                     deepFetchPromises.push(
                         db.doc(deepPath).get().then(deepSnap => {
                             if (deepSnap.exists) {
                                 const deepChunk = tryDecompress(deepSnap.data());
-                                // Merge deep positions into overall data
                                 for (const [uid, pData] of Object.entries(chunkData)) {
                                     if (deepChunk[uid] && deepChunk[uid].positions) {
                                         pData.DeepPositions = deepChunk[uid].positions;
                                     }
                                 }
                             }
-                            // Tag internal type for ContextFactory
                             for (const pData of Object.values(chunkData)) {
                                 pData._userType = 'POPULAR_INVESTOR';
                             }
                             return chunkData;
-                        }).catch(err => {
-                            // If deep fetch fails, return chunkData as is (graceful degradation)
-                            return chunkData;
-                        })
+                        }).catch(() => chunkData)
                     );
                 } else if (meta.type === 'SIGNED_IN_USER') {
                     for (const pData of Object.values(chunkData)) {
@@ -220,14 +203,12 @@ async function loadDataByRefs(config, deps, refObjects) {
                     }
                     deepFetchPromises.push(Promise.resolve(chunkData));
                 } else {
-                    // Standard Part
                     deepFetchPromises.push(Promise.resolve(chunkData));
                 }
             }
         }
 
         const resolvedChunks = await Promise.all(deepFetchPromises);
-        
         resolvedChunks.forEach(chunk => {
              if (chunk && typeof chunk === 'object') {
                 Object.assign(mergedPortfolios, chunk);
@@ -403,76 +384,76 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     return result;
 }
 
-/** Stage 6: Get history part references for a given date */
-async function getHistoryPartRefs(config, deps, dateString) {
+/** * Stage 6: Get history part references for a given date 
+ * [UPDATED] Accepts requiredUserTypes to filter collections.
+ */
+async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes = null) {
     const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
-    const { getCollectionPath } = collectionRegistry || {};
     
-    logger.log('INFO', `Getting history part references for ${dateString}`);
+    // Normalize required types
+    const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
+    const fetchAll = !types || types.size === 0 || types.has('ALL');
+
+    logger.log('INFO', `Getting history part references for ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
     
     const allPartRefs = [];
     
-    // NEW STRUCTURE: Read from date-based collections
-    // Structure: Collection/{date}/{cid}/{cid}
+    // NEW STRUCTURE
     try {
-        // Signed-In User History: SignedInUserTradeHistoryData/{date}/{cid}/{cid}
-        const signedInHistCollectionName = 'SignedInUserTradeHistoryData';
-        const signedInHistDateDoc = db.collection(signedInHistCollectionName).doc(dateString);
-        const signedInHistSubcollections = await withRetry(
-            () => signedInHistDateDoc.listCollections(),
-            `listSignedInHistory(${dateString})`
-        );
-        
-        signedInHistSubcollections.forEach(subcol => {
-            const cid = subcol.id;
-            const cidDocRef = subcol.doc(cid);
-            allPartRefs.push({ 
-                ref: cidDocRef, 
-                type: 'SIGNED_IN_USER',
-                cid: cid,
-                collectionType: 'NEW_STRUCTURE'
+        if (fetchAll || types.has('SIGNED_IN_USER')) {
+            const signedInHistCollectionName = 'SignedInUserTradeHistoryData';
+            const signedInHistDateDoc = db.collection(signedInHistCollectionName).doc(dateString);
+            const signedInHistSubcollections = await withRetry(
+                () => signedInHistDateDoc.listCollections(),
+                `listSignedInHistory(${dateString})`
+            );
+            signedInHistSubcollections.forEach(subcol => {
+                allPartRefs.push({ 
+                    ref: subcol.doc(subcol.id), 
+                    type: 'SIGNED_IN_USER',
+                    cid: subcol.id,
+                    collectionType: 'NEW_STRUCTURE'
+                });
             });
-        });
-        
-        // Popular Investor History: PopularInvestorTradeHistoryData/{date}/{cid}/{cid}
-        const piHistCollectionName = 'PopularInvestorTradeHistoryData';
-        const piHistDateDoc = db.collection(piHistCollectionName).doc(dateString);
-        const piHistSubcollections = await withRetry(
-            () => piHistDateDoc.listCollections(),
-            `listPIHistory(${dateString})`
-        );
+        }
         
-        piHistSubcollections.forEach(subcol => {
-            const cid = subcol.id;
-            const cidDocRef = subcol.doc(cid);
-            allPartRefs.push({ 
-                ref: cidDocRef, 
-                type: 'POPULAR_INVESTOR',
-                cid: cid,
-                collectionType: 'NEW_STRUCTURE'
+        if (fetchAll || types.has('POPULAR_INVESTOR')) {
+            const piHistCollectionName = 'PopularInvestorTradeHistoryData';
+            const piHistDateDoc = db.collection(piHistCollectionName).doc(dateString);
+            const piHistSubcollections = await withRetry(
+                () => piHistDateDoc.listCollections(),
+                `listPIHistory(${dateString})`
+            );
+            piHistSubcollections.forEach(subcol => {
+                allPartRefs.push({ 
+                    ref: subcol.doc(subcol.id), 
+                    type: 'POPULAR_INVESTOR',
+                    cid: subcol.id,
+                    collectionType: 'NEW_STRUCTURE'
+                });
             });
-        });
-        
-        logger.log('INFO', `Found ${allPartRefs.length} history refs from new structure for ${dateString}`);
+        }
     } catch (newStructError) {
-        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
+        logger.log('WARN', `Failed to load from new structure: ${newStructError.message}`);
     }
     
-    // LEGACY STRUCTURE: Read from block-based collections
-    const collectionsToQuery = [
-        config.normalUserHistoryCollection, 
-        config.speculatorHistoryCollection,
-        config.piHistoryCollection,
-        config.signedInHistoryCollection
-    ].filter(Boolean);
+    // LEGACY STRUCTURE
+    const collectionsToQuery = [];
+    if ((fetchAll || types.has('NORMAL')) && config.normalUserHistoryCollection) 
+        collectionsToQuery.push(config.normalUserHistoryCollection);
+    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorHistoryCollection) 
+        collectionsToQuery.push(config.speculatorHistoryCollection);
+    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piHistoryCollection) 
+        collectionsToQuery.push(config.piHistoryCollection);
+    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInHistoryCollection) 
+        collectionsToQuery.push(config.signedInHistoryCollection);
 
     for (const collectionName of collectionsToQuery) {
         try {
             const blockDocsQuery = db.collection(collectionName);
             const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-            
-            if (!blockDocRefs.length) { continue; }
+            if (!blockDocRefs.length) continue;
             
             const partsPromises = blockDocRefs.map(blockDocRef => { 
                 const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots')
@@ -482,7 +463,6 @@ async function getHistoryPartRefs(config, deps, dateString) {
             
             const partDocArrays = await Promise.all(partsPromises);
             partDocArrays.forEach(partDocs => { 
-                // History parts are standard, no deep merge needed usually
                 allPartRefs.push(...partDocs.map(ref => ({ 
                     ref, 
                     type: 'PART',
@@ -498,10 +478,12 @@ async function getHistoryPartRefs(config, deps, dateString) {
     return allPartRefs;
 }
 
-/** Stage 7: Stream portfolio data in chunks */
-async function* streamPortfolioData(config, deps, dateString, providedRefs = null) {
+/** * Stage 7: Stream portfolio data in chunks 
+ * [UPDATED] Passes requiredUserTypes to getPortfolioPartRefs
+ */
+async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
-    const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
+    const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
     
     const batchSize = config.partRefBatchSize || 10; 
@@ -515,10 +497,12 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
 
-/** Stage 8: Stream history data in chunks */
-async function* streamHistoryData(config, deps, dateString, providedRefs = null) {
+/** * Stage 8: Stream history data in chunks 
+ * [UPDATED] Passes requiredUserTypes to getHistoryPartRefs
+ */
+async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
-    const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
+    const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString, requiredUserTypes));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
     
     const batchSize = config.partRefBatchSize || 10;