bulltrackers-module 1.0.497 → 1.0.499

package/functions/computation-system/utils/data_loader.js

@@ -24,52 +24,107 @@ function tryDecompress(data) {
 
 /** Stage 1: Get portfolio part document references for a given date */
 async function getPortfolioPartRefs(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
+    const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
+    const { getCollectionPath } = collectionRegistry || {};
     
     logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
     
-    // We scan ALL portfolio collections that use the standard 'snapshots/{date}/parts' structure
+    const allPartRefs = [];
+    
+    // NEW STRUCTURE: Read from date-based collections (per-user documents)
+    // Structure: Collection/{date}/{cid}/{cid} where {date} is document, {cid} is subcollection, {cid} is document
+    try {
+        // Signed-In User Portfolios: SignedInUserPortfolioData/{date}/{cid}/{cid}
+        const signedInPortCollectionName = 'SignedInUserPortfolioData';
+        const signedInPortDateDoc = db.collection(signedInPortCollectionName).doc(dateString);
+        const signedInPortSubcollections = await withRetry(
+            () => signedInPortDateDoc.listCollections(),
+            `listSignedInPortfolios(${dateString})`
+        );
+        
+        signedInPortSubcollections.forEach(subcol => {
+            // Each subcollection is a CID, the document ID is also the CID
+            const cid = subcol.id;
+            const cidDocRef = subcol.doc(cid);
+            allPartRefs.push({ 
+                ref: cidDocRef, 
+                type: 'SIGNED_IN_USER',
+                cid: cid,
+                collectionType: 'NEW_STRUCTURE'
+            });
+        });
+        
+        // Popular Investor Portfolios: PopularInvestorPortfolioData/{date}/{cid}/{cid}
+        const piPortCollectionName = 'PopularInvestorPortfolioData';
+        const piPortDateDoc = db.collection(piPortCollectionName).doc(dateString);
+        const piPortSubcollections = await withRetry(
+            () => piPortDateDoc.listCollections(),
+            `listPIPortfolios(${dateString})`
+        );
+        
+        piPortSubcollections.forEach(subcol => {
+            const cid = subcol.id;
+            const cidDocRef = subcol.doc(cid);
+            allPartRefs.push({ 
+                ref: cidDocRef, 
+                type: 'POPULAR_INVESTOR',
+                cid: cid,
+                collectionType: 'NEW_STRUCTURE'
+            });
+        });
+        
+        logger.log('INFO', `Found ${allPartRefs.length} portfolio refs from new structure for ${dateString}`);
+    } catch (newStructError) {
+        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
+    }
+    
+    // LEGACY STRUCTURE: Read from block-based collections (for backward compatibility)
     const collectionsToQuery = [
         config.normalUserPortfolioCollection, 
         config.speculatorPortfolioCollection,
-        config.piPortfolioCollection,        // New: PI Overall
-        config.signedInUsersCollection       // New: Signed-In Users
+        config.piPortfolioCollection,        // Legacy: PI Overall
+        config.signedInUsersCollection       // Legacy: Signed-In Users
     ].filter(Boolean);
 
-    const allPartRefs = [];
-    
     for (const collectionName of collectionsToQuery) {
-        // Assume standard structure: Collection -> Block(e.g. 19M) -> snapshots -> date -> parts
-        // List documents to find the Blocks
-        const blockDocsQuery = db.collection(collectionName);
-        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-        
-        if (!blockDocRefs.length) { 
-            continue; 
-        }
+        try {
+            // Assume standard structure: Collection -> Block(e.g. 19M) -> snapshots -> date -> parts
+            const blockDocsQuery = db.collection(collectionName);
+            const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+            
+            if (!blockDocRefs.length) { 
+                continue; 
+            }
 
-        const partsPromises = blockDocRefs.map(blockDocRef => { 
-            const partsCollectionRef = blockDocRef
-                .collection(config.snapshotsSubcollection || 'snapshots')
-                .doc(dateString)
-                .collection(config.partsSubcollection || 'parts'); 
-            return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
-        });
+            const partsPromises = blockDocRefs.map(blockDocRef => { 
+                const partsCollectionRef = blockDocRef
+                    .collection(config.snapshotsSubcollection || 'snapshots')
+                    .doc(dateString)
+                    .collection(config.partsSubcollection || 'parts'); 
+                return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
+            });
 
-        const partDocArrays = await Promise.all(partsPromises);
-        
-        partDocArrays.forEach(partDocs => { 
-            // Tag them so loadDataByRefs knows how to handle them (especially PI Deep Fetch)
-            let type = 'PART';
-            if (collectionName === config.piPortfolioCollection) type = 'POPULAR_INVESTOR';
-            if (collectionName === config.signedInUsersCollection) type = 'SIGNED_IN_USER';
+            const partDocArrays = await Promise.all(partsPromises);
             
-            allPartRefs.push(...partDocs.map(ref => ({ ref, type }))); 
-        }); 
+            partDocArrays.forEach(partDocs => { 
+                // Tag them so loadDataByRefs knows how to handle them
+                let type = 'PART';
+                if (collectionName === config.piPortfolioCollection) type = 'POPULAR_INVESTOR';
+                if (collectionName === config.signedInUsersCollection) type = 'SIGNED_IN_USER';
+                
+                allPartRefs.push(...partDocs.map(ref => ({ 
+                    ref, 
+                    type,
+                    collectionType: 'LEGACY'
+                }))); 
+            });
+        } catch (legacyError) {
+            logger.log('WARN', `Failed to load legacy collection ${collectionName}: ${legacyError.message}`);
+        }
     }
     
-    logger.log('INFO', `Found ${allPartRefs.length} portfolio part refs for ${dateString}`);
+    logger.log('INFO', `Found ${allPartRefs.length} total portfolio refs for ${dateString}`);
     return allPartRefs;
 }
 
@@ -101,48 +156,73 @@ async function loadDataByRefs(config, deps, refObjects) {
             if (!doc.exists) continue;
 
             const rawData = doc.data();
-            const chunkData = tryDecompress(rawData); // Map: { userId: data }
-
-            if (meta.type === 'POPULAR_INVESTOR' && config.piDeepPortfolioCollection) {
-                // Construct Deep Path
-                // Current: pi_portfolios_overall/19M/snapshots/{date}/parts/{part_X}
-                // Target:  pi_portfolios_deep/19M/snapshots/{date}/parts/{part_X}
+            let chunkData;
+            
+            // NEW STRUCTURE: Single user document per CID
+            if (meta.collectionType === 'NEW_STRUCTURE') {
+                const cid = meta.cid || doc.id;
+                // Data is stored directly in the document, not as a map
+                const userData = tryDecompress(rawData);
+                // Convert to map format: { cid: data }
+                chunkData = { [cid]: userData };
                 
-                const pathSegments = doc.ref.path.split('/'); // [col, block, snap, date, parts, partId]
-                // Replace collection name with deep collection name
-                const deepCollection = config.piDeepPortfolioCollection;
-                const deepPath = `${deepCollection}/${pathSegments[1]}/${pathSegments[2]}/${pathSegments[3]}/${pathSegments[4]}/${pathSegments[5]}`;
+                // Tag user type
+                if (meta.type === 'POPULAR_INVESTOR') {
+                    chunkData[cid]._userType = 'POPULAR_INVESTOR';
+                    // Check for deep positions in the same document
+                    if (chunkData[cid].deepPositions) {
+                        chunkData[cid].DeepPositions = chunkData[cid].deepPositions;
+                    }
+                } else if (meta.type === 'SIGNED_IN_USER') {
+                    chunkData[cid]._userType = 'SIGNED_IN_USER';
+                }
                 
-                // Fetch deeply
-                deepFetchPromises.push(
-                    db.doc(deepPath).get().then(deepSnap => {
-                        if (deepSnap.exists) {
-                            const deepChunk = tryDecompress(deepSnap.data());
-                            // Merge deep positions into overall data
-                            for (const [uid, pData] of Object.entries(chunkData)) {
-                                if (deepChunk[uid] && deepChunk[uid].positions) {
-                                    pData.DeepPositions = deepChunk[uid].positions;
+                deepFetchPromises.push(Promise.resolve(chunkData));
+            } else {
+                // LEGACY STRUCTURE: Sharded parts with multiple users per document
+                chunkData = tryDecompress(rawData); // Map: { userId: data }
+
+                if (meta.type === 'POPULAR_INVESTOR' && config.piDeepPortfolioCollection) {
+                    // Construct Deep Path
+                    // Current: pi_portfolios_overall/19M/snapshots/{date}/parts/{part_X}
+                    // Target:  pi_portfolios_deep/19M/snapshots/{date}/parts/{part_X}
+                    
+                    const pathSegments = doc.ref.path.split('/'); // [col, block, snap, date, parts, partId]
+                    // Replace collection name with deep collection name
+                    const deepCollection = config.piDeepPortfolioCollection;
+                    const deepPath = `${deepCollection}/${pathSegments[1]}/${pathSegments[2]}/${pathSegments[3]}/${pathSegments[4]}/${pathSegments[5]}`;
+                    
+                    // Fetch deeply
+                    deepFetchPromises.push(
+                        db.doc(deepPath).get().then(deepSnap => {
+                            if (deepSnap.exists) {
+                                const deepChunk = tryDecompress(deepSnap.data());
+                                // Merge deep positions into overall data
+                                for (const [uid, pData] of Object.entries(chunkData)) {
+                                    if (deepChunk[uid] && deepChunk[uid].positions) {
+                                        pData.DeepPositions = deepChunk[uid].positions;
+                                    }
                                 }
                             }
-                        }
-                        // Tag internal type for ContextFactory
-                        for (const pData of Object.values(chunkData)) {
-                            pData._userType = 'POPULAR_INVESTOR';
-                        }
-                        return chunkData;
-                    }).catch(err => {
-                        // If deep fetch fails, return chunkData as is (graceful degradation)
-                        return chunkData;
-                    })
-                );
-            } else if (meta.type === 'SIGNED_IN_USER') {
-                for (const pData of Object.values(chunkData)) {
-                    pData._userType = 'SIGNED_IN_USER';
+                            // Tag internal type for ContextFactory
+                            for (const pData of Object.values(chunkData)) {
+                                pData._userType = 'POPULAR_INVESTOR';
+                            }
+                            return chunkData;
+                        }).catch(err => {
+                            // If deep fetch fails, return chunkData as is (graceful degradation)
+                            return chunkData;
+                        })
+                    );
+                } else if (meta.type === 'SIGNED_IN_USER') {
+                    for (const pData of Object.values(chunkData)) {
+                        pData._userType = 'SIGNED_IN_USER';
+                    }
+                    deepFetchPromises.push(Promise.resolve(chunkData));
+                } else {
+                    // Standard Part
+                    deepFetchPromises.push(Promise.resolve(chunkData));
                 }
-                deepFetchPromises.push(Promise.resolve(chunkData));
-            } else {
-                // Standard Part
-                deepFetchPromises.push(Promise.resolve(chunkData));
             }
         }
 
@@ -187,8 +267,9 @@ async function loadDailyInsights(config, deps, dateString) {
 
 /** Stage 5: Load and Partition Social Data */
 async function loadDailySocialPostInsights(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
+    const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
+    const { getCollectionPath } = collectionRegistry || {};
     
     logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
     
@@ -199,6 +280,70 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
         signedIn: {}            // Map<UserId, Map<PostId, Data>> - For Signed-In Users
     };
 
+    // NEW STRUCTURE: Read from date-based collections
+    // Structure: Collection/{date}/{cid}/{cid} for user social, Collection/{date}/posts/{postId} for instrument
+    try {
+        // Signed-In User Social: SignedInUserSocialPostData/{date}/{cid}/{cid}
+        const signedInSocialCollectionName = 'SignedInUserSocialPostData';
+        const signedInSocialDateDoc = db.collection(signedInSocialCollectionName).doc(dateString);
+        const signedInSocialSubcollections = await withRetry(
+            () => signedInSocialDateDoc.listCollections(),
+            `listSignedInSocial(${dateString})`
+        );
+        
+        for (const subcol of signedInSocialSubcollections) {
+            const cid = subcol.id;
+            const cidDoc = await subcol.doc(cid).get();
+            if (cidDoc.exists) {
+                const cidData = tryDecompress(cidDoc.data());
+                if (cidData.posts && typeof cidData.posts === 'object') {
+                    if (!result.signedIn[cid]) result.signedIn[cid] = {};
+                    // Posts are stored as a map in the document
+                    Object.assign(result.signedIn[cid], cidData.posts);
+                }
+            }
+        }
+        
+        // Popular Investor Social: PopularInvestorSocialPostData/{date}/{cid}/{cid}
+        const piSocialCollectionName = 'PopularInvestorSocialPostData';
+        const piSocialDateDoc = db.collection(piSocialCollectionName).doc(dateString);
+        const piSocialSubcollections = await withRetry(
+            () => piSocialDateDoc.listCollections(),
+            `listPISocial(${dateString})`
+        );
+        
+        for (const subcol of piSocialSubcollections) {
+            const cid = subcol.id;
+            const cidDoc = await subcol.doc(cid).get();
+            if (cidDoc.exists) {
+                const cidData = tryDecompress(cidDoc.data());
+                if (cidData.posts && typeof cidData.posts === 'object') {
+                    if (!result.pi[cid]) result.pi[cid] = {};
+                    Object.assign(result.pi[cid], cidData.posts);
+                }
+            }
+        }
+        
+        // Instrument Social: InstrumentFeedSocialPostData/{date}/posts/{postId}
+        const instrumentSocialCollectionName = 'InstrumentFeedSocialPostData';
+        const instrumentSocialDateDoc = db.collection(instrumentSocialCollectionName).doc(dateString);
+        const instrumentSocialPostsCol = instrumentSocialDateDoc.collection('posts');
+        const instrumentSocialSnapshot = await withRetry(
+            () => instrumentSocialPostsCol.limit(1000).get(),
+            `getInstrumentSocial(${dateString})`
+        );
+        
+        instrumentSocialSnapshot.forEach(doc => {
+            const data = tryDecompress(doc.data());
+            result.generic[doc.id] = data;
+        });
+        
+        logger.log('INFO', `Loaded Social Data (NEW): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
+    } catch (newStructError) {
+        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
+    }
+
+    // LEGACY STRUCTURE: CollectionGroup query (for backward compatibility)
     const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
     const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
 
@@ -246,7 +391,7 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     result.generic[doc.id] = data;
                 }
             });
-            logger.log('INFO', `Loaded Social Data: ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
+            logger.log('INFO', `Loaded Social Data (LEGACY): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
         } else {
             logger.log('WARN', `No social posts found for ${dateString} via CollectionGroup.`);
         }
@@ -260,39 +405,96 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
 
 /** Stage 6: Get history part references for a given date */
 async function getHistoryPartRefs(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
+    const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
+    const { getCollectionPath } = collectionRegistry || {};
+    
     logger.log('INFO', `Getting history part references for ${dateString}`);
     
-    // Include PI History
+    const allPartRefs = [];
+    
+    // NEW STRUCTURE: Read from date-based collections
+    // Structure: Collection/{date}/{cid}/{cid}
+    try {
+        // Signed-In User History: SignedInUserTradeHistoryData/{date}/{cid}/{cid}
+        const signedInHistCollectionName = 'SignedInUserTradeHistoryData';
+        const signedInHistDateDoc = db.collection(signedInHistCollectionName).doc(dateString);
+        const signedInHistSubcollections = await withRetry(
+            () => signedInHistDateDoc.listCollections(),
+            `listSignedInHistory(${dateString})`
+        );
+        
+        signedInHistSubcollections.forEach(subcol => {
+            const cid = subcol.id;
+            const cidDocRef = subcol.doc(cid);
+            allPartRefs.push({ 
+                ref: cidDocRef, 
+                type: 'SIGNED_IN_USER',
+                cid: cid,
+                collectionType: 'NEW_STRUCTURE'
+            });
+        });
+        
+        // Popular Investor History: PopularInvestorTradeHistoryData/{date}/{cid}/{cid}
+        const piHistCollectionName = 'PopularInvestorTradeHistoryData';
+        const piHistDateDoc = db.collection(piHistCollectionName).doc(dateString);
+        const piHistSubcollections = await withRetry(
+            () => piHistDateDoc.listCollections(),
+            `listPIHistory(${dateString})`
+        );
+        
+        piHistSubcollections.forEach(subcol => {
+            const cid = subcol.id;
+            const cidDocRef = subcol.doc(cid);
+            allPartRefs.push({ 
+                ref: cidDocRef, 
+                type: 'POPULAR_INVESTOR',
+                cid: cid,
+                collectionType: 'NEW_STRUCTURE'
+            });
+        });
+        
+        logger.log('INFO', `Found ${allPartRefs.length} history refs from new structure for ${dateString}`);
+    } catch (newStructError) {
+        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
+    }
+    
+    // LEGACY STRUCTURE: Read from block-based collections
     const collectionsToQuery = [
         config.normalUserHistoryCollection, 
         config.speculatorHistoryCollection,
         config.piHistoryCollection,
-        config.signedInHistoryCollection // if distinct
+        config.signedInHistoryCollection
     ].filter(Boolean);
 
-    const allPartRefs = [];
-    
     for (const collectionName of collectionsToQuery) {
-        const blockDocsQuery = db.collection(collectionName);
-        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-        
-        if (!blockDocRefs.length) { continue; }
-        
-        const partsPromises = blockDocRefs.map(blockDocRef => { 
-            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots')
-                .doc(dateString).collection(config.partsSubcollection || 'parts');
-            return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
-        });
-        
-        const partDocArrays = await Promise.all(partsPromises);
-        partDocArrays.forEach(partDocs => { 
-            // History parts are standard, no deep merge needed usually
-            allPartRefs.push(...partDocs.map(ref => ({ ref, type: 'PART' }))); 
-        }); 
+        try {
+            const blockDocsQuery = db.collection(collectionName);
+            const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+            
+            if (!blockDocRefs.length) { continue; }
+            
+            const partsPromises = blockDocRefs.map(blockDocRef => { 
+                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots')
+                    .doc(dateString).collection(config.partsSubcollection || 'parts');
+                return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
+            });
+            
+            const partDocArrays = await Promise.all(partsPromises);
+            partDocArrays.forEach(partDocs => { 
+                // History parts are standard, no deep merge needed usually
+                allPartRefs.push(...partDocs.map(ref => ({ 
+                    ref, 
+                    type: 'PART',
+                    collectionType: 'LEGACY'
+                }))); 
+            });
+        } catch (legacyError) {
+            logger.log('WARN', `Failed to load legacy history collection ${collectionName}: ${legacyError.message}`);
+        }
     }
-    logger.log('INFO', `Found ${allPartRefs.length} history part refs for ${dateString}`);
+    
+    logger.log('INFO', `Found ${allPartRefs.length} total history refs for ${dateString}`);
     return allPartRefs;
 }
 

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -16,11 +16,29 @@ const SHARD_SIZE = 40;
  * @returns {Promise<{success: boolean, message: string, instrumentsProcessed?: number}>}
  */
 exports.fetchAndStorePrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
+    const { db, logger, headerManager, proxyManager, collectionRegistry } = dependencies;
     logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
     let selectedHeader = null;
     let wasSuccessful = false;
-    const priceCollectionName =  'asset_prices';
+    
+    // Get collection names from registry if available, fallback to hardcoded
+    const { getCollectionPath } = collectionRegistry || {};
+    let priceCollectionName = 'asset_prices';
+    let priceTrackingCollectionName = 'pricedatastoreddates';
+    
+    if (getCollectionPath) {
+        try {
+            // Extract collection name from registry path: asset_prices/shard_{shardId}
+            const basePath = getCollectionPath('rootData', 'assetPrices', { shardId: '0' });
+            priceCollectionName = basePath.split('/')[0];
+            
+            // Extract price tracking collection name
+            const trackingPath = getCollectionPath('rootData', 'priceTracking', { fetchDate: '2025-01-01' });
+            priceTrackingCollectionName = trackingPath.split('/')[0];
+        } catch (e) {
+            logger.log('WARN', `[PriceFetcherHelpers] Failed to get collections from registry, using defaults: ${e.message}`);
+        }
+    }
     try { if (!config.etoroApiUrl) { throw new Error("Missing required configuration: etoroApiUrl."); }
         selectedHeader = await headerManager.selectHeader();
         if (!selectedHeader || !selectedHeader.header) { throw new Error("Could not select a valid header for the request."); }
@@ -81,7 +99,7 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         
         // Write date tracking document
         const today = new Date().toISOString().split('T')[0];
-        const dateTrackingRef = db.collection('pricedatastoreddates').doc(today);
+        const dateTrackingRef = db.collection(priceTrackingCollectionName).doc(today);
         const priceDatesArray = Array.from(priceDatesSet).sort();
         
         await dateTrackingRef.set({

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -13,12 +13,27 @@ const zlib = require('zlib'); // [NEW] Required for compression
  * @returns {Promise<{success: boolean, message: string, instrumentCount?: number}>}
  */
 exports.fetchAndStoreInsights = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
+    const { db, logger, headerManager, proxyManager, collectionRegistry } = dependencies;
     logger.log('INFO', '[FetchInsightsHelpers] Starting eToro insights data fetch...');
     let selectedHeader = null; let wasSuccessful = false;
     
+    // Get collection name from registry if available, fallback to config
+    const { getCollectionPath } = collectionRegistry || {};
+    let insightsCollectionName = config.insightsCollectionName;
+    
+    if (getCollectionPath) {
+        try {
+            // Extract collection name from registry path: daily_instrument_insights/{date}
+            const basePath = getCollectionPath('rootData', 'instrumentInsights', { date: '2025-01-01' });
+            // Path is like "daily_instrument_insights/2025-01-01", extract collection name
+            insightsCollectionName = basePath.split('/')[0];
+        } catch (e) {
+            logger.log('WARN', `[FetchInsightsHelpers] Failed to get collection from registry, using config: ${e.message}`);
+        }
+    }
+    
     try {
-        if (!config.etoroInsightsUrl || !config.insightsCollectionName) { 
+        if (!config.etoroInsightsUrl || !insightsCollectionName) { 
             throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName."); 
         }
         
@@ -71,7 +86,7 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
         }
         
         const today = new Date().toISOString().slice(0, 10);
-        const docRef = db.collection(config.insightsCollectionName).doc(today);
+        const docRef = db.collection(insightsCollectionName).doc(today);
 
         // [FIX] --- COMPRESSION LOGIC START ---
         
@@ -130,7 +145,7 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
                     ...config.rootDataIndexer,
                     collections: {
                         ...config.rootDataIndexer.collections,
-                        insights: config.insightsCollectionName // Override with actual collection name used
+                        insights: insightsCollectionName // Override with actual collection name used
                     },
                     targetDate: today // Index only today's date for speed
                 };

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -16,10 +16,25 @@ const { IntelligentHeaderManager } = require('../../core/utils/intelligent_heade
  * @param {object} config.headerConfig - Configuration for the IntelligentHeaderManager.
  */
 async function fetchAndStorePopularInvestors(config, dependencies) {
-    const { db, logger } = dependencies;
+    const { db, logger, collectionRegistry } = dependencies;
     const { rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig } = config;
 
-    if (!rankingsApiUrl || !rankingsCollectionName || !proxyConfig || !headerConfig) {
+    // Get collection name from registry if available, fallback to config
+    const { getCollectionPath } = collectionRegistry || {};
+    let finalRankingsCollectionName = rankingsCollectionName;
+    
+    if (getCollectionPath) {
+        try {
+            // Extract collection name from registry path: popular_investor_rankings/{date}
+            const basePath = getCollectionPath('rootData', 'popularInvestorRankings', { date: '2025-01-01' });
+            // Path is like "popular_investor_rankings/2025-01-01", extract collection name
+            finalRankingsCollectionName = basePath.split('/')[0];
+        } catch (e) {
+            logger.log('WARN', `[PopularInvestorFetch] Failed to get collection from registry, using config: ${e.message}`);
+        }
+    }
+
+    if (!rankingsApiUrl || !finalRankingsCollectionName || !proxyConfig || !headerConfig) {
         throw new Error("[PopularInvestorFetch] Missing required config (rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig).");
     }
 
@@ -103,7 +118,7 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
     // 6. Final Validation & Storage
     if (data && data.Items && Array.isArray(data.Items)) {
         try {
-            const docRef = db.collection(rankingsCollectionName).doc(today);
+            const docRef = db.collection(finalRankingsCollectionName).doc(today);
             
             await docRef.set({
                 fetchedAt: new Date(),
@@ -112,7 +127,7 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
                 ...data
             });
 
-            logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${rankingsCollectionName}/${today}`);
+            logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}`);
             
             // Update root data indexer for today's date after rankings data is stored
             try {
@@ -132,7 +147,7 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
                         ...rootDataIndexerConfig,
                         collections: {
                             ...rootDataIndexerConfig.collections,
-                            piRankings: rankingsCollectionName // Override with actual collection name used
+                            piRankings: finalRankingsCollectionName // Override with actual collection name used
                         },
                         targetDate: today // Index only today's date for speed
                     };