bulltrackers-module 1.0.727 → 1.0.729

package/functions/api-v2/index.js

@@ -18,7 +18,7 @@ const errorHandler = require('./middleware/error_handler.js');
 const allowedOrigins = [
     'https://bulltrackers.web.app',
     'http://172.26.96.1:8000',
-    'http://127.0.0.1:8000/'
+    'http://127.0.0.1:8000'
 ];
 
 /**

package/functions/computation-system/data/DependencyFetcher.js

@@ -142,7 +142,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
                 return JSON.parse(fileContent.toString('utf8'));
             }
         } catch (e) {
-            log.error(`[DependencyFetcher] ❌ GCS Fetch Failed for ${name}: ${e.message}`);
+            log.log('ERROR', `[DependencyFetcher] ❌ GCS Fetch Failed for ${name}: ${e.message}`);
             // Depending on strictness, we might return null here or allow it to fail hard.
             // Returning null allows 'isDataEmpty' to catch it as "MISSING"
             return null; 
@@ -158,7 +158,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             data = { ...data, ...realData }; // Merge payload into base
             delete data.payload;
         } catch (e) {
-            log.error(`[DependencyFetcher] ❌ ${e.message} at ${docRef.path}`);
+            log.log('ERROR', `[DependencyFetcher] ❌ ${e.message} at ${docRef.path}`);
             return null;
         }
     }
@@ -169,7 +169,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
     if (data._sharded) {
         const shardSnaps = await docRef.collection('_shards').get();
         if (shardSnaps.empty) {
-            log.error(`[DependencyFetcher] ❌ Sharded doc has no shards: ${docRef.path}`);
+            log.log('ERROR', `[DependencyFetcher] ❌ Sharded doc has no shards: ${docRef.path}`);
             return null;
         }
 
@@ -186,7 +186,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
                     const decomp = tryDecompress(sData.payload);
                     sData = (typeof decomp === 'string') ? JSON.parse(decomp) : decomp;
                 } catch (e) {
-                    log.error(`[DependencyFetcher] ❌ Shard decompression failed (${shard.id}): ${e.message}`);
+                    log.log('ERROR', `[DependencyFetcher] ❌ Shard decompression failed (${shard.id}): ${e.message}`);
                     continue; 
                 }
             }

package/functions/computation-system/utils/data_loader.js

@@ -321,4 +321,251 @@ async function loadRetailFirestore(db, collectionName, dateStr) {
     } catch (e) {
         return {};
     }
-}
+}
+
+// =============================================================================
+// 8. STREAMING DATA (For StandardExecutor)
+// =============================================================================
+
+/**
+ * Get portfolio part references for streaming.
+ * In BigQuery mode, returns virtual refs that signal to load from BQ.
+ * @param {object} config - Configuration object
+ * @param {object} deps - Dependencies (db, logger)
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array<string>|null} userTypes - User types to filter (null = all)
+ * @returns {Promise<Array>} Array of part references
+ */
+exports.getPortfolioPartRefs = async (config, deps, dateStr, userTypes = null) => {
+    const { logger } = deps;
+    
+    // In BigQuery mode, return virtual refs with metadata
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        logger.log('INFO', `[DataLoader] Using BigQuery virtual refs for portfolios (${dateStr})`);
+        return [{ 
+            _bigquery: true, 
+            type: 'portfolio', 
+            date: dateStr, 
+            userTypes: userTypes 
+        }];
+    }
+    
+    // Firestore fallback - return actual document references
+    const { db } = deps;
+    const refs = [];
+    
+    // Check both PI and SignedIn collections
+    const collections = [
+        { name: 'PopularInvestorPortfolios', userType: 'POPULAR_INVESTOR' },
+        { name: 'SignedInUserPortfolios', userType: 'SIGNED_IN_USER' }
+    ];
+    
+    for (const col of collections) {
+        if (userTypes && !userTypes.includes(col.userType)) continue;
+        
+        try {
+            const partsSnap = await db.collection(col.name)
+                .doc('latest')
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .listDocuments();
+            
+            for (const docRef of partsSnap) {
+                refs.push({ ref: docRef, userType: col.userType, collection: col.name });
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] Failed to get portfolio refs from ${col.name}: ${e.message}`);
+        }
+    }
+    
+    return refs;
+};
+
+/**
+ * Get history part references for streaming.
+ * @param {object} config - Configuration object
+ * @param {object} deps - Dependencies (db, logger)
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array<string>|null} userTypes - User types to filter (null = all)
+ * @returns {Promise<Array>} Array of part references
+ */
+exports.getHistoryPartRefs = async (config, deps, dateStr, userTypes = null) => {
+    const { logger } = deps;
+    
+    // In BigQuery mode, return virtual refs with metadata
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        logger.log('INFO', `[DataLoader] Using BigQuery virtual refs for history (${dateStr})`);
+        return [{ 
+            _bigquery: true, 
+            type: 'history', 
+            date: dateStr, 
+            userTypes: userTypes 
+        }];
+    }
+    
+    // Firestore fallback - return actual document references
+    const { db } = deps;
+    const refs = [];
+    
+    const collections = [
+        { name: 'PopularInvestorTradeHistory', userType: 'POPULAR_INVESTOR' },
+        { name: 'SignedInUserTradeHistory', userType: 'SIGNED_IN_USER' }
+    ];
+    
+    for (const col of collections) {
+        if (userTypes && !userTypes.includes(col.userType)) continue;
+        
+        try {
+            const partsSnap = await db.collection(col.name)
+                .doc('latest')
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .listDocuments();
+            
+            for (const docRef of partsSnap) {
+                refs.push({ ref: docRef, userType: col.userType, collection: col.name });
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] Failed to get history refs from ${col.name}: ${e.message}`);
+        }
+    }
+    
+    return refs;
+};
+
+/**
+ * Stream portfolio data in chunks (async generator).
+ * Yields objects of { cid: portfolioData } for memory efficiency.
+ * @param {object} config - Configuration object
+ * @param {object} deps - Dependencies (db, logger)
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} refs - Part references from getPortfolioPartRefs
+ * @param {Array<string>|null} userTypes - User types to filter
+ * @yields {object} Chunk of portfolio data { cid: data }
+ */
+exports.streamPortfolioData = async function* (config, deps, dateStr, refs, userTypes = null) {
+    const { logger } = deps;
+    
+    if (!refs || refs.length === 0) {
+        logger.log('WARN', `[DataLoader] No portfolio refs provided for streaming (${dateStr})`);
+        return;
+    }
+    
+    // BigQuery mode - load all at once and yield in chunks
+    const isBigQuery = refs.some(r => r && r._bigquery === true);
+    
+    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const types = userTypes || ['POPULAR_INVESTOR', 'SIGNED_IN_USER'];
+            const data = await queryPortfolioData(dateStr, null, types, logger);
+            
+            if (data && Object.keys(data).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming ${Object.keys(data).length} portfolios from BigQuery (${dateStr})`);
+                
+                // Yield in chunks of 100 users for memory efficiency
+                const CHUNK_SIZE = 100;
+                const entries = Object.entries(data);
+                
+                for (let i = 0; i < entries.length; i += CHUNK_SIZE) {
+                    const chunk = {};
+                    entries.slice(i, i + CHUNK_SIZE).forEach(([cid, portfolio]) => {
+                        chunk[cid] = portfolio;
+                    });
+                    yield chunk;
+                }
+                return;
+            }
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] BigQuery portfolio stream failed: ${e.message}`);
+        }
+    }
+    
+    // Firestore fallback - stream from refs
+    for (const refInfo of refs) {
+        if (!refInfo.ref) continue;
+        
+        try {
+            const snap = await refInfo.ref.get();
+            if (snap.exists) {
+                const data = snap.data();
+                // Add user type metadata
+                const enriched = {};
+                for (const [cid, portfolio] of Object.entries(data)) {
+                    enriched[cid] = { ...portfolio, _userType: refInfo.userType };
+                }
+                yield enriched;
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] Failed to stream portfolio part: ${e.message}`);
+        }
+    }
+};
+
+/**
+ * Stream history data in chunks (async generator).
+ * @param {object} config - Configuration object
+ * @param {object} deps - Dependencies (db, logger)
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} refs - Part references from getHistoryPartRefs
+ * @param {Array<string>|null} userTypes - User types to filter
+ * @yields {object} Chunk of history data { cid: data }
+ */
+exports.streamHistoryData = async function* (config, deps, dateStr, refs, userTypes = null) {
+    const { logger } = deps;
+    
+    if (!refs || refs.length === 0) {
+        logger.log('WARN', `[DataLoader] No history refs provided for streaming (${dateStr})`);
+        return;
+    }
+    
+    // BigQuery mode - load all at once and yield in chunks
+    const isBigQuery = refs.some(r => r && r._bigquery === true);
+    
+    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const types = userTypes || ['POPULAR_INVESTOR', 'SIGNED_IN_USER'];
+            const data = await queryHistoryData(dateStr, null, types, logger);
+            
+            if (data && Object.keys(data).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming ${Object.keys(data).length} history records from BigQuery (${dateStr})`);
+                
+                // Yield in chunks of 100 users
+                const CHUNK_SIZE = 100;
+                const entries = Object.entries(data);
+                
+                for (let i = 0; i < entries.length; i += CHUNK_SIZE) {
+                    const chunk = {};
+                    entries.slice(i, i + CHUNK_SIZE).forEach(([cid, history]) => {
+                        chunk[cid] = history;
+                    });
+                    yield chunk;
+                }
+                return;
+            }
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] BigQuery history stream failed: ${e.message}`);
+        }
+    }
+    
+    // Firestore fallback - stream from refs
+    for (const refInfo of refs) {
+        if (!refInfo.ref) continue;
+        
+        try {
+            const snap = await refInfo.ref.get();
+            if (snap.exists) {
+                const data = snap.data();
+                // Add user type metadata
+                const enriched = {};
+                for (const [cid, history] of Object.entries(data)) {
+                    enriched[cid] = { ...history, _userType: refInfo.userType };
+                }
+                yield enriched;
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] Failed to stream history part: ${e.message}`);
+        }
+    }
+};

package/functions/computation-system/utils/utils.js

@@ -55,11 +55,32 @@ async function withRetry(fn, operationName, maxRetries = 3) {
 
 /** * Recursively sanitizes data for Firestore compatibility.
  * Converts 'undefined' to 'null' to prevent validation errors.
+ * Converts invalid Dates to null to prevent Timestamp serialization errors.
+ * Converts NaN numbers to null to prevent serialization errors.
  */
 function sanitizeForFirestore(data) {
     if (data === undefined) return null;
     if (data === null) return null;
-    if (data instanceof Date) return data;
+    
+    // Handle Date objects - validate they're not Invalid Date (NaN timestamp)
+    if (data instanceof Date) {
+        if (isNaN(data.getTime())) {
+            // Invalid Date - return null instead of crashing Firestore
+            return null;
+        }
+        return data;
+    }
+    
+    // Handle NaN numbers (would cause serialization issues)
+    if (typeof data === 'number' && isNaN(data)) {
+        return null;
+    }
+    
+    // Handle Infinity (also not valid for Firestore)
+    if (data === Infinity || data === -Infinity) {
+        return null;
+    }
+    
     // Handle Firestore Types if necessary (e.g. GeoPoint, Timestamp), passed through as objects usually.
     
     if (Array.isArray(data)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.727",
+  "version": "1.0.729",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [