bulltrackers-module 1.0.127 → 1.0.129

package/functions/computation-system/utils/data_loader.js

@@ -12,231 +12,164 @@
  * @param {string} dateString - The date in YYYY-MM-DD format.
  * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
  */
-async function getPortfolioPartRefs(config, dependencies, dateString) {
-    // <<< FIX: Destructure all dependencies here, inside the function >>>
-    const { db, logger, calculationUtils } = dependencies;
+/** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
+
+/** Stage 1: Get portfolio part document references for a given date */
+async function getPortfolioPartRefs(config, deps, dateString) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // <<< END FIX >>>
 
     logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
 
     for (const collectionName of collectionsToQuery) {
-        const blockDocsQuery = db.collection(collectionName); // Use db
-        const blockDocRefs = await withRetry(
-            () => blockDocsQuery.listDocuments(),
-            `listDocuments(${collectionName})`
-        );
-
-        if (blockDocRefs.length === 0) {
-             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-             continue;
-         }
-
-        for (const blockDocRef of blockDocRefs) {
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+
+        // --- START MODIFICATION ---
+        // Run all "listDocuments" calls in parallel instead of a sequential loop
+        const partsPromises = blockDocRefs.map(blockDocRef => {
             const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-            const partDocs = await withRetry(
-                () => partsCollectionRef.listDocuments(),
-                `listDocuments(${partsCollectionRef.path})`
-            );
+            // Each call is individually retried
+            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`);
+        });
+
+        // Wait for all parallel queries to finish
+        const partDocArrays = await Promise.all(partsPromises);
+        
+        // Flatten the arrays of arrays into the final list
+        partDocArrays.forEach(partDocs => {
             allPartRefs.push(...partDocs);
-        }
+        });
+        // --- END MODIFICATION ---
     }
 
-    logger.log('INFO', `Found ${allPartRefs.length} part document references for ${dateString}.`);
+    logger.log('INFO', `Found ${allPartRefs.length} portfolio part refs for ${dateString}`);
     return allPartRefs;
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.dataLoader.loadDataByRefs
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {Firestore.DocumentReference[]} refs - An array of DocumentReferences to load.
- * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
- */
-async function loadDataByRefs(config, dependencies, refs) {
-    // <<< FIX: Destructure all dependencies here, inside the function >>>
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 2: Load data from an array of document references */
+async function loadDataByRefs(config, deps, refs) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // <<< END FIX >>>
 
-    if (!refs || refs.length === 0) { return {}; }
+    if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
     const batchSize = config.partRefBatchSize || 50;
 
     for (let i = 0; i < refs.length; i += batchSize) {
         const batchRefs = refs.slice(i, i + batchSize);
-        // Use db from dependencies
-        const snapshots = await withRetry(
-            () => db.getAll(...batchRefs),
-            `getAll(batch ${Math.floor(i / batchSize)})`
-        );
+        const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
+
         for (const doc of snapshots) {
-            if (doc.exists) {
-                const data = doc.data();
-                 if (data && typeof data === 'object') {
-                    Object.assign(mergedPortfolios, data);
-                } else {
-                     logger.log('WARN', `Document ${doc.id} exists but data is not an object. Data:`, data);
-                 }
-            }
+            if (!doc.exists) continue;
+            const data = doc.data();
+            if (data && typeof data === 'object') Object.assign(mergedPortfolios, data);
+            else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data);
         }
     }
+
     return mergedPortfolios;
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.dataLoader.loadFullDayMap
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
- * @param {Firestore.DocumentReference[]} partRefs - Array of part document references.
- * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
- */
-async function loadFullDayMap(config, dependencies, partRefs) {
-    // <<< FIX: Destructure only what's needed for this specific function >>>
-    const { logger } = dependencies;
-    // <<< END FIX >>>
+/** Stage 3: Load a full day map by delegating to loadDataByRefs */
+async function loadFullDayMap(config, deps, partRefs) {
+    const { logger } = deps;
+    if (!partRefs.length) return {};
 
-    if (partRefs.length === 0) return {};
     logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
-
-    // Pass config and dependencies to sub-pipe
-    const fullMap = await loadDataByRefs(config, dependencies, partRefs);
-    logger.log('TRACE', `Full day map loaded with ${Object.keys(fullMap).length} users.`);
+    const fullMap = await loadDataByRefs(config, deps, partRefs);
+    logger.log('TRACE', `Full day map loaded with ${Object.keys(fullMap).length} users`);
     return fullMap;
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.dataLoader.loadDailyInsights
- * Fetches the daily instrument insights document for a specific date.
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @returns {Promise<object|null>} The insights data object or null if not found/error.
- */
-async function loadDailyInsights(config, dependencies, dateString) {
-    // <<< FIX: Destructure all dependencies here, inside the function >>>
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 4: Load daily instrument insights */
+async function loadDailyInsights(config, deps, dateString) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // <<< END FIX >>>
 
-    const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights'; // Use config or default
-    logger.log('INFO', `Loading daily insights for date: ${dateString} from ${insightsCollectionName}`);
+    const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
+    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
+
     try {
         const docRef = db.collection(insightsCollectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
-
-        if (!docSnap.exists) {
-            logger.log('WARN', `Daily insights document not found for ${dateString}`);
-            return null;
-        }
-        logger.log('TRACE', `Successfully loaded insights for ${dateString}.`);
+        if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
+        logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
         return docSnap.data();
     } catch (error) {
         logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
-        return null; // Return null on error to allow computations to proceed partially if possible
+        return null;
     }
 }
 
-/**
- * --- NEW ---
- * Sub-pipe: pipe.computationSystem.dataLoader.loadDailySocialPostInsights
- * Fetches all analyzed social post documents for a specific date.
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @returns {Promise<object|null>} An object map of { [postId]: postData } or null.
- */
-async function loadDailySocialPostInsights(config, dependencies, dateString) {
-    // <<< FIX: Destructure all dependencies here, inside the function >>>
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 5: Load daily social post insights */
+async function loadDailySocialPostInsights(config, deps, dateString) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // <<< END FIX >>>
 
-    // Use the new config property, or fall back to a default
-    const socialInsightsCollectionName = config.socialInsightsCollectionName || 'daily_social_insights';
-    logger.log('INFO', `Loading social post insights for date: ${dateString} from ${socialInsightsCollectionName}`);
-    
+    const collectionName = config.socialInsightsCollectionName || 'daily_social_insights';
+    logger.log('INFO', `Loading social post insights for ${dateString} from ${collectionName}`);
+
     try {
-        const postsCollectionRef = db.collection(socialInsightsCollectionName).doc(dateString).collection('posts');
+        const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
         const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
+        if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
 
-        if (querySnapshot.empty) {
-            logger.log('WARN', `No social post insights found for ${dateString}.`);
-            return null;
-        }
-        
         const postsMap = {};
-        querySnapshot.forEach(doc => {
-            postsMap[doc.id] = doc.data();
-        });
-        
-        logger.log('TRACE', `Successfully loaded ${Object.keys(postsMap).length} social post insights for ${dateString}.`);
+        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+        logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
         return postsMap;
-
     } catch (error) {
         logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
         return null;
     }
 }
-// --- END NEW ---
 
-
-
-/**
- * --- NEW ---
- * Sub-pipe: pipe.computationSystem.dataLoader.getHistoryPartRefs
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
- */
-async function getHistoryPartRefs(config, dependencies, dateString) {
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 6: Get history part references for a given date */
+async function getHistoryPartRefs(config, deps, dateString) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
 
-    logger.log('INFO', `Getting history part references for date: ${dateString}`);
+    logger.log('INFO', `Getting history part references for ${dateString}`);
     const allPartRefs = [];
-    // --- MODIFIED: Use new history collection config keys ---
     const collectionsToQuery = [config.normalUserHistoryCollection, config.speculatorHistoryCollection];
 
     for (const collectionName of collectionsToQuery) {
-        if (!collectionName) { // Add a check in case config is missing
-             logger.log('WARN', `History collection name is undefined. Skipping.`);
-             continue;
-        }
+        if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
         const blockDocsQuery = db.collection(collectionName);
-        const blockDocRefs = await withRetry(
-            () => blockDocsQuery.listDocuments(),
-            `listDocuments(${collectionName})`
-        );
-
-        if (blockDocRefs.length === 0) {
-             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-             continue;
-         }
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
 
-        for (const blockDocRef of blockDocRefs) {
+        // --- START MODIFICATION ---
+        // Run all "listDocuments" calls in parallel instead of a sequential loop
+        const partsPromises = blockDocRefs.map(blockDocRef => {
             const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-            const partDocs = await withRetry(
-                () => partsCollectionRef.listDocuments(),
-                `listDocuments(${partsCollectionRef.path})`
-            );
+            // Each call is individually retried
+            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`);
+        });
+
+        // Wait for all parallel queries to finish
+        const partDocArrays = await Promise.all(partsPromises);
+        
+        // Flatten the arrays of arrays into the final list
+        partDocArrays.forEach(partDocs => {
             allPartRefs.push(...partDocs);
-        }
+        });
+        // --- END MODIFICATION ---
     }
 
-    logger.log('INFO', `Found ${allPartRefs.length} history part document references for ${dateString}.`);
+    logger.log('INFO', `Found ${allPartRefs.length} history part refs for ${dateString}`);
     return allPartRefs;
 }
 
-
 module.exports = {
     getPortfolioPartRefs,
     loadDataByRefs,
     loadFullDayMap,
     loadDailyInsights,
-    loadDailySocialPostInsights, 
-    getHistoryPartRefs, // <-- EXPORT NEW FUNCTION
+    loadDailySocialPostInsights,
+    getHistoryPartRefs,
 };

package/functions/computation-system/utils/utils.js

@@ -3,56 +3,37 @@
  * REFACTORED: Now stateless and receive dependencies where needed.
  * DYNAMIC: Categorization logic is removed, replaced by manifest.
  */
+/** --- Computation System Sub-Pipes & Utils (Stateless) --- */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
-/**
- * Normalizes a calculation name to kebab-case.
- * @param {string} name 
- * @returns {string}
- */
+/** Stage 1: Normalize a calculation name to kebab-case */
 function normalizeName(name) {
     return name.replace(/_/g, '-');
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.commitBatchInChunks
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {Array<object>} writes - Array of { ref: DocumentReference, data: object }.
- * @param {string} operationName - Name for logging.
- */
-async function commitBatchInChunks(config, dependencies, writes, operationName) {
-    // --- MODIFIED: Get withRetry from dependencies ---
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 2: Commit a batch of writes in chunks */
+async function commitBatchInChunks(config, deps, writes, operationName) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // --- END MODIFIED ---
-    
+
     const batchSizeLimit = config.batchSizeLimit || 450;
-    
-    if (writes.length === 0) {
-        logger.log('WARN', `[${operationName}] No writes to commit.`);
-        return;
-    }
+    if (!writes.length) { logger.log('WARN', `[${operationName}] No writes to commit.`); return; }
+
     for (let i = 0; i < writes.length; i += batchSizeLimit) {
-        const batch = db.batch(); // Use db
         const chunk = writes.slice(i, i + batchSizeLimit);
+        const batch = db.batch();
         chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
 
         const chunkNum = Math.floor(i / batchSizeLimit) + 1;
         const totalChunks = Math.ceil(writes.length / batchSizeLimit);
-        await withRetry(
-            () => batch.commit(),
-            `${operationName} (Chunk ${chunkNum}/${totalChunks})`
-        );
+        await withRetry(() => batch.commit(), `${operationName} (Chunk ${chunkNum}/${totalChunks})`);
+
         logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`);
     }
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.getExpectedDateStrings
- * (Stateless)
- */
+/** Stage 3: Generate an array of expected date strings between two dates */
 function getExpectedDateStrings(startDate, endDate) {
     const dateStrings = [];
     if (startDate <= endDate) {
@@ -65,86 +46,62 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/**
- * Internal helper: Finds the earliest date document in a collection.
- */
-async function getFirstDateFromCollection(config, dependencies, collectionName) {
-    // --- MODIFIED: Get withRetry from dependencies ---
-    const { db, logger, calculationUtils } = dependencies;
+/** Stage 4: Get the earliest date in a collection */
+async function getFirstDateFromCollection(config, deps, collectionName) {
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    // --- END MODIFIED ---
+
     let earliestDate = null;
     try {
-        const blockDocRefs = await withRetry(
-            () => db.collection(collectionName).listDocuments(), // Use db
-            `GetBlocks(${collectionName})`
-        );
-
-         if (blockDocRefs.length === 0) {
-             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-             return null;
-         }
-
-         for (const blockDocRef of blockDocRefs) {
-             const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
-                 .where(FieldPath.documentId(), '>=', '2000-01-01')
-                 .orderBy(FieldPath.documentId(), 'asc')
-                 .limit(1);
-
-             const snapshotSnap = await withRetry(
-                 () => snapshotQuery.get(),
-                 `GetEarliestSnapshot(${blockDocRef.path})`
-             );
-
-             if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                 const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-                 if (!earliestDate || foundDate < earliestDate) {
-                     earliestDate = foundDate;
-                 }
-             }
-         }
+        const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
+
+        for (const blockDocRef of blockDocRefs) {
+            const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
+                .where(FieldPath.documentId(), '>=', '2000-01-01')
+                .orderBy(FieldPath.documentId(), 'asc')
+                .limit(1);
 
+            const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
+            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
+                const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
+                if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate;
+            }
+        }
     } catch (e) {
         logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
     }
+
     return earliestDate;
 }
 
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.getFirstDateFromSourceData
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<Date>} The earliest date found or a default fallback date.
- */
-async function getFirstDateFromSourceData(config, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', 'Querying for the earliest date from source portfolio data...');
-    
-    // Pass dependencies to sub-pipe
-    const investorDate = await getFirstDateFromCollection(config, dependencies, config.normalUserPortfolioCollection);
-    const speculatorDate = await getFirstDateFromCollection(config, dependencies, config.speculatorPortfolioCollection);
-
-     let earliestDate;
-     if (investorDate && speculatorDate) {
-         earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-     } else {
-         earliestDate = investorDate || speculatorDate;
-     }
-
-     if (earliestDate) {
-         logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-         return earliestDate;
-     } else {
-         const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-         logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
-         return fallbackDate;
-     }
+/** Stage 5: Determine the earliest date from source data across both user types */
+async function getFirstDateFromSourceData(config, deps) {
+    const { logger } = deps;
+    logger.log('INFO', 'Querying for earliest date from source portfolio data...');
+
+    const investorDate = await getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection);
+    const speculatorDate = await getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection);
+
+    let earliestDate;
+    if (investorDate && speculatorDate) earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
+    else earliestDate = investorDate || speculatorDate;
+
+    if (earliestDate) {
+        logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
+        return earliestDate;
+    } else {
+        const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+        logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
+        return fallbackDate;
+    }
 }
 
 module.exports = {
-    FieldValue, FieldPath,
+    FieldValue,
+    FieldPath,
     normalizeName,
     commitBatchInChunks,
-    getExpectedDateStrings, 
+    getExpectedDateStrings,
     getFirstDateFromSourceData,
-};
+};