bulltrackers-module 1.0.537 → 1.0.538

package/functions/task-engine/helpers/root_data_indexer_helpers.js

@@ -5,17 +5,62 @@
 
 const { FieldValue } = require('@google-cloud/firestore');
 
+/**
+ * Get the detail field name for a data type
+ * @param {string} dataType - Data type identifier
+ * @returns {string} - Detail field name in the index document
+ */
+function getDetailFieldName(dataType) {
+    const dataTypeMap = {
+        'signedInUserPortfolio': 'signedInUserPortfolio',
+        'signedInUserHistory': 'signedInUserHistory',
+        'signedInUserSocial': 'signedInSocial',
+        'piPortfolios': 'piPortfolios',
+        'piHistory': 'piHistory',
+        'piSocial': 'piSocial',
+        'normalPortfolios': 'normalPortfolio',
+        'normalHistory': 'normalHistory',
+        'speculatorPortfolios': 'speculatorPortfolio',
+        'speculatorHistory': 'speculatorHistory'
+    };
+    return dataTypeMap[dataType] || dataType;
+}
+
 /**
  * Check if root data is already indexed for a specific date
  * @param {Firestore} db - Firestore instance
  * @param {string} dateStr - Date string (YYYY-MM-DD)
  * @param {string} availabilityCollection - Collection name for root data index
- * @returns {Promise<boolean>} - True if already indexed
+ * @param {Array<string>} dataTypesRun - Optional array of data types to verify are indexed (e.g., ['signedInUserPortfolio', 'piSocial'])
+ * @returns {Promise<boolean>} - True if already indexed (and all requested data types are indexed if provided)
  */
-async function isRootDataIndexed(db, dateStr, availabilityCollection = 'system_root_data_index') {
+async function isRootDataIndexed(db, dateStr, availabilityCollection = 'system_root_data_index', dataTypesRun = []) {
     try {
         const indexDoc = await db.collection(availabilityCollection).doc(dateStr).get();
-        return indexDoc.exists;
+        if (!indexDoc.exists) {
+            return false;
+        }
+        
+        // If specific data types were requested, verify they are actually indexed
+        if (dataTypesRun && dataTypesRun.length > 0) {
+            const indexData = indexDoc.data();
+            const details = indexData?.details || {};
+            
+            // Check if all requested data types are indexed (have true values)
+            // If any are false or missing, return false (needs indexing)
+            for (const dataType of dataTypesRun) {
+                const detailField = getDetailFieldName(dataType);
+                if (details[detailField] !== true) {
+                    // At least one requested data type is not indexed
+                    return false;
+                }
+            }
+            // All requested data types are indexed as true
+            return true;
+        }
+        
+        // No specific data types requested - just check if document exists
+        return true;
     } catch (error) {
         // If we can't check, assume not indexed to be safe
         return false;
@@ -39,11 +84,43 @@ async function conditionallyRunRootDataIndexer({ db, logger, dateStr, rootDataIn
     const { runRootDataIndexer } = require('../../root-data-indexer/index');
     const availabilityCollection = rootDataIndexerConfig?.availabilityCollection || 'system_root_data_index';
     
-    // Check if already indexed
-    const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection);
-    if (alreadyIndexed) {
-        logger.log('INFO', `[RootDataIndexer] Root data already indexed for ${dateStr}, skipping`);
-        return false;
+    // IMPORTANT: Only index data types that were successfully stored (dataTypesRun contains only successful operations)
+    // If a write failed, it won't be in dataTypesRun, so we won't try to index it
+    // However, we still run the indexer to check/update the types that did succeed
+    
+    // Check if already indexed (verify specific data types if provided)
+    // Only check the data types that were actually run - if they're all already true, skip indexing (never overwrite true values)
+    if (dataTypesRun.length > 0) {
+        const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection, dataTypesRun);
+        if (alreadyIndexed) {
+            logger.log('INFO', `[RootDataIndexer] All requested data types (${dataTypesRun.join(', ')}) are already indexed as true for ${dateStr}, skipping (never overwrite true values)`);
+            return false;
+        }
+        
+        // If document exists but data types aren't indexed, log it
+        try {
+            const indexDoc = await db.collection(availabilityCollection).doc(dateStr).get();
+            if (indexDoc.exists) {
+                const indexData = indexDoc.data();
+                const details = indexData?.details || {};
+                const missingTypes = dataTypesRun.filter(dt => {
+                    const detailField = getDetailFieldName(dt);
+                    return details[detailField] !== true;
+                });
+                if (missingTypes.length > 0) {
+                    logger.log('INFO', `[RootDataIndexer] Index document exists for ${dateStr} but data types (${missingTypes.join(', ')}) are not indexed. Re-indexing...`);
+                }
+            }
+        } catch (e) {
+            // Ignore errors checking document
+        }
+    } else {
+        // No specific data types - check if document exists (legacy behavior)
+        const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection, []);
+        if (alreadyIndexed) {
+            logger.log('INFO', `[RootDataIndexer] Root data already indexed for ${dateStr}, skipping`);
+            return false;
+        }
     }
     
     // Use Firestore transaction to ensure only one instance runs the indexer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.537",
+  "version": "1.0.538",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [