bulltrackers-module 1.0.297 → 1.0.298

package/functions/computation-system/utils/utils.js

@@ -6,12 +6,14 @@ const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
 // [NEW] Single Source of Truth for Data Availability
+// These are defaults/fallbacks. The dynamic function getEarliestDataDates below is the primary source.
 const DEFINITIVE_EARLIEST_DATES = {
-    portfolio: new Date('2025-09-25T00:00:00Z'),
-    history:   new Date('2025-11-05T00:00:00Z'),
-    social:    new Date('2025-10-30T00:00:00Z'),
-    insights:  new Date('2025-08-26T00:00:00Z'),
-    price:     new Date('2025-08-01T00:00:00Z')
+    portfolio: new Date('2025-01-01T00:00:00Z'),
+    history:   new Date('2025-08-01T00:00:00Z'),
+    social:    new Date('2025-08-01T00:00:00Z'),
+    insights:  new Date('2025-08-01T00:00:00Z'),
+    price:     new Date('2025-08-01T00:00:00Z'),
+    absoluteEarliest: new Date('2023-08-01T00:00:00Z')
 };
 
 /** Stage 1: Normalize a calculation name to kebab-case */
@@ -162,95 +164,75 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/** Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates. */
-async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
-    const { db, logger } = deps;
-    try {
-        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`); return null; }
-        const query = db.collection(collectionName).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
-        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
-        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) { return new Date(snapshot.docs[0].id + 'T00:00:00Z'); }
-    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
-    return null;
-}
-
-/** Stage 4: Get the earliest date in a sharded collection */
-async function getFirstDateFromCollection(config, deps, collectionName) {
+/** * Stage 4: Get the earliest date from the Centralized Root Data Index.
+ * This REPLACES the expensive/error-prone raw collection scanning.
+ * It queries the 'system_root_data_index' to find the first date where data flags are TRUE.
+ */
+async function getEarliestDataDates(config, deps) {
     const { db, logger } = deps;
-    let earliestDate = null;
-    try {
-        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`); return null; }
-        const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
-        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
-        for (const blockDocRef of blockDocRefs) {
-            const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
-            const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
-            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-                if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate;
+    // Default to 'system_root_data_index' if not configured
+    const indexCollection = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 
+                           config.rootDataAvailabilityCollection || 
+                           'system_root_data_index';
+    
+    // Helper to find earliest date where a specific flag is true
+    // Efficient: Uses the document ID (date string) index.
+    const getEarliestForType = async (flagName) => {
+        try {
+            const snapshot = await db.collection(indexCollection)
+                .where(flagName, '==', true)
+                .orderBy(FieldPath.documentId(), 'asc')
+                .limit(1)
+                .get();
+
+            if (!snapshot.empty) {
+                const dateStr = snapshot.docs[0].id; // YYYY-MM-DD
+                // Safety check on date format
+                if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) {
+                    return new Date(dateStr + 'T00:00:00Z');
+                }
             }
+        } catch (e) {
+            logger.log('WARN', `[Utils] Failed to query index for ${flagName}: ${e.message}`);
         }
-    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
-    return earliestDate;
-}
+        return null;
+    };
 
-/** Stage 5: Determine the earliest date from *all* source data. */
-async function getEarliestDataDates(config, deps) {
-    const { logger } = deps;
-    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate, priceDate ] = await Promise.all([
-        getFirstDateFromCollection       (config, deps, config.normalUserPortfolioCollection),
-        getFirstDateFromCollection       (config, deps, config.speculatorPortfolioCollection),
-        getFirstDateFromCollection       (config, deps, config.normalUserHistoryCollection),
-        getFirstDateFromCollection       (config, deps, config.speculatorHistoryCollection),
-        getFirstDateFromSimpleCollection (config, deps, config.insightsCollectionName),
-        getFirstDateFromSimpleCollection (config, deps, config.socialInsightsCollectionName),
-        getFirstDateFromPriceCollection  (config, deps)
+    // Parallel query for all data types
+    const [portfolioDate, historyDate, socialDate, insightsDate, priceDate] = await Promise.all([
+        getEarliestForType('hasPortfolio'),
+        getEarliestForType('hasHistory'),
+        getEarliestForType('hasSocial'),
+        getEarliestForType('hasInsights'),
+        getEarliestForType('hasPrices')
     ]);
 
-    const getMinDate = (...dates) => {
-        const validDates = dates.filter(Boolean);
-        if (validDates.length === 0) return null;
-        return new Date(Math.min(...validDates));
-    };
-
-    const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate  = getMinDate(insightsDate);
-    const earliestSocialDate    = getMinDate(socialDate);
-    const earliestPriceDate     = getMinDate(priceDate);
-    const absoluteEarliest      = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate, earliestPriceDate);
+    // Calculate absolute earliest among found dates
+    const foundDates = [portfolioDate, historyDate, socialDate, insightsDate, priceDate].filter(d => d !== null);
+    
+    let absoluteEarliest = null;
+    if (foundDates.length > 0) {
+        absoluteEarliest = new Date(Math.min(...foundDates));
+    } else {
+        // Fallback if index is empty
+        const configStart = config.earliestComputationDate || '2023-01-01';
+        absoluteEarliest = new Date(configStart + 'T00:00:00Z');
+        logger.log('WARN', `[Utils] No data found in Root Data Index (${indexCollection}). Defaulting to ${configStart}`);
+    }
 
-    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+    // Update the static export for consumers who use it (best effort synchronization)
+    DEFINITIVE_EARLIEST_DATES.absoluteEarliest = absoluteEarliest;
 
     return {
-        portfolio:        earliestPortfolioDate || new Date('2999-12-31T00:00:00Z'),
-        history:          earliestHistoryDate   || new Date('2999-12-31T00:00:00Z'),
-        insights:         earliestInsightsDate  || new Date('2999-12-31T00:00:00Z'),
-        social:           earliestSocialDate    || new Date('2999-12-31T00:00:00Z'),
-        price:            earliestPriceDate     || new Date('2999-12-31T00:00:00Z'),
-        absoluteEarliest: absoluteEarliest      || fallbackDate
+        portfolio:        portfolioDate || new Date('2999-12-31T00:00:00Z'),
+        history:          historyDate   || new Date('2999-12-31T00:00:00Z'),
+        insights:         insightsDate  || new Date('2999-12-31T00:00:00Z'),
+        social:           socialDate    || new Date('2999-12-31T00:00:00Z'),
+        price:            priceDate     || new Date('2999-12-31T00:00:00Z'),
+        absoluteEarliest: absoluteEarliest
     };
 }
 
-async function getFirstDateFromPriceCollection(config, deps) {
-    const { db, logger } = deps;
-    const collection = config.priceCollection || 'asset_prices';
-    try {
-        const snapshot = await withRetry(() => db.collection(collection).limit(10).get(), `GetPriceShards(${collection})`);
-        let earliestDate = null;
-        snapshot.forEach(doc => {
-            const shardData = doc.data();
-            for (const instrumentId in shardData) {
-                const instrumentData = shardData[instrumentId];
-                if (!instrumentData.prices) continue;
-                const dates = Object.keys(instrumentData.prices).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort();
-                if (dates.length > 0) { const firstDate = new Date(dates[0] + 'T00:00:00Z'); if (!earliestDate || firstDate < earliestDate) earliestDate = firstDate; }
-            }
-        });
-        return earliestDate;
-    } catch (e) { logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message }); return null; }
-}
-
 module.exports = { 
     FieldValue, 
     FieldPath, 
@@ -259,7 +241,7 @@ module.exports = {
     getExpectedDateStrings, 
     getEarliestDataDates, 
     generateCodeHash,
-    generateDataHash, // Exported
+    generateDataHash, 
     withRetry,
     DEFINITIVE_EARLIEST_DATES
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.297",
+  "version": "1.0.298",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [