bulltrackers-module 1.0.306 → 1.0.308

package/functions/computation-system/utils/utils.js

@@ -1,247 +1,130 @@
 /**
  * FILENAME: computation-system/utils/utils.js
+ * UPDATED: Removed hardcoded date logic. Earliest dates are now fully data-driven.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
-// [NEW] Single Source of Truth for Data Availability
-// These are defaults/fallbacks. The dynamic function getEarliestDataDates below is the primary source.
+// [UPDATED] Registry for Data Availability. 
+// Populated dynamically by getEarliestDataDates().
 const DEFINITIVE_EARLIEST_DATES = {
-    portfolio: new Date('2025-01-01T00:00:00Z'),
-    history:   new Date('2025-08-01T00:00:00Z'),
-    social:    new Date('2025-08-01T00:00:00Z'),
-    insights:  new Date('2025-08-01T00:00:00Z'),
-    price:     new Date('2025-08-01T00:00:00Z'),
-    absoluteEarliest: new Date('2023-08-01T00:00:00Z')
+    portfolio: null,
+    history:   null,
+    social:    null,
+    insights:  null,
+    price:     null,
+    absoluteEarliest: null
 };
 
-/** Stage 1: Normalize a calculation name to kebab-case */
+/** Normalizes a calculation name to kebab-case */
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
-/**
- * Generates a SHA-256 hash of a code string.
- */
+/** Generates a SHA-256 hash of a code string. */
 function generateCodeHash(codeString) {
     if (!codeString) return 'unknown';
-    let clean = codeString.replace(/\/\/.*$/gm, '');
-    clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
-    clean = clean.replace(/\s+/g, '');
+    let clean = codeString.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\s+/g, '');
     return crypto.createHash('sha256').update(clean).digest('hex');
 }
 
-/**
- * [NEW] Generates a stable SHA-256 hash of a data object.
- * Keys are sorted to ensure determinism.
- */
+/** Generates a stable SHA-256 hash of a data object. */
 function generateDataHash(data) {
     if (data === undefined) return 'undefined';
-    
-    // Recursive stable stringify
     const stableStringify = (obj) => {
-        if (typeof obj !== 'object' || obj === null) {
-            return JSON.stringify(obj);
-        }
-        if (Array.isArray(obj)) {
-            return '[' + obj.map(stableStringify).join(',') + ']';
-        }
-        return '{' + Object.keys(obj).sort().map(k => 
-            JSON.stringify(k) + ':' + stableStringify(obj[k])
-        ).join(',') + '}';
+        if (typeof obj !== 'object' || obj === null) return JSON.stringify(obj);
+        if (Array.isArray(obj)) return '[' + obj.map(stableStringify).join(',') + ']';
+        return '{' + Object.keys(obj).sort().map(k => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',') + '}';
     };
-
     try {
-        const str = stableStringify(data);
-        return crypto.createHash('sha256').update(str).digest('hex');
-    } catch (e) {
-        return 'hash_error';
-    }
+        return crypto.createHash('sha256').update(stableStringify(data)).digest('hex');
+    } catch (e) { return 'hash_error'; }
 }
 
-/**
- * Executes a function with exponential backoff retry logic.
- * @param {Function} fn - Async function to execute
- * @param {string} operationName - Label for logging
- * @param {number} maxRetries - Max attempts (default 3)
- */
+/** Exponential backoff retry logic. */
 async function withRetry(fn, operationName, maxRetries = 3) {
     let attempt = 0;
     while (attempt < maxRetries) {
-        try {
-            return await fn();
-        } catch (error) {
+        try { return await fn(); } catch (error) {
             attempt++;
-            console.warn(`[Retry] ${operationName} failed (Attempt ${attempt}/${maxRetries}): ${error.message}`);
             if (attempt >= maxRetries) throw error;
-            // Exponential backoff: 1s, 2s, 4s...
             await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1)));
         }
     }
 }
 
-/** Stage 2: Commit a batch of writes in chunks 
- * UPDATED: Now supports { type: 'DELETE' } in the write object.
- */
+/** Commit a batch of writes in chunks. */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger } = deps;
-    // Use the local withRetry if not provided in deps
-    const retryFn = (deps.calculationUtils && deps.calculationUtils.withRetry) ? deps.calculationUtils.withRetry : withRetry;
-
-    if (!writes || !writes.length) {
-        logger.log('WARN', `[${operationName}] No writes to commit.`);
-        return;
-    }
-
-    const MAX_BATCH_OPS = 300;
-    const MAX_BATCH_BYTES = 9 * 1024 * 1024; 
+    const retryFn = (deps.calculationUtils?.withRetry) || withRetry;
+    if (!writes?.length) return;
 
-    let currentBatch = db.batch();
-    let currentOpsCount = 0;
-    let currentBytesEst = 0;
-    let batchIndex = 1;
+    const MAX_BATCH_OPS = 300, MAX_BATCH_BYTES = 9 * 1024 * 1024; 
+    let currentBatch = db.batch(), currentOpsCount = 0, currentBytesEst = 0, batchIndex = 1;
 
     const commitAndReset = async () => {
         if (currentOpsCount > 0) {
-            try {
-                await retryFn(
-                    () => currentBatch.commit(),
-                    `${operationName} (Chunk ${batchIndex})`
-                );
-                logger.log('INFO', `[${operationName}] Committed chunk ${batchIndex} (${currentOpsCount} ops, ~${(currentBytesEst / 1024 / 1024).toFixed(2)} MB).`);
-                batchIndex++;
-            } catch (err) {
-                logger.log('ERROR', `[${operationName}] Failed to commit chunk ${batchIndex}. Size: ${(currentBytesEst / 1024 / 1024).toFixed(2)} MB.`, { error: err.message });
-                throw err;
-            }
+            await retryFn(() => currentBatch.commit(), `${operationName} (Chunk ${batchIndex})`);
+            batchIndex++;
         }
-        currentBatch = db.batch();
-        currentOpsCount = 0;
-        currentBytesEst = 0;
+        currentBatch = db.batch(); currentOpsCount = 0; currentBytesEst = 0;
     };
 
     for (const write of writes) {
-        // [NEW] Handle DELETE operations
         if (write.type === 'DELETE') {
-            if ((currentOpsCount + 1 > MAX_BATCH_OPS)) {
-                await commitAndReset();
-            }
-            currentBatch.delete(write.ref);
-            currentOpsCount++;
-            continue;
+            if (currentOpsCount + 1 > MAX_BATCH_OPS) await commitAndReset();
+            currentBatch.delete(write.ref); currentOpsCount++; continue;
         }
-
-        // Standard SET/UPDATE operations
         let docSize = 100;
         try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
-
-        if (docSize > 900 * 1024) {
-            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB).`);
-        }
-
-        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) {
-            await commitAndReset();
-        }
-
-        // USE PROVIDED OPTIONS OR DEFAULT TO MERGE: TRUE
-        const options = write.options || { merge: true };
-        currentBatch.set(write.ref, write.data, options);
-        
-        currentOpsCount++;
-        currentBytesEst += docSize;
+        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) await commitAndReset();
+        currentBatch.set(write.ref, write.data, write.options || { merge: true });
+        currentOpsCount++; currentBytesEst += docSize;
     }
-
     await commitAndReset();
 }
 
-/** Stage 3: Generate an array of expected date strings between two dates */
+/** Generate array of date strings between two dates. */
 function getExpectedDateStrings(startDate, endDate) {
     const dateStrings = [];
-    if (startDate <= endDate) {
+    if (startDate && endDate && startDate <= endDate) {
         const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
-        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
-        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); }
+        for (let d = startUTC; d <= endDate; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); }
     }
     return dateStrings;
 }
 
-/** * Stage 4: Get the earliest date from the Centralized Root Data Index.
- * This REPLACES the expensive/error-prone raw collection scanning.
- * It queries the 'system_root_data_index' to find the first date where data flags are TRUE.
+/**
+ * [UPDATED] Single Source of Truth for Data Availability.
+ * Updates the global DEFINITIVE_EARLIEST_DATES object.
  */
 async function getEarliestDataDates(config, deps) {
     const { db, logger } = deps;
-    // Default to 'system_root_data_index' if not configured
-    const indexCollection = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 
-                           config.rootDataAvailabilityCollection || 
-                           'system_root_data_index';
+    const indexCollection = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
     
-    // Helper to find earliest date where a specific flag is true
-    // Efficient: Uses the document ID (date string) index.
     const getEarliestForType = async (flagName) => {
         try {
-            const snapshot = await db.collection(indexCollection)
-                .where(flagName, '==', true)
-                .orderBy(FieldPath.documentId(), 'asc')
-                .limit(1)
-                .get();
-
-            if (!snapshot.empty) {
-                const dateStr = snapshot.docs[0].id; // YYYY-MM-DD
-                // Safety check on date format
-                if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) {
-                    return new Date(dateStr + 'T00:00:00Z');
-                }
-            }
-        } catch (e) {
-            logger.log('WARN', `[Utils] Failed to query index for ${flagName}: ${e.message}`);
-        }
+            const snap = await db.collection(indexCollection).where(flagName, '==', true).orderBy(FieldPath.documentId(), 'asc').limit(1).get();
+            if (!snap.empty) return new Date(snap.docs[0].id + 'T00:00:00Z');
+        } catch (e) { logger.log('WARN', `[Utils] Index query failed for ${flagName}`); }
         return null;
     };
 
-    // Parallel query for all data types
-    const [portfolioDate, historyDate, socialDate, insightsDate, priceDate] = await Promise.all([
-        getEarliestForType('hasPortfolio'),
-        getEarliestForType('hasHistory'),
-        getEarliestForType('hasSocial'),
-        getEarliestForType('hasInsights'),
-        getEarliestForType('hasPrices')
+    const [portfolio, history, social, insights, price] = await Promise.all([
+        getEarliestForType('hasPortfolio'), getEarliestForType('hasHistory'),
+        getEarliestForType('hasSocial'), getEarliestForType('hasInsights'), getEarliestForType('hasPrices')
     ]);
 
-    // Calculate absolute earliest among found dates
-    const foundDates = [portfolioDate, historyDate, socialDate, insightsDate, priceDate].filter(d => d !== null);
-    
-    let absoluteEarliest = null;
-    if (foundDates.length > 0) {
-        absoluteEarliest = new Date(Math.min(...foundDates));
-    } else {
-        // Fallback if index is empty
-        const configStart = config.earliestComputationDate || '2023-01-01';
-        absoluteEarliest = new Date(configStart + 'T00:00:00Z');
-        logger.log('WARN', `[Utils] No data found in Root Data Index (${indexCollection}). Defaulting to ${configStart}`);
-    }
+    const found = [portfolio, history, social, insights, price].filter(d => d !== null);
+    const absoluteEarliest = found.length > 0 ? new Date(Math.min(...found)) : new Date('2023-01-01T00:00:00Z');
 
-    // Update the static export for consumers who use it (best effort synchronization)
-    DEFINITIVE_EARLIEST_DATES.absoluteEarliest = absoluteEarliest;
+    // Sync the global registry
+    Object.assign(DEFINITIVE_EARLIEST_DATES, { portfolio, history, social, insights, price, absoluteEarliest });
 
-    return {
-        portfolio:        portfolioDate || new Date('2999-12-31T00:00:00Z'),
-        history:          historyDate   || new Date('2999-12-31T00:00:00Z'),
-        insights:         insightsDate  || new Date('2999-12-31T00:00:00Z'),
-        social:           socialDate    || new Date('2999-12-31T00:00:00Z'),
-        price:            priceDate     || new Date('2999-12-31T00:00:00Z'),
-        absoluteEarliest: absoluteEarliest
-    };
+    return DEFINITIVE_EARLIEST_DATES;
 }
 
 module.exports = { 
-    FieldValue, 
-    FieldPath, 
-    normalizeName, 
-    commitBatchInChunks, 
-    getExpectedDateStrings, 
-    getEarliestDataDates, 
-    generateCodeHash,
-    generateDataHash, 
-    withRetry,
-    DEFINITIVE_EARLIEST_DATES
+    FieldValue, FieldPath, normalizeName, commitBatchInChunks, 
+    getExpectedDateStrings, getEarliestDataDates, generateCodeHash,
+    generateDataHash, withRetry, DEFINITIVE_EARLIEST_DATES 
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.306",
+  "version": "1.0.308",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [