bulltrackers-module 1.0.203 → 1.0.204

package/functions/computation-system/utils/utils.js

@@ -1,40 +1,93 @@
 /**
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
- * DYNAMIC: Categorization logic is removed, replaced by manifest.
- * --- MODIFIED: getFirstDateFromSourceData is now getEarliestDataDates
- * and queries all data sources to build an availability map. ---
+ * FIXED: 'commitBatchInChunks' now respects Firestore 10MB size limit.
  */
-/** --- Computation System Sub-Pipes & Utils (Stateless, Dependency-Injection) --- */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
 /** Stage 1: Normalize a calculation name to kebab-case */
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
-/** Stage 2: Commit a batch of writes in chunks */
+/** * Stage 2: Commit a batch of writes in chunks 
+ * FIXED: Now splits batches by SIZE (9MB limit) and COUNT (450 docs) 
+ * to prevent "Request payload size exceeds the limit" errors.
+ */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const batchSizeLimit = config.batchSizeLimit || 450;
-    if (!writes.length) { logger.log('WARN', `[${operationName}] No writes to commit.`); return; }
-    for (let i = 0; i < writes.length; i += batchSizeLimit) {
-    const chunk = writes.slice(i, i + batchSizeLimit);
-    const batch = db.batch();
-    chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
-    const chunkNum = Math.floor(i / batchSizeLimit) + 1;
-    const totalChunks = Math.ceil(writes.length / batchSizeLimit);
-    await withRetry(() => batch.commit(), `${operationName} (Chunk ${chunkNum}/${totalChunks})`);
-    logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`); }
+
+    if (!writes || !writes.length) {
+        logger.log('WARN', `[${operationName}] No writes to commit.`);
+        return;
+    }
+
+    // Firestore Constraints
+    const MAX_BATCH_OPS = 300;          // Safety limit (Max 500)
+    const MAX_BATCH_BYTES = 9 * 1024 * 1024; // 9MB Safety limit (Max 10MB)
+
+    let currentBatch = db.batch();
+    let currentOpsCount = 0;
+    let currentBytesEst = 0;
+    let batchIndex = 1;
+    let totalChunks = 0; // We don't know total chunks in advance now due to dynamic sizing
+
+    // Helper to commit the current batch and reset
+    const commitAndReset = async () => {
+        if (currentOpsCount > 0) {
+            try {
+                await withRetry(
+                    () => currentBatch.commit(),
+                    `${operationName} (Chunk ${batchIndex})`
+                );
+                logger.log('INFO', `[${operationName}] Committed chunk ${batchIndex} (${currentOpsCount} ops, ~${(currentBytesEst / 1024 / 1024).toFixed(2)} MB).`);
+                batchIndex++;
+            } catch (err) {
+                logger.log('ERROR', `[${operationName}] Failed to commit chunk ${batchIndex}. Size: ${(currentBytesEst / 1024 / 1024).toFixed(2)} MB.`, { error: err.message });
+                throw err;
+            }
+        }
+        currentBatch = db.batch();
+        currentOpsCount = 0;
+        currentBytesEst = 0;
+    };
+
+    for (const write of writes) {
+        // 1. Estimate Size: JSON stringify is a decent proxy for Firestore payload size
+        // We handle potential circular refs or failures gracefully by assuming a minimum size
+        let docSize = 100;
+        try {
+            if (write.data) docSize = JSON.stringify(write.data).length;
+        } catch (e) { /* ignore size check error */ }
+
+        // 2. Warn if a SINGLE document is approaching the 1MB limit
+        if (docSize > 900 * 1024) {
+            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB). This allows few ops per batch.`);
+        }
+
+        // 3. Check if adding this write would overflow the batch
+        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) {
+            await commitAndReset();
+        }
+
+        // 4. Add to batch
+        currentBatch.set(write.ref, write.data, { merge: true });
+        currentOpsCount++;
+        currentBytesEst += docSize;
+    }
+
+    // 5. Commit remaining
+    await commitAndReset();
 }
 
 /** Stage 3: Generate an array of expected date strings between two dates */
 function getExpectedDateStrings(startDate, endDate) {
     const dateStrings = [];
     if (startDate <= endDate) {
-    const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
-    const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
-    for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); } }
+        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
+        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
+        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); }
+    }
     return dateStrings;
 }
 
@@ -46,10 +99,10 @@ async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     try {
-    if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`); return null; }
-    const query = db.collection(collectionName) .where(FieldPath.documentId(), '>=', '2000-01-01') .orderBy(FieldPath.documentId(), 'asc') .limit(1);
-    const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
-    if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) { return new Date(snapshot.docs[0].id + 'T00:00:00Z'); }
+        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`); return null; }
+        const query = db.collection(collectionName).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
+        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
+        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) { return new Date(snapshot.docs[0].id + 'T00:00:00Z'); }
     } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
     return null;
 }
@@ -59,13 +112,19 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     let earliestDate = null;
-    try { if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`); return null; }
-    const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
-    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
-    for (const blockDocRef of blockDocRefs) { const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection) .where(FieldPath.documentId(), '>=', '2000-01-01') .orderBy(FieldPath.documentId(), 'asc') .limit(1);
-    const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
-    if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) { const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-    if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate; } } } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
+    try {
+        if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`); return null; }
+        const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
+        for (const blockDocRef of blockDocRefs) {
+            const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection).where(FieldPath.documentId(), '>=', '2000-01-01').orderBy(FieldPath.documentId(), 'asc').limit(1);
+            const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
+            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
+                const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
+                if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate;
+            }
+        }
+    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
     return earliestDate;
 }
 
@@ -75,15 +134,15 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
     logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-    
-    const [ 
-        investorDate, 
-        speculatorDate, 
-        investorHistoryDate, 
-        speculatorHistoryDate, 
-        insightsDate, 
+
+    const [
+        investorDate,
+        speculatorDate,
+        investorHistoryDate,
+        speculatorHistoryDate,
+        insightsDate,
         socialDate,
-        priceDate  
+        priceDate
     ] = await Promise.all([
         getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
@@ -91,90 +150,84 @@ async function getEarliestDataDates(config, deps) {
         getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
         getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName),
-        getFirstDateFromPriceCollection(config, deps)  //TODO, Why no config.pricecollectionname here, looks ugly.
+        getFirstDateFromPriceCollection(config, deps)
     ]);
-    
-    const getMinDate = (...dates) => { 
-        const validDates = dates.filter(Boolean); 
-        if (validDates.length === 0) return null; 
-        return new Date(Math.min(...validDates)); 
+
+    const getMinDate = (...dates) => {
+        const validDates = dates.filter(Boolean);
+        if (validDates.length === 0) return null;
+        return new Date(Math.min(...validDates));
     };
-    
+
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate  = getMinDate(insightsDate);
-    const earliestSocialDate    = getMinDate(socialDate); 
-    const earliestPriceDate     = getMinDate(priceDate);  
-    const absoluteEarliest      = getMinDate(
-        earliestPortfolioDate, 
-        earliestHistoryDate, 
-        earliestInsightsDate, 
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate);
+    const earliestSocialDate = getMinDate(socialDate);
+    const earliestPriceDate = getMinDate(priceDate);
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate,
+        earliestHistoryDate,
+        earliestInsightsDate,
         earliestSocialDate,
-        earliestPriceDate  
+        earliestPriceDate
     );
-    
+
     const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-    
-    const result = { 
-        portfolio: earliestPortfolioDate   || new Date('2999-12-31T00:00:00Z'), 
-        history: earliestHistoryDate       || new Date('2999-12-31T00:00:00Z'), 
-        insights: earliestInsightsDate     || new Date('2999-12-31T00:00:00Z'), 
-        social: earliestSocialDate         || new Date('2999-12-31T00:00:00Z'),
-        price: earliestPriceDate           || new Date('2999-12-31T00:00:00Z'),  
+
+    const result = {
+        portfolio: earliestPortfolioDate || new Date('2999-12-31T00:00:00Z'),
+        history: earliestHistoryDate || new Date('2999-12-31T00:00:00Z'),
+        insights: earliestInsightsDate || new Date('2999-12-31T00:00:00Z'),
+        social: earliestSocialDate || new Date('2999-12-31T00:00:00Z'),
+        price: earliestPriceDate || new Date('2999-12-31T00:00:00Z'),
         absoluteEarliest: absoluteEarliest || fallbackDate
     };
-    
-    logger.log('INFO', 'Earliest data availability map built:', { 
-        portfolio:        result.portfolio.toISOString().slice(0, 10), 
-        history:          result.history.toISOString().slice(0, 10), 
-        insights:         result.insights.toISOString().slice(0, 10), 
-        social:           result.social.toISOString().slice(0, 10),
-        price:            result.price.toISOString().slice(0, 10),  
-        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) 
+
+    logger.log('INFO', 'Earliest data availability map built:', {
+        portfolio: result.portfolio.toISOString().slice(0, 10),
+        history: result.history.toISOString().slice(0, 10),
+        insights: result.insights.toISOString().slice(0, 10),
+        social: result.social.toISOString().slice(0, 10),
+        price: result.price.toISOString().slice(0, 10),
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
     });
-    
+
     return result;
 }
 
 /**
  * NEW HELPER: Get the earliest date from price collection
- * Price data is sharded differently - each shard contains instrumentId -> {prices: {date: price}}
  */
 async function getFirstDateFromPriceCollection(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collection = config.priceCollection || 'asset_prices'; // TODO  This hardcode is right, but we should really be passing the config directly as other data sources do.
-    
+    const collection = config.priceCollection || 'asset_prices';
+
     try {
         logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
-        
-        // Get all shards (limit to first few for performance)
+
         const snapshot = await withRetry(
-            () => db.collection(collection).limit(10).get(), 
+            () => db.collection(collection).limit(10).get(),
             `GetPriceShards(${collection})`
         );
-        
+
         if (snapshot.empty) {
             logger.log('WARN', `No price shards found in ${collection}`);
             return null;
         }
-        
+
         let earliestDate = null;
-        
-        // Iterate through shards to find the earliest date across all instruments
+
         snapshot.forEach(doc => {
             const shardData = doc.data();
-            
-            // Each shard has structure: { instrumentId: { ticker, prices: { "YYYY-MM-DD": price } } }
             for (const instrumentId in shardData) {
                 const instrumentData = shardData[instrumentId];
                 if (!instrumentData.prices) continue;
-                
-                // Get all dates for this instrument
+
                 const dates = Object.keys(instrumentData.prices)
                     .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
                     .sort();
-                
+
                 if (dates.length > 0) {
                     const firstDate = new Date(dates[0] + 'T00:00:00Z');
                     if (!earliestDate || firstDate < earliestDate) {
@@ -183,13 +236,13 @@ async function getFirstDateFromPriceCollection(config, deps) {
                 }
             }
         });
-        
+
         if (earliestDate) {
-            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`); // TODO, WTF IS THIS TS ERROR. Property 'toISOString' does not exist on type 'never'.ts(2339)
+            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
         }
-        
+
         return earliestDate;
-        
+
     } catch (e) {
         logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message });
         return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.203",
+  "version": "1.0.204",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [