bulltrackers-module 1.0.715 → 1.0.716

package/functions/computation-system/persistence/ResultCommitter.js

@@ -555,15 +555,15 @@ async function writeToBigQuery(result, name, dateContext, category, logger, isAl
         
         const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
         
-        // Use streaming inserts for alert computations (immediate, time-sensitive)
-        // Use load jobs for non-alert computations (batched, free)
-        const { insertRows: insertRowsLoadJob, insertRowsStreaming } = require('../../core/utils/bigquery_utils');
+        // Use MERGE operation to overwrite existing results (by date + computation_name + category)
+        // This ensures re-running a computation overwrites the old result
+        // Key fields: date, computation_name, category (ignoring created_at)
+        const { insertRowsWithMerge } = require('../../core/utils/bigquery_utils');
+        const keyFields = ['date', 'computation_name', 'category'];
         
-        if (isAlertComputation) {
-            await insertRowsStreaming(datasetId, 'computation_results', [row], logger);
-        } else {
-            await insertRowsLoadJob(datasetId, 'computation_results', [row], logger);
-        }
+        // For alert computations, we still want to use MERGE but it will use load jobs (free)
+        // This ensures overwrites work correctly for both alert and non-alert computations
+        await insertRowsWithMerge(datasetId, 'computation_results', [row], keyFields, logger);
         
     } catch (error) {
         // Log but don't throw - BigQuery write failure shouldn't break Firestore writes
@@ -663,8 +663,29 @@ function calculateFirestoreBytes(value) {
 }
 
 function calculateExpirationDate(dateStr, ttlDays) {
+    // Validate inputs
+    if (!dateStr || typeof dateStr !== 'string') {
+        return null; // Invalid date string
+    }
+    
+    if (ttlDays === undefined || ttlDays === null || isNaN(Number(ttlDays))) {
+        return null; // Invalid TTL days
+    }
+    
     const base = new Date(dateStr);
-    base.setDate(base.getDate() + ttlDays);
+    
+    // Check if date is valid (invalid dates have NaN getTime())
+    if (isNaN(base.getTime())) {
+        return null; // Invalid date
+    }
+    
+    base.setDate(base.getDate() + Number(ttlDays));
+    
+    // Double-check the result is still valid
+    if (isNaN(base.getTime())) {
+        return null; // Resulting date is invalid
+    }
+    
     return base;
 }
 

package/functions/core/utils/bigquery_utils.js

@@ -196,27 +196,17 @@ async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger =
             logger.log('INFO', `[BigQuery] Loaded ${validRows.length} rows into temp table ${tempTableId} using LOAD JOB (free)`);
         }
         
-        // Use MERGE to insert only new rows (SQL-native deduplication)
+        // Use MERGE to insert new rows or update existing rows (SQL-native deduplication/overwrite)
         // This is more efficient than checking in JavaScript
         const mergeConditions = keyFields.map(f => `target.${f} = source.${f}`).join(' AND ');
-        const mergeQuery = `
-            MERGE \`${tablePath}\` AS target
-            USING \`${tempTablePath}\` AS source
-            ON ${mergeConditions}
-            WHEN NOT MATCHED THEN
-                INSERT ROW
-        `;
-        
-        await query(mergeQuery, {}, logger);
         
-        // Get count of rows that were actually inserted (not matched = new rows)
-        // We can't directly get this from MERGE, so we'll query the temp table
-        // and subtract what already exists
-        const [existingBefore] = await query(`SELECT COUNT(*) as cnt FROM \`${tablePath}\``, {}, logger);
-        const countBefore = existingBefore[0]?.cnt || 0;
+        // Build UPDATE clause - update all non-key fields
+        const allFields = schema.map(f => f.name);
+        const nonKeyFields = allFields.filter(f => !keyFields.includes(f));
+        const updateClause = nonKeyFields.map(f => `${f} = source.${f}`).join(', ');
         
-        // Actually, MERGE doesn't return row count directly
-        // Let's use a different approach - query what was inserted
+        // Count rows that will be inserted (don't exist in target) vs updated (already exist)
+        // Query BEFORE the MERGE to get accurate counts
         const [insertedCountResult] = await query(`
             SELECT COUNT(*) as inserted
             FROM \`${tempTablePath}\` AS source
@@ -227,12 +217,30 @@ async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger =
         `, {}, logger);
         
         const rowsInserted = insertedCountResult[0]?.inserted || 0;
+        const rowsUpdated = validRows.length - rowsInserted;
+        
+        // Now perform the MERGE (inserts new rows, updates existing rows)
+        const mergeQuery = `
+            MERGE \`${tablePath}\` AS target
+            USING \`${tempTablePath}\` AS source
+            ON ${mergeConditions}
+            WHEN MATCHED THEN
+                UPDATE SET ${updateClause}
+            WHEN NOT MATCHED THEN
+                INSERT ROW
+        `;
+        
+        await query(mergeQuery, {}, logger);
         
         // Drop temp table
         await tempTable.delete();
         
         if (logger) {
-            logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
+            if (rowsUpdated > 0) {
+                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted, ${rowsUpdated} existing rows updated in ${tablePath}`);
+            } else {
+                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
+            }
         }
         
         return rowsInserted;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.715",
+  "version": "1.0.716",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [