bulltrackers-module 1.0.715 → 1.0.717

package/functions/computation-system/persistence/ResultCommitter.js

@@ -185,6 +185,12 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 
                 runMetrics.io.writes += 1;
                 
+                // 4. Write to BigQuery (for analytics) - same structure as other computations
+                // Page computations store the full result object { cid1: {...}, cid2: {...}, ... } in result_data
+                await writeToBigQuery(result, name, dStr, calc.manifest.category, logger, false).catch(err => {
+                    logger.log('WARN', `[BigQuery] Failed to write page computation ${name} for ${dStr}: ${err.message}`);
+                });
+                
                 if (isFinalFlush && calc.manifest.hash) { 
                     successUpdates[name] = { 
                         hash: calc.manifest.hash, simHash: simHash, resultHash: resultHash,
@@ -555,15 +561,15 @@ async function writeToBigQuery(result, name, dateContext, category, logger, isAl
         
         const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
         
-        // Use streaming inserts for alert computations (immediate, time-sensitive)
-        // Use load jobs for non-alert computations (batched, free)
-        const { insertRows: insertRowsLoadJob, insertRowsStreaming } = require('../../core/utils/bigquery_utils');
+        // Use MERGE operation to overwrite existing results (by date + computation_name + category)
+        // This ensures re-running a computation overwrites the old result
+        // Key fields: date, computation_name, category (ignoring created_at)
+        const { insertRowsWithMerge } = require('../../core/utils/bigquery_utils');
+        const keyFields = ['date', 'computation_name', 'category'];
         
-        if (isAlertComputation) {
-            await insertRowsStreaming(datasetId, 'computation_results', [row], logger);
-        } else {
-            await insertRowsLoadJob(datasetId, 'computation_results', [row], logger);
-        }
+        // For alert computations, we still want to use MERGE but it will use load jobs (free)
+        // This ensures overwrites work correctly for both alert and non-alert computations
+        await insertRowsWithMerge(datasetId, 'computation_results', [row], keyFields, logger);
         
     } catch (error) {
         // Log but don't throw - BigQuery write failure shouldn't break Firestore writes
@@ -663,8 +669,29 @@ function calculateFirestoreBytes(value) {
 }
 
 function calculateExpirationDate(dateStr, ttlDays) {
+    // Validate inputs
+    if (!dateStr || typeof dateStr !== 'string') {
+        return null; // Invalid date string
+    }
+    
+    if (ttlDays === undefined || ttlDays === null || isNaN(Number(ttlDays))) {
+        return null; // Invalid TTL days
+    }
+    
     const base = new Date(dateStr);
-    base.setDate(base.getDate() + ttlDays);
+    
+    // Check if date is valid (invalid dates have NaN getTime())
+    if (isNaN(base.getTime())) {
+        return null; // Invalid date
+    }
+    
+    base.setDate(base.getDate() + Number(ttlDays));
+    
+    // Double-check the result is still valid
+    if (isNaN(base.getTime())) {
+        return null; // Resulting date is invalid
+    }
+    
     return base;
 }
 

package/functions/core/utils/bigquery_utils.js

@@ -196,27 +196,17 @@ async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger =
             logger.log('INFO', `[BigQuery] Loaded ${validRows.length} rows into temp table ${tempTableId} using LOAD JOB (free)`);
         }
         
-        // Use MERGE to insert only new rows (SQL-native deduplication)
+        // Use MERGE to insert new rows or update existing rows (SQL-native deduplication/overwrite)
         // This is more efficient than checking in JavaScript
         const mergeConditions = keyFields.map(f => `target.${f} = source.${f}`).join(' AND ');
-        const mergeQuery = `
-            MERGE \`${tablePath}\` AS target
-            USING \`${tempTablePath}\` AS source
-            ON ${mergeConditions}
-            WHEN NOT MATCHED THEN
-                INSERT ROW
-        `;
-        
-        await query(mergeQuery, {}, logger);
         
-        // Get count of rows that were actually inserted (not matched = new rows)
-        // We can't directly get this from MERGE, so we'll query the temp table
-        // and subtract what already exists
-        const [existingBefore] = await query(`SELECT COUNT(*) as cnt FROM \`${tablePath}\``, {}, logger);
-        const countBefore = existingBefore[0]?.cnt || 0;
+        // Build UPDATE clause - update all non-key fields
+        const allFields = schema.map(f => f.name);
+        const nonKeyFields = allFields.filter(f => !keyFields.includes(f));
+        const updateClause = nonKeyFields.map(f => `${f} = source.${f}`).join(', ');
         
-        // Actually, MERGE doesn't return row count directly
-        // Let's use a different approach - query what was inserted
+        // Count rows that will be inserted (don't exist in target) vs updated (already exist)
+        // Query BEFORE the MERGE to get accurate counts
         const [insertedCountResult] = await query(`
             SELECT COUNT(*) as inserted
             FROM \`${tempTablePath}\` AS source
@@ -227,12 +217,30 @@ async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger =
         `, {}, logger);
         
         const rowsInserted = insertedCountResult[0]?.inserted || 0;
+        const rowsUpdated = validRows.length - rowsInserted;
+        
+        // Now perform the MERGE (inserts new rows, updates existing rows)
+        const mergeQuery = `
+            MERGE \`${tablePath}\` AS target
+            USING \`${tempTablePath}\` AS source
+            ON ${mergeConditions}
+            WHEN MATCHED THEN
+                UPDATE SET ${updateClause}
+            WHEN NOT MATCHED THEN
+                INSERT ROW
+        `;
+        
+        await query(mergeQuery, {}, logger);
         
         // Drop temp table
         await tempTable.delete();
         
         if (logger) {
-            logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
+            if (rowsUpdated > 0) {
+                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted, ${rowsUpdated} existing rows updated in ${tablePath}`);
+            } else {
+                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
+            }
         }
         
         return rowsInserted;
@@ -617,6 +625,11 @@ const SCHEMAS = {
         { name: 'instrument_id', type: 'INT64', mode: 'REQUIRED' },
         { name: 'insights_data', type: 'JSON', mode: 'REQUIRED' },
         { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    ticker_mappings: [
+        { name: 'instrument_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'ticker', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
     ]
 };
 
@@ -804,6 +817,27 @@ async function ensureInstrumentInsightsTable(logger = null) {
     );
 }
 
+/**
+ * Ensure ticker_mappings table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureTickerMappingsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'ticker_mappings';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            clusterFields: ['instrument_id']
+        },
+        logger
+    );
+}
+
 /**
  * Query portfolio data from BigQuery
  * @param {string} dateStr - Date string (YYYY-MM-DD)
@@ -1623,6 +1657,60 @@ async function queryAssetPrices(startDateStr = null, endDateStr = null, instrume
     }
 }
 
+/**
+ * Query ticker mappings from BigQuery
+ * Returns data in format: { instrumentId: "TICKER", ... }
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Ticker mappings object, or null if not found/error
+ */
+async function queryTickerMappings(logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Ticker mappings query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.ticker_mappings`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                instrument_id,
+                ticker
+            FROM \`${tablePath}\`
+            ORDER BY instrument_id ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying ticker mappings from ${tablePath}`);
+        }
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No ticker mappings found in ${tablePath}`);
+            return null;
+        }
+        
+        // Transform to expected format: { instrumentId: "TICKER" }
+        const mappings = {};
+        for (const row of rows) {
+            mappings[String(row.instrument_id)] = row.ticker;
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${Object.keys(mappings).length} ticker mappings from ${tablePath}`);
+        }
+        
+        return mappings;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Ticker mappings query failed for ${tablePath}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
 module.exports = {
     getBigQueryClient,
     getOrCreateDataset,
@@ -1639,6 +1727,7 @@ module.exports = {
     ensurePIMasterListTable,
     ensurePIRankingsTable,
     ensureInstrumentInsightsTable,
+    ensureTickerMappingsTable,
     queryPortfolioData,
     queryHistoryData,
     querySocialData,
@@ -1646,6 +1735,7 @@ module.exports = {
     queryPIMasterList,
     queryPIRankings,
     queryInstrumentInsights,
+    queryTickerMappings,
     queryComputationResult,
     queryComputationResultsRange,
     checkExistingRows,

package/index.js

@@ -65,6 +65,8 @@ const { runPopularInvestorFetch }                         = require('./functions
 const { backfillTaskEngineData }                          = require('./functions/maintenance/backfill-task-engine-data/index');
 const { backfillPIMasterListRankings }                    = require('./functions/maintenance/backfill-pi-master-list-rankings/index');
 const { backfillInstrumentInsights }                      = require('./functions/maintenance/backfill-instrument-insights/index');
+const { backfillTickerMappings }                          = require('./functions/maintenance/backfill-ticker-mappings/index');
+const { backfillPriceData }                               = require('./functions/maintenance/backfill-price-data-from-firestore/index');
 
 // Alert System
 const { handleAlertTrigger, handleComputationResultWrite, checkAndSendAllClearNotifications } = require('./functions/alert-system/index');
@@ -135,7 +137,9 @@ const maintenance = {
   // [NEW] BigQuery backfills
   backfillTaskEngineData,
   backfillPIMasterListRankings,
-  backfillInstrumentInsights
+  backfillInstrumentInsights,
+  backfillTickerMappings,
+  backfillPriceData
 };
 
 const proxy = { handlePost };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.715",
+  "version": "1.0.717",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [