bulltrackers-module 1.0.710 → 1.0.713

package/functions/core/utils/bigquery_utils.js

@@ -0,0 +1,1655 @@
+/**
+ * @fileoverview BigQuery utility functions for BullTrackers
+ * Handles table creation, data insertion, and querying with automatic schema management
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+// Singleton BigQuery client
+let bigqueryClient = null;
+
+/**
+ * Get or create BigQuery client
+ */
+function getBigQueryClient() {
+    if (!bigqueryClient) {
+        const projectId = process.env.GCP_PROJECT_ID || 'stocks-12345';
+        bigqueryClient = new BigQuery({ projectId });
+    }
+    return bigqueryClient;
+}
+
+/**
+ * Get dataset reference, creating it if it doesn't exist
+ * @param {string} datasetId - Dataset ID (e.g., 'bulltrackers_data')
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Dataset>}
+ */
+async function getOrCreateDataset(datasetId, logger = null) {
+    const bigquery = getBigQueryClient();
+    const dataset = bigquery.dataset(datasetId);
+    
+    try {
+        const [exists] = await dataset.exists();
+        if (!exists) {
+            if (logger) logger.log('INFO', `[BigQuery] Creating dataset: ${datasetId}`);
+            await dataset.create({
+                location: 'europe-west1', // Match your Cloud Functions region
+                description: 'BullTrackers analytical data'
+            });
+            if (logger) logger.log('INFO', `[BigQuery] Dataset ${datasetId} created successfully`);
+        }
+        return dataset;
+    } catch (error) {
+        if (logger) logger.log('ERROR', `[BigQuery] Error with dataset ${datasetId}: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Ensure a table exists with the given schema, creating it if necessary
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {Array} schema - BigQuery schema array
+ * @param {object} options - Additional options (partitionField, clusterFields)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureTableExists(datasetId, tableId, schema, options = {}, logger = null) {
+    const dataset = await getOrCreateDataset(datasetId, logger);
+    const table = dataset.table(tableId);
+    
+    try {
+        const [exists] = await table.exists();
+        if (!exists) {
+            if (logger) logger.log('INFO', `[BigQuery] Creating table: ${datasetId}.${tableId}`);
+            
+            const tableOptions = {
+                schema: schema,
+                description: `Auto-created table for ${tableId}`
+            };
+            
+            // Add partitioning if specified
+            if (options.partitionField) {
+                tableOptions.timePartitioning = {
+                    field: options.partitionField,
+                    type: 'DAY' // Partition by day
+                };
+            }
+            
+            // Add clustering if specified
+            if (options.clusterFields && options.clusterFields.length > 0) {
+                tableOptions.clustering = {
+                    fields: options.clusterFields
+                };
+            }
+            
+            await table.create(tableOptions);
+            if (logger) logger.log('INFO', `[BigQuery] Table ${datasetId}.${tableId} created successfully`);
+        } else {
+            // Table exists - verify schema matches (optional, can be enhanced)
+            if (logger) logger.log('DEBUG', `[BigQuery] Table ${datasetId}.${tableId} already exists`);
+        }
+        
+        return table;
+    } catch (error) {
+        if (logger) logger.log('ERROR', `[BigQuery] Error ensuring table ${datasetId}.${tableId}: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Insert rows using BigQuery MERGE statement (handles duplicates natively via SQL)
+ * More efficient than checking then inserting - BigQuery handles deduplication in SQL
+ * Uses a temporary table and MERGE statement for atomic deduplication
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {Array} rows - Array of row objects
+ * @param {Array} keyFields - Fields that form unique key (e.g., ['date', 'user_id', 'user_type'])
+ * @param {object} logger - Logger instance
+ * @returns {Promise<number>} Number of rows actually inserted (not duplicates)
+ */
+async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger = null) {
+    if (!rows || rows.length === 0) {
+        if (logger) logger.log('WARN', `[BigQuery] No rows to merge into ${datasetId}.${tableId}`);
+        return 0;
+    }
+    
+    const MAX_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit
+    const validRows = rows.filter(row => {
+        const rowSize = JSON.stringify(row).length;
+        return rowSize <= MAX_ROW_SIZE;
+    });
+    
+    if (validRows.length === 0) {
+        if (logger) logger.log('WARN', `[BigQuery] All rows too large for MERGE into ${datasetId}.${tableId}`);
+        return 0;
+    }
+    
+    try {
+        const tablePath = `${datasetId}.${tableId}`;
+        const keyFieldsStr = keyFields.join(', ');
+        const tempTableId = `${tableId}_temp_${Date.now()}`;
+        const tempTablePath = `${datasetId}.${tempTableId}`;
+        
+        // Get table schema
+        const dataset = await getOrCreateDataset(datasetId, logger);
+        const table = dataset.table(tableId);
+        const [tableMetadata] = await table.getMetadata();
+        const schema = tableMetadata.schema.fields;
+        
+        // Create temp table with same schema
+        const tempTable = dataset.table(tempTableId);
+        await tempTable.create({
+            schema: schema,
+            description: 'Temporary table for merge operation'
+        });
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] Created temp table ${tempTableId} for MERGE operation`);
+        }
+        
+        // Insert all rows into temp table using LOAD JOB (free, not streaming)
+        // Write to temporary file (load jobs require a file, not a stream)
+        const tempFile = path.join(os.tmpdir(), `bigquery_merge_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+        const ndjson = validRows.map(r => JSON.stringify(r)).join('\n');
+        
+        try {
+            fs.writeFileSync(tempFile, ndjson, 'utf8');
+            
+            // Load into temp table using load job (FREE) from temp file
+            // Use createLoadJob to get a Job object we can wait on
+            const [loadJob] = await tempTable.createLoadJob(tempFile, {
+                sourceFormat: 'NEWLINE_DELIMITED_JSON',
+                writeDisposition: 'WRITE_APPEND',
+                autodetect: false // Use existing table schema
+            });
+            
+            // [FIX] Use native job.promise() instead of custom polling
+            // This automatically polls and waits for completion
+            await loadJob.promise();
+            
+            // Get job metadata to check for errors
+            const [jobMetadata] = await loadJob.getMetadata();
+            
+            // Check for errors
+            if (jobMetadata.status?.errorResult) {
+                throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
+            }
+        } finally {
+            // Clean up temp file
+            try {
+                if (fs.existsSync(tempFile)) {
+                    fs.unlinkSync(tempFile);
+                }
+            } catch (cleanupError) {
+                if (logger) {
+                    logger.log('WARN', `[BigQuery] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
+                }
+            }
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] Loaded ${validRows.length} rows into temp table ${tempTableId} using LOAD JOB (free)`);
+        }
+        
+        // Use MERGE to insert only new rows (SQL-native deduplication)
+        // This is more efficient than checking in JavaScript
+        const mergeConditions = keyFields.map(f => `target.${f} = source.${f}`).join(' AND ');
+        const mergeQuery = `
+            MERGE \`${tablePath}\` AS target
+            USING \`${tempTablePath}\` AS source
+            ON ${mergeConditions}
+            WHEN NOT MATCHED THEN
+                INSERT ROW
+        `;
+        
+        await query(mergeQuery, {}, logger);
+        
+        // Get count of rows that were actually inserted (not matched = new rows)
+        // We can't directly get this from MERGE, so we'll query the temp table
+        // and subtract what already exists
+        const [existingBefore] = await query(`SELECT COUNT(*) as cnt FROM \`${tablePath}\``, {}, logger);
+        const countBefore = existingBefore[0]?.cnt || 0;
+        
+        // Actually, MERGE doesn't return row count directly
+        // Let's use a different approach - query what was inserted
+        const [insertedCountResult] = await query(`
+            SELECT COUNT(*) as inserted
+            FROM \`${tempTablePath}\` AS source
+            WHERE NOT EXISTS (
+                SELECT 1 FROM \`${tablePath}\` AS target
+                WHERE ${mergeConditions}
+            )
+        `, {}, logger);
+        
+        const rowsInserted = insertedCountResult[0]?.inserted || 0;
+        
+        // Drop temp table
+        await tempTable.delete();
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
+        }
+        
+        return rowsInserted;
+    } catch (error) {
+        const errorDetails = {
+            message: error.message,
+            code: error.code,
+            errors: error.errors
+        };
+        if (logger) {
+            logger.log('ERROR', `[BigQuery] MERGE failed for ${datasetId}.${tableId}:`, JSON.stringify(errorDetails, null, 2));
+        }
+        // Try to clean up temp table if it exists
+        try {
+            const dataset = await getOrCreateDataset(datasetId, logger);
+            const tempTableId = `${tableId}_temp_${Date.now()}`;
+            await dataset.table(tempTableId).delete().catch(() => {}); // Ignore cleanup errors
+        } catch {}
+        throw error;
+    }
+}
+
+/**
+ * Insert rows into a BigQuery table using LOAD JOB (FREE, not streaming inserts)
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {Array} rows - Array of row objects
+ * @param {object} logger - Logger instance
+ * @returns {Promise<void>}
+ */
+async function insertRows(datasetId, tableId, rows, logger = null) {
+    if (!rows || rows.length === 0) {
+        if (logger) logger.log('WARN', `[BigQuery] No rows to insert into ${datasetId}.${tableId}`);
+        return;
+    }
+    
+    const MAX_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit (BigQuery limit is 10MB)
+    const MAX_LOAD_JOB_SIZE = 100 * 1024 * 1024; // 100MB per load job (BigQuery limit is 10GB, but we batch smaller)
+    
+    try {
+        const dataset = await getOrCreateDataset(datasetId, logger);
+        const table = dataset.table(tableId);
+        
+        // Filter out rows that are too large
+        const validRows = [];
+        const skippedRows = [];
+        
+        for (const row of rows) {
+            const rowSize = JSON.stringify(row).length;
+            if (rowSize > MAX_ROW_SIZE) {
+                skippedRows.push({ size: rowSize, row: Object.keys(row) });
+                if (logger) {
+                    logger.log('WARN', `[BigQuery] Skipping row in ${datasetId}.${tableId}: Row too large (${(rowSize/1024/1024).toFixed(2)}MB, limit: ${(MAX_ROW_SIZE/1024/1024).toFixed(2)}MB)`);
+                }
+            } else {
+                validRows.push(row);
+            }
+        }
+        
+        if (skippedRows.length > 0 && logger) {
+            logger.log('WARN', `[BigQuery] Skipped ${skippedRows.length} rows in ${datasetId}.${tableId} due to size limits`);
+        }
+        
+        if (validRows.length === 0) {
+            if (logger) logger.log('WARN', `[BigQuery] No valid rows to insert into ${datasetId}.${tableId} (all rows too large)`);
+            return;
+        }
+        
+        // Use LOAD JOBS (free) instead of streaming inserts (expensive)
+        // Batch rows into load jobs of reasonable size
+        let totalInserted = 0;
+        let currentBatch = [];
+        let currentBatchSize = 0;
+        
+        for (let i = 0; i < validRows.length; i++) {
+            const row = validRows[i];
+            const rowSize = JSON.stringify(row).length;
+            
+            // If adding this row would exceed batch size, load current batch first
+            if (currentBatch.length > 0 && (currentBatchSize + rowSize) > MAX_LOAD_JOB_SIZE) {
+                // Write batch to temporary file (load jobs require a file, not a stream)
+                const tempFile = path.join(os.tmpdir(), `bigquery_load_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+                const ndjson = currentBatch.map(r => JSON.stringify(r)).join('\n');
+                
+                try {
+                    fs.writeFileSync(tempFile, ndjson, 'utf8');
+                    
+                    // Load current batch using load job (FREE) from temp file
+                    // Use createLoadJob to get a Job object we can wait on
+                    const [job] = await table.createLoadJob(tempFile, {
+                        sourceFormat: 'NEWLINE_DELIMITED_JSON',
+                        writeDisposition: 'WRITE_APPEND',
+                        autodetect: false // Use existing table schema
+                    });
+                    
+                    // [FIX] Use native job.promise() instead of custom polling
+                    // This automatically polls and waits for completion
+                    await job.promise();
+                    
+                    // Get job metadata to check for errors and get row count
+                    const [jobMetadata] = await job.getMetadata();
+                    
+                    // Check for errors
+                    if (jobMetadata.status?.errorResult) {
+                        throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
+                    }
+                    
+                    const rowsLoaded = jobMetadata.statistics?.load?.outputRows || currentBatch.length;
+                    totalInserted += rowsLoaded;
+                    
+                    if (logger) {
+                        logger.log('INFO', `[BigQuery] Load job completed: ${rowsLoaded} rows loaded into ${datasetId}.${tableId}`);
+                    }
+                } finally {
+                    // Clean up temp file
+                    try {
+                        if (fs.existsSync(tempFile)) {
+                            fs.unlinkSync(tempFile);
+                        }
+                    } catch (cleanupError) {
+                        if (logger) {
+                            logger.log('WARN', `[BigQuery] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
+                        }
+                    }
+                }
+                
+                // Reset batch
+                currentBatch = [];
+                currentBatchSize = 0;
+            }
+            
+            // Add row to current batch
+            currentBatch.push(row);
+            currentBatchSize += rowSize;
+        }
+        
+        // Load remaining batch
+        if (currentBatch.length > 0) {
+            // Write batch to temporary file (load jobs require a file, not a stream)
+            const tempFile = path.join(os.tmpdir(), `bigquery_load_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+            const ndjson = currentBatch.map(r => JSON.stringify(r)).join('\n');
+            
+            try {
+                fs.writeFileSync(tempFile, ndjson, 'utf8');
+                
+                // Load using load job (FREE) from temp file
+                // Use createLoadJob to get a Job object we can wait on
+                const [job] = await table.createLoadJob(tempFile, {
+                    sourceFormat: 'NEWLINE_DELIMITED_JSON',
+                    writeDisposition: 'WRITE_APPEND',
+                    autodetect: false // Use existing table schema
+                });
+                
+                // Wait for job to complete using polling
+                let jobMetadata;
+                const maxAttempts = 60; // 5 minutes max (5 second intervals)
+                const pollInterval = 5000; // 5 seconds
+                
+                for (let attempt = 0; attempt < maxAttempts; attempt++) {
+                    [jobMetadata] = await job.getMetadata();
+                    const state = jobMetadata.status?.state;
+                    
+                    if (state === 'DONE') {
+                        break;
+                    }
+                    
+                    if (state === 'PENDING' || state === 'RUNNING') {
+                        // Wait before next poll
+                        await new Promise(resolve => setTimeout(resolve, pollInterval));
+                    } else {
+                        throw new Error(`Unexpected job state: ${state}`);
+                    }
+                }
+                
+                // Check if we timed out
+                if (jobMetadata.status?.state !== 'DONE') {
+                    throw new Error(`Load job did not complete within ${maxAttempts * pollInterval / 1000} seconds`);
+                }
+                
+                // Check for errors
+                if (jobMetadata.status?.errorResult) {
+                    throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
+                }
+                
+                const rowsLoaded = jobMetadata.statistics?.load?.outputRows || currentBatch.length;
+                totalInserted += rowsLoaded;
+                
+                if (logger) {
+                    logger.log('INFO', `[BigQuery] Load job completed: ${rowsLoaded} rows loaded into ${datasetId}.${tableId}`);
+                }
+            } finally {
+                // Clean up temp file
+                try {
+                    if (fs.existsSync(tempFile)) {
+                        fs.unlinkSync(tempFile);
+                    }
+                } catch (cleanupError) {
+                    if (logger) {
+                        logger.log('WARN', `[BigQuery] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
+                    }
+                }
+            }
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] Loaded ${totalInserted}/${validRows.length} rows into ${datasetId}.${tableId} using LOAD JOBS (free)${skippedRows.length > 0 ? ` (${skippedRows.length} skipped due to size)` : ''}`);
+        }
+    } catch (error) {
+        const errorDetails = {
+            message: error.message,
+            code: error.code,
+            errors: error.errors,
+            stack: error.stack
+        };
+        if (logger) {
+            logger.log('ERROR', `[BigQuery] Error loading into ${datasetId}.${tableId}:`, JSON.stringify(errorDetails, null, 2));
+        }
+        throw error;
+    }
+}
+
+/**
+ * Insert rows into BigQuery using STREAMING INSERTS (immediate, costs ~$0.05/GB)
+ * Use this for time-sensitive data like alert computations
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {Array} rows - Array of row objects
+ * @param {object} logger - Logger instance
+ * @returns {Promise<void>}
+ */
+async function insertRowsStreaming(datasetId, tableId, rows, logger = null) {
+    if (!rows || rows.length === 0) {
+        if (logger) logger.log('WARN', `[BigQuery] No rows to stream into ${datasetId}.${tableId}`);
+        return;
+    }
+    
+    const MAX_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit (BigQuery limit is 10MB)
+    const MAX_BATCH_SIZE = 100; // Streaming insert batch size
+    
+    try {
+        const dataset = await getOrCreateDataset(datasetId, logger);
+        const table = dataset.table(tableId);
+        
+        // Filter out rows that are too large
+        const validRows = rows.filter(row => {
+            const rowSize = JSON.stringify(row).length;
+            return rowSize <= MAX_ROW_SIZE;
+        });
+        
+        if (validRows.length === 0) {
+            if (logger) logger.log('WARN', `[BigQuery] No valid rows to stream into ${datasetId}.${tableId} (all rows too large)`);
+            return;
+        }
+        
+        // Stream inserts in batches
+        let insertedCount = 0;
+        for (let i = 0; i < validRows.length; i += MAX_BATCH_SIZE) {
+            const batch = validRows.slice(i, i + MAX_BATCH_SIZE);
+            
+            try {
+                const [result] = await table.insert(batch);
+                
+                if (result.insertErrors && result.insertErrors.length > 0) {
+                    const errors = result.insertErrors.map(e => e.errors).flat();
+                    if (logger) logger.log('ERROR', `[BigQuery] Streaming insert errors for batch in ${datasetId}.${tableId}:`, errors);
+                    // Continue with next batch
+                } else {
+                    insertedCount += batch.length;
+                }
+            } catch (batchError) {
+                if (logger) {
+                    logger.log('WARN', `[BigQuery] Streaming insert batch failed for ${datasetId}.${tableId}: ${batchError.message}`);
+                }
+                // Continue with next batch
+            }
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] Streamed ${insertedCount}/${validRows.length} rows into ${datasetId}.${tableId} using STREAMING INSERTS`);
+        }
+    } catch (error) {
+        const errorDetails = {
+            message: error.message,
+            code: error.code,
+            errors: error.errors
+        };
+        if (logger) {
+            logger.log('ERROR', `[BigQuery] Error streaming into ${datasetId}.${tableId}:`, JSON.stringify(errorDetails, null, 2));
+        }
+        throw error;
+    }
+}
+
+/**
+ * Query BigQuery and return results
+ * @param {string} query - SQL query string
+ * @param {object} options - Query options
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Array>} Array of row objects
+ */
+async function query(query, options = {}, logger = null) {
+    const bigquery = getBigQueryClient();
+    
+    try {
+        const [rows] = await bigquery.query({
+            query: query,
+            location: 'europe-west1',
+            ...options
+        });
+        
+        if (logger) logger.log('INFO', `[BigQuery] Query returned ${rows.length} rows`);
+        return rows;
+    } catch (error) {
+        if (logger) logger.log('ERROR', `[BigQuery] Query error: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Schema definitions for BullTrackers tables
+ */
+const SCHEMAS = {
+    computation_results: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'category', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'result_data', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'metadata', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'created_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    portfolio_snapshots: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'user_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'user_type', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'portfolio_data', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    trade_history_snapshots: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'user_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'user_type', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'history_data', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    social_post_snapshots: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'user_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'user_type', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'posts_data', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    asset_prices: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'instrument_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'ticker', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'price', type: 'FLOAT64', mode: 'REQUIRED' },
+        { name: 'open', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'high', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'low', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'close', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'volume', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    pi_master_list: [
+        { name: 'cid', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'username', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'first_seen_at', type: 'TIMESTAMP', mode: 'REQUIRED' },
+        { name: 'last_seen_at', type: 'TIMESTAMP', mode: 'REQUIRED' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    pi_rankings: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'pi_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'username', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'rank', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'category', type: 'STRING', mode: 'NULLABLE' },
+        { name: 'rankings_data', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    instrument_insights: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'instrument_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'insights_data', type: 'JSON', mode: 'REQUIRED' },
+        { name: 'fetched_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ]
+};
+
+/**
+ * Get schema for a table
+ * @param {string} tableName - Table name
+ * @returns {Array} Schema array
+ */
+function getSchema(tableName) {
+    return SCHEMAS[tableName] || null;
+}
+
+/**
+ * Ensure computation_results table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureComputationResultsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'computation_results';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['computation_name', 'category']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure portfolio_snapshots table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePortfolioSnapshotsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'portfolio_snapshots';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['user_type', 'user_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure trade_history_snapshots table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureTradeHistorySnapshotsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'trade_history_snapshots';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['user_type', 'user_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure social_post_snapshots table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureSocialPostSnapshotsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'social_post_snapshots';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['user_type', 'user_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure asset_prices table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureAssetPricesTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'asset_prices';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['ticker', 'instrument_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure pi_master_list table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePIMasterListTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'pi_master_list';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            clusterFields: ['cid']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure pi_rankings table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePIRankingsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'pi_rankings';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['pi_id', 'category']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure instrument_insights table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureInstrumentInsightsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'instrument_insights';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['instrument_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Query portfolio data from BigQuery
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} userIds - Optional array of user IDs to filter
+ * @param {Array} userTypes - Optional array of user types to filter (e.g., ['POPULAR_INVESTOR', 'SIGNED_IN_USER'])
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Object>} Map of user_id -> portfolio_data, or null if no data/error
+ */
+async function queryPortfolioData(dateStr, userIds = null, userTypes = null, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Portfolio query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.portfolio_snapshots`;
+    
+    try {
+        // Build WHERE clause
+        const conditions = [`date = '${dateStr}'`];
+        
+        if (userIds && userIds.length > 0) {
+            const userIdList = userIds.map(id => String(id)).join(',');
+            conditions.push(`user_id IN (${userIdList})`);
+        }
+        
+        if (userTypes && userTypes.length > 0) {
+            const typeList = userTypes.map(t => `'${t.toUpperCase()}'`).join(',');
+            conditions.push(`user_type IN (${typeList})`);
+        }
+        
+        const whereClause = conditions.join(' AND ');
+        
+        const sqlQuery = `
+            SELECT 
+                user_id,
+                user_type,
+                portfolio_data,
+                fetched_at
+            FROM \`${tablePath}\`
+            WHERE ${whereClause}
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying portfolio data from ${tablePath} for date ${dateStr}${userTypes ? ` (types: ${userTypes.join(',')})` : ''}${userIds ? ` (${userIds.length} users)` : ''}`);
+        }
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No portfolio data found in ${tablePath} for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to map: user_id -> { portfolio_data, user_type, fetched_at }
+        const result = {};
+        for (const row of rows) {
+            result[String(row.user_id)] = {
+                portfolio_data: row.portfolio_data || {},
+                user_type: row.user_type,
+                fetched_at: row.fetched_at
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${rows.length} portfolio records from ${tablePath} for ${dateStr}`);
+        }
+        
+        return result;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Portfolio query failed for ${tablePath} (${dateStr}): ${error.message}`);
+        }
+        return null; // Return null to trigger Firestore fallback
+    }
+}
+
+/**
+ * Query trade history data from BigQuery
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} userIds - Optional array of user IDs to filter
+ * @param {Array} userTypes - Optional array of user types to filter
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Object>} Map of user_id -> history_data, or null if no data/error
+ */
+async function queryHistoryData(dateStr, userIds = null, userTypes = null, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] History query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.trade_history_snapshots`;
+    
+    try {
+        const conditions = [`date = '${dateStr}'`];
+        
+        if (userIds && userIds.length > 0) {
+            const userIdList = userIds.map(id => String(id)).join(',');
+            conditions.push(`user_id IN (${userIdList})`);
+        }
+        
+        if (userTypes && userTypes.length > 0) {
+            const typeList = userTypes.map(t => `'${t.toUpperCase()}'`).join(',');
+            conditions.push(`user_type IN (${typeList})`);
+        }
+        
+        const whereClause = conditions.join(' AND ');
+        
+        const sqlQuery = `
+            SELECT 
+                user_id,
+                user_type,
+                history_data,
+                fetched_at
+            FROM \`${tablePath}\`
+            WHERE ${whereClause}
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying trade history from ${tablePath} for date ${dateStr}${userTypes ? ` (types: ${userTypes.join(',')})` : ''}${userIds ? ` (${userIds.length} users)` : ''}`);
+        }
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No history data found in ${tablePath} for ${dateStr}`);
+            return null;
+        }
+        
+        const result = {};
+        for (const row of rows) {
+            result[String(row.user_id)] = {
+                history_data: row.history_data || {},
+                user_type: row.user_type,
+                fetched_at: row.fetched_at
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${rows.length} history records from ${tablePath} for ${dateStr}`);
+        }
+        
+        return result;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] History query failed for ${tablePath} (${dateStr}): ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Check which rows already exist in BigQuery (for deduplication)
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} rows - Array of rows to check (must have user_id and user_type)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Set>} Set of existing keys as "date|user_id|user_type"
+ */
+async function checkExistingRows(datasetId, tableId, dateStr, rows, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false' || !rows || rows.length === 0) {
+        return new Set();
+    }
+    
+    try {
+        const tablePath = `${datasetId}.${tableId}`;
+        
+        // Extract unique user_id and user_type combinations
+        const userKeys = new Set();
+        for (const row of rows) {
+            if (row.user_id && row.user_type) {
+                userKeys.add(`${row.user_id}|${row.user_type}`);
+            }
+        }
+        
+        if (userKeys.size === 0) {
+            return new Set();
+        }
+        
+        // Build WHERE clause for user combinations
+        const conditions = [`date = '${dateStr}'`];
+        const userConditions = [];
+        for (const key of userKeys) {
+            const [userId, userType] = key.split('|');
+            userConditions.push(`(user_id = ${userId} AND user_type = '${userType}')`);
+        }
+        conditions.push(`(${userConditions.join(' OR ')})`);
+        
+        const whereClause = conditions.join(' AND ');
+        
+        const sqlQuery = `
+            SELECT 
+                user_id,
+                user_type
+            FROM \`${tablePath}\`
+            WHERE ${whereClause}
+        `;
+        
+        if (logger) {
+            logger.log('DEBUG', `[BigQuery] Checking for existing rows in ${tablePath} for ${dateStr} (${userKeys.size} unique users)`);
+        }
+        
+        const existingRows = await query(sqlQuery, {}, logger);
+        
+        // Build set of existing keys
+        const existingKeys = new Set();
+        for (const row of existingRows) {
+            existingKeys.add(`${row.user_id}|${row.user_type}`);
+        }
+        
+        if (logger && existingKeys.size > 0) {
+            logger.log('INFO', `[BigQuery] Found ${existingKeys.size} existing rows in ${tablePath} for ${dateStr}, will skip duplicates`);
+        }
+        
+        return existingKeys;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Error checking existing rows in ${datasetId}.${tableId}: ${error.message}`);
+        }
+        // On error, return empty set (will attempt insert, might create duplicates but safer than skipping)
+        return new Set();
+    }
+}
+
+/**
+ * Query social post data from BigQuery
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Array} userIds - Optional array of user IDs to filter
+ * @param {Array} userTypes - Optional array of user types to filter
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Object>} Map of user_id -> posts_data, or null if no data/error
+ */
+async function querySocialData(dateStr, userIds = null, userTypes = null, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Social query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.social_post_snapshots`;
+    
+    try {
+        const conditions = [`date = '${dateStr}'`];
+        
+        if (userIds && userIds.length > 0) {
+            const userIdList = userIds.map(id => String(id)).join(',');
+            conditions.push(`user_id IN (${userIdList})`);
+        }
+        
+        if (userTypes && userTypes.length > 0) {
+            const typeList = userTypes.map(t => `'${t.toUpperCase()}'`).join(',');
+            conditions.push(`user_type IN (${typeList})`);
+        }
+        
+        const whereClause = conditions.join(' AND ');
+        
+        const sqlQuery = `
+            SELECT 
+                user_id,
+                user_type,
+                posts_data,
+                fetched_at
+            FROM \`${tablePath}\`
+            WHERE ${whereClause}
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying social posts from ${tablePath} for date ${dateStr}${userTypes ? ` (types: ${userTypes.join(',')})` : ''}${userIds ? ` (${userIds.length} users)` : ''}`);
+        }
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No social data found in ${tablePath} for ${dateStr}`);
+            return null;
+        }
+        
+        const result = {};
+        for (const row of rows) {
+            result[String(row.user_id)] = {
+                posts_data: row.posts_data || {},
+                user_type: row.user_type,
+                fetched_at: row.fetched_at
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${rows.length} social post records from ${tablePath} for ${dateStr}`);
+        }
+        
+        return result;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Social query failed for ${tablePath} (${dateStr}): ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query a single computation result from BigQuery for a specific date
+ * @param {string} computationName - Computation name
+ * @param {string} category - Category (e.g., 'popular-investor', 'alerts')
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Result data object, or null if not found/error
+ */
+async function queryComputationResult(computationName, category, dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Computation result query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.computation_results`;
+    
+    try {
+        const sqlQuery = `
+            SELECT result_data
+            FROM \`${tablePath}\`
+            WHERE date = '${dateStr}'
+              AND computation_name = '${computationName}'
+              AND category = '${category}'
+            ORDER BY created_at DESC
+            LIMIT 1
+        `;
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('DEBUG', `[BigQuery] No computation result found for ${computationName} (${dateStr}, ${category})`);
+            return null;
+        }
+        
+        const result = rows[0].result_data;
+        if (logger) logger.log('INFO', `[BigQuery] ✅ Retrieved computation result for ${computationName} (${dateStr})`);
+        return result;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Computation result query failed for ${computationName} (${dateStr}): ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query computation results from BigQuery for a date range
+ * @param {string} computationName - Computation name
+ * @param {string} category - Category (e.g., 'popular-investor', 'alerts')
+ * @param {string} startDateStr - Start date (YYYY-MM-DD)
+ * @param {string} endDateStr - End date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Array>} Array of {date, result_data} objects, or null if error
+ */
+async function queryComputationResultsRange(computationName, category, startDateStr, endDateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Computation results range query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.computation_results`;
+    
+    try {
+        // [FIX] Use parameterized queries to prevent SQL injection
+        const sqlQuery = `
+            SELECT 
+                date,
+                result_data,
+                category
+            FROM \`${tablePath}\`
+            WHERE computation_name = @computationName
+              AND category = @category
+              AND date BETWEEN @startDate AND @endDate
+            ORDER BY date DESC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying computation results from ${tablePath} for ${computationName} (${category}) from ${startDateStr} to ${endDateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                computationName: computationName,
+                category: category,
+                startDate: startDateStr,
+                endDate: endDateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No computation results found in ${tablePath} for ${computationName} in date range`);
+            return [];
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${rows.length} computation result records from ${tablePath} for ${computationName}`);
+        }
+        
+        return rows.map(row => ({
+            date: row.date,
+            data: row.result_data || {},
+            category: row.category
+        }));
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Computation results range query failed for ${tablePath}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Remove duplicate rows from a BigQuery table (keeps the most recent row per unique key)
+ * This is cheaper than checking duplicates before each insert for large backfills
+ * Uses CREATE OR REPLACE to overwrite the table with deduplicated data
+ * @param {string} datasetId - Dataset ID
+ * @param {string} tableId - Table ID
+ * @param {string} dateField - Date field name (e.g., 'date')
+ * @param {Array} keyFields - Array of field names that form the unique key (e.g., ['user_id', 'user_type'])
+ * @param {object} logger - Logger instance
+ * @returns {Promise<number>} Number of duplicates removed
+ */
+async function removeDuplicates(datasetId, tableId, dateField, keyFields, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Deduplication skipped (BIGQUERY_ENABLED=false)');
+        return 0;
+    }
+    
+    try {
+        const tablePath = `${datasetId}.${tableId}`;
+        const keyFieldsStr = keyFields.join(', ');
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔄 Deduplicating ${tablePath} by (${keyFieldsStr})`);
+        }
+        
+        // Get original count
+        const [originalCountResult] = await query(`SELECT COUNT(*) as cnt FROM \`${tablePath}\``, {}, logger);
+        const originalCount = originalCountResult[0]?.cnt || 0;
+        
+        // Check for duplicates before deduplication (for logging)
+        // Use CONCAT to create a composite key for COUNT(DISTINCT)
+        const keyConcat = `${dateField}, '-', ${keyFieldsStr.split(', ').join(", '-', ")}`;
+        const duplicateCheckQuery = `
+            SELECT 
+                COUNT(*) as total_rows,
+                COUNT(DISTINCT CONCAT(${keyConcat})) as unique_keys
+            FROM \`${tablePath}\`
+        `;
+        const [duplicateInfo] = await query(duplicateCheckQuery, {}, logger);
+        const uniqueKeys = duplicateInfo[0]?.unique_keys || 0;
+        const duplicateCount = originalCount - uniqueKeys;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] Before deduplication: ${originalCount} total rows, ${uniqueKeys} unique keys, ${duplicateCount} duplicates expected`);
+        }
+        
+        // Get table metadata to preserve partitioning and clustering
+        const dataset = await getOrCreateDataset(datasetId, logger);
+        const table = dataset.table(tableId);
+        const [tableMetadata] = await table.getMetadata();
+        const partitioning = tableMetadata.timePartitioning;
+        const clustering = tableMetadata.clustering;
+        
+        // Build partitioning clause
+        let partitionClause = '';
+        if (partitioning && partitioning.field) {
+            partitionClause = `PARTITION BY ${partitioning.field}`;
+        }
+        
+        // Build clustering clause
+        let clusterClause = '';
+        if (clustering && clustering.fields && clustering.fields.length > 0) {
+            clusterClause = `CLUSTER BY ${clustering.fields.join(', ')}`;
+        }
+        
+        // [FIX] Use CREATE OR REPLACE TABLE instead of DELETE + INSERT
+        // This is atomic, metadata-based swap that is free/cheap and instant
+        // Strategy:
+        // 1. For rows with same (date, user_id, user_type), keep the one with latest fetched_at
+        // 2. If multiple rows have same (date, user_id, user_type, fetched_at), keep one (they're effectively identical)
+        // The ORDER BY with fetched_at DESC ensures we keep the most recent snapshot
+        const createReplaceQuery = `
+            CREATE OR REPLACE TABLE \`${tablePath}\`
+            ${partitionClause}
+            ${clusterClause}
+            AS
+            SELECT * EXCEPT(row_num)
+            FROM (
+                SELECT 
+                    *,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY ${dateField}, ${keyFieldsStr}
+                        ORDER BY fetched_at DESC
+                    ) AS row_num
+                FROM \`${tablePath}\`
+            )
+            WHERE row_num = 1
+        `;
+        
+        await query(createReplaceQuery, {}, logger);
+        
+        // Get deduplicated count
+        const [dedupedCountResult] = await query(`SELECT COUNT(*) as cnt FROM \`${tablePath}\``, {}, logger);
+        const dedupedCount = dedupedCountResult[0]?.cnt || 0;
+        
+        const duplicatesRemoved = originalCount - dedupedCount;
+        
+        if (logger) {
+            if (duplicatesRemoved > 0) {
+                logger.log('INFO', `[BigQuery] ✅ Removed ${duplicatesRemoved} duplicate rows from ${tablePath} (${originalCount} → ${dedupedCount})`);
+            } else {
+                logger.log('INFO', `[BigQuery] ✅ No duplicates found in ${tablePath}`);
+            }
+        }
+        
+        return duplicatesRemoved;
+    } catch (error) {
+        if (logger) {
+            logger.log('ERROR', `[BigQuery] Deduplication failed for ${datasetId}.${tableId}: ${error.message}`);
+        }
+        throw error;
+    }
+}
+
+/**
+ * Query Popular Investor master list from BigQuery
+ * Returns data in format: { cid: { cid, username, firstSeenAt, lastSeenAt } }
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object>} Master list map in format { cid: { cid, username, firstSeenAt, lastSeenAt } }
+ */
+async function queryPIMasterList(logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] PI master list query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.pi_master_list`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                cid,
+                username,
+                first_seen_at,
+                last_seen_at
+            FROM \`${tablePath}\`
+            ORDER BY cid
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying PI master list from ${tablePath}`);
+        }
+        
+        const rows = await query(sqlQuery, {}, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No PI master list found in ${tablePath}`);
+            return {};
+        }
+        
+        // Transform to expected format: { cid: { cid, username, firstSeenAt, lastSeenAt } }
+        const masterList = {};
+        for (const row of rows) {
+            const cid = String(row.cid);
+            masterList[cid] = {
+                cid: cid,
+                username: row.username,
+                firstSeenAt: row.first_seen_at,
+                lastSeenAt: row.last_seen_at
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${Object.keys(masterList).length} PIs from master list`);
+        }
+        
+        return masterList;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] PI master list query failed for ${tablePath}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query instrument insights from BigQuery for a specific date
+ * Returns data in format: array of insights objects (same as Firestore)
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Array|null>} Array of insights objects, or null if not found/error
+ */
+async function queryInstrumentInsights(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Instrument insights query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.instrument_insights`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                instrument_id,
+                insights_data
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY instrument_id ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying instrument insights from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No instrument insights found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: array of insights objects
+        // insights_data is already a JSON object, so we can use it directly
+        const insights = rows.map(row => row.insights_data);
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${insights.length} instrument insights for ${dateStr}`);
+        }
+        
+        return insights;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Instrument insights query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query Popular Investor rankings from BigQuery for a specific date
+ * Returns data in format matching Firestore structure (Items array)
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Rankings data with Items array, or null if not found/error
+ */
+async function queryPIRankings(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] PI rankings query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.pi_rankings`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                pi_id,
+                username,
+                rank,
+                category,
+                rankings_data
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY rank ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying PI rankings from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No PI rankings found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: { Items: [...], TotalRows: N }
+        // Use rankings_data if available (full item), otherwise reconstruct from fields
+        const items = rows.map(row => {
+            if (row.rankings_data) {
+                return row.rankings_data; // Full item data stored as JSON
+            } else {
+                // Reconstruct item from individual fields
+                return {
+                    CustomerId: row.pi_id,
+                    UserName: row.username,
+                    Rank: row.rank,
+                    Category: row.category
+                };
+            }
+        });
+        
+        const result = {
+            Items: items,
+            TotalRows: items.length
+        };
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ${items.length} PI rankings for ${dateStr}`);
+        }
+        
+        return result;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] PI rankings query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query all asset prices from BigQuery
+ * Returns data in format: { instrumentId: { "YYYY-MM-DD": price, ... } }
+ * @param {string} startDateStr - Start date (YYYY-MM-DD), optional
+ * @param {string} endDateStr - End date (YYYY-MM-DD), optional
+ * @param {Array} instrumentIds - Optional array of instrument IDs to filter
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object>} Price data map in format { instrumentId: { "YYYY-MM-DD": price } }
+ */
+async function queryAssetPrices(startDateStr = null, endDateStr = null, instrumentIds = null, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Asset prices query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.asset_prices`;
+    
+    try {
+        // Build WHERE clause
+        const conditions = [];
+        
+        if (startDateStr && endDateStr) {
+            conditions.push(`date BETWEEN @startDate AND @endDate`);
+        } else if (startDateStr) {
+            conditions.push(`date >= @startDate`);
+        } else if (endDateStr) {
+            conditions.push(`date <= @endDate`);
+        }
+        
+        if (instrumentIds && instrumentIds.length > 0) {
+            // For IN clause with parameters, we need to use UNNEST
+            conditions.push(`instrument_id IN UNNEST(@instrumentIds)`);
+        }
+        
+        const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        
+        const sqlQuery = `
+            SELECT 
+                instrument_id,
+                date,
+                price,
+                ticker
+            FROM \`${tablePath}\`
+            ${whereClause}
+            ORDER BY instrument_id, date DESC
+        `;
+        
+        const params = {};
+        if (startDateStr) params.startDate = startDateStr;
+        if (endDateStr) params.endDate = endDateStr;
+        if (instrumentIds && instrumentIds.length > 0) {
+            params.instrumentIds = instrumentIds.map(id => parseInt(id, 10));
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying asset prices from ${tablePath}${startDateStr ? ` (${startDateStr} to ${endDateStr || 'latest'})` : ''}`);
+        }
+        
+        const rows = await query(sqlQuery, { params }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No asset prices found in ${tablePath}`);
+            return {};
+        }
+        
+        // Transform to expected format: { instrumentId: { "YYYY-MM-DD": price } }
+        const priceMap = {};
+        for (const row of rows) {
+            const instrumentId = String(row.instrument_id);
+            const dateStr = row.date; // Already in YYYY-MM-DD format from BigQuery DATE type
+            
+            if (!priceMap[instrumentId]) {
+                priceMap[instrumentId] = {};
+            }
+            
+            // Use close price if available, otherwise use price
+            priceMap[instrumentId][dateStr] = row.price || null;
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved prices for ${Object.keys(priceMap).length} instruments from ${tablePath}`);
+        }
+        
+        return priceMap;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Asset prices query failed for ${tablePath}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+module.exports = {
+    getBigQueryClient,
+    getOrCreateDataset,
+    ensureTableExists,
+    insertRows,
+    insertRowsStreaming,
+    query,
+    getSchema,
+    ensureComputationResultsTable,
+    ensurePortfolioSnapshotsTable,
+    ensureTradeHistorySnapshotsTable,
+    ensureSocialPostSnapshotsTable,
+    ensureAssetPricesTable,
+    ensurePIMasterListTable,
+    ensurePIRankingsTable,
+    ensureInstrumentInsightsTable,
+    queryPortfolioData,
+    queryHistoryData,
+    querySocialData,
+    queryAssetPrices,
+    queryPIMasterList,
+    queryPIRankings,
+    queryInstrumentInsights,
+    queryComputationResult,
+    queryComputationResultsRange,
+    checkExistingRows,
+    removeDuplicates,
+    insertRowsWithMerge,
+    SCHEMAS
+};