bulltrackers-module 1.0.710 → 1.0.712

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -2,13 +2,20 @@
  * @fileoverview Data Storage Helpers for New Collection Structure
  * 
  * Stores data to:
- * 1. Root data collections (date-based, per-user) - for computation system
+ * 1. BigQuery (date-based, per-user) - for computation system (NEW)
  * 2. User-centric collections (latest snapshot) - for fallback/quick access
  * 
+ * UPDATED: Removed root data Firestore writes, now writes to BigQuery instead
  * Uses the centralized collection registry for all paths.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
+const { 
+    ensurePortfolioSnapshotsTable, 
+    ensureTradeHistorySnapshotsTable, 
+    ensureSocialPostSnapshotsTable,
+    insertRows 
+} = require('../../core/utils/bigquery_utils');
 
 /**
  * Store portfolio data for a signed-in user
@@ -19,23 +26,38 @@ const { FieldValue } = require('@google-cloud/firestore');
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.portfolioData - Portfolio data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserPortfolioData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserPortfolioData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...portfolioData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                portfolio_data: JSON.stringify(portfolioData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addPortfolioRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensurePortfolioSnapshotsTable(logger);
+                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for signed-in user ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -63,23 +85,38 @@ async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid,
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.historyData - Trade history data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date, historyData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserTradeHistoryData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserTradeHistoryData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...historyData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date, historyData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addHistoryRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureTradeHistorySnapshotsTable(logger);
+                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for signed-in user ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -107,30 +144,39 @@ async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, c
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {Array} params.posts - Array of social posts
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date, posts }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserSocialPostData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserSocialPostData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const postsMap = {};
-    for (const post of posts) {
-        if (post.id || post.postId) {
-            postsMap[post.id || post.postId] = post;
+async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date, posts, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            await ensureSocialPostSnapshotsTable(logger);
+            
+            const postsMap = {};
+            for (const post of posts) {
+                if (post.id || post.postId) {
+                    postsMap[post.id || post.postId] = post;
+                }
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            await insertRows(datasetId, 'social_post_snapshots', [row], logger);
+            logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for signed-in user ${cid} (date: ${date}, ${posts.length} posts)`);
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
         }
     }
     
-    await rootDataRef.set({
-        posts: postsMap,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid),
-        postCount: posts.length
-    }, { merge: false });
-    
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: SignedInUsers/{cid}/posts/{postId}
     // Construct path directly - we know the structure
@@ -180,31 +226,48 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.portfolioData - Portfolio data to store
  * @param {object} params.deepPortfolioData - Optional deep portfolio data
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, deepPortfolioData = null }) {
-    // 1. Store overall portfolio to root data collection
-    // Structure: PopularInvestorPortfolioData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorPortfolioData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const portfolioDoc = {
-        ...portfolioData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    };
-    
-    // 2. If deep portfolio data exists, merge it
-    if (deepPortfolioData && deepPortfolioData.positions) {
-        portfolioDoc.deepPositions = deepPortfolioData.positions;
-        portfolioDoc.deepFetchedAt = FieldValue.serverTimestamp();
+async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, deepPortfolioData = null, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const portfolioDoc = {
+                ...portfolioData,
+                cid: String(cid)
+            };
+            
+            // If deep portfolio data exists, merge it
+            if (deepPortfolioData && deepPortfolioData.positions) {
+                portfolioDoc.deepPositions = deepPortfolioData.positions;
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                portfolio_data: JSON.stringify(portfolioDoc), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addPortfolioRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensurePortfolioSnapshotsTable(logger);
+                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for PI ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
     }
     
-    await rootDataRef.set(portfolioDoc, { merge: false });
-    
-    // 3. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -231,23 +294,38 @@ async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, c
  * @param {string} params.cid - PI CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.historyData - Trade history data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date, historyData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: PopularInvestorTradeHistoryData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorTradeHistoryData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...historyData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date, historyData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addHistoryRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureTradeHistorySnapshotsTable(logger);
+                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for PI ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -275,30 +353,44 @@ async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry
  * @param {string} params.cid - PI CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {Array} params.posts - Array of social posts
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date, posts }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: PopularInvestorSocialPostData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorSocialPostData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const postsMap = {};
-    for (const post of posts) {
-        if (post.id || post.postId) {
-            postsMap[post.id || post.postId] = post;
+async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date, posts, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const postsMap = {};
+            for (const post of posts) {
+                if (post.id || post.postId) {
+                    postsMap[post.id || post.postId] = post;
+                }
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addSocialRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureSocialPostSnapshotsTable(logger);
+                await insertRows(datasetId, 'social_post_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for PI ${cid} (date: ${date}, ${posts.length} posts)`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
         }
     }
     
-    await rootDataRef.set({
-        posts: postsMap,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid),
-        postCount: posts.length
-    }, { merge: false });
-    
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: PopularInvestors/{piCid}/posts/{postId}
     // Construct path directly - we know the structure

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -96,7 +96,8 @@ async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType
 }
 
 async function processPortfolio(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const url = `${config.ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
     
@@ -132,7 +133,8 @@ async function processPortfolio(context, config, taskData, isPI) {
 }
 
 async function processHistory(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, uuid, today, requestOptions } = taskData;
     const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
     const url = `${config.ETORO_API_HISTORY_URL}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
@@ -145,15 +147,16 @@ async function processHistory(context, config, taskData, isPI) {
         data.PublicHistoryPositions = data.PublicHistoryPositions.filter(p => [0, 1, 5].includes(p.CloseReason));
     }
 
-    if (isPI) await storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data });
-    else await storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data });
+    if (isPI) await storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data, bigqueryBatchManager });
+    else await storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data, bigqueryBatchManager });
 
     await updateLastUpdated(db, collectionRegistry, cid, isPI ? 'popularInvestor' : 'signedInUser', 'tradeHistory', logger);
     return true;
 }
 
 async function processSocial(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
     
@@ -172,8 +175,11 @@ async function processSocial(context, config, taskData, isPI) {
         }));
 
         // Store posts even if empty (to mark that social fetch was attempted)
-        if (isPI) await storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts });
-        else await storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts });
+        const { batchManager } = context;
+        const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
+        
+        if (isPI) await storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts, bigqueryBatchManager });
+        else await storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts, bigqueryBatchManager });
         
         // Update lastUpdated timestamp to indicate social fetch completed (even if 0 posts)
         await updateLastUpdated(db, collectionRegistry, cid, isPI ? 'popularInvestor' : 'signedInUser', 'socialPosts', logger);

package/functions/task-engine/utils/bigquery_batch_manager.js

@@ -0,0 +1,201 @@
+/**
+ * @fileoverview BigQuery Batch Manager for Task Engine
+ * 
+ * Batches BigQuery writes and flushes them using LOAD JOBS (FREE).
+ * Integrates with FirestoreBatchManager to flush together.
+ * 
+ * Lessons learned from backfill script:
+ * - Use createLoadJob() not table.load()
+ * - Use temp files for load jobs
+ * - Wait for job completion with getMetadata() polling
+ * - Handle errors gracefully
+ * - Clean up temp files properly
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { 
+    getOrCreateDataset,
+    ensurePortfolioSnapshotsTable,
+    ensureTradeHistorySnapshotsTable,
+    ensureSocialPostSnapshotsTable
+} = require('../../core/utils/bigquery_utils');
+
+class BigQueryBatchManager {
+    constructor(logger) {
+        this.logger = logger;
+        this.datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        
+        // Buffers for each table
+        this.portfolioBuffer = [];
+        this.historyBuffer = [];
+        this.socialBuffer = [];
+        
+        // Track if tables are ensured (avoid repeated checks)
+        this.tablesEnsured = {
+            portfolio: false,
+            history: false,
+            social: false
+        };
+    }
+    
+    /**
+     * Add portfolio row to buffer
+     */
+    async addPortfolioRow(row) {
+        if (!this.tablesEnsured.portfolio) {
+            await ensurePortfolioSnapshotsTable(this.logger);
+            this.tablesEnsured.portfolio = true;
+        }
+        this.portfolioBuffer.push(row);
+    }
+    
+    /**
+     * Add trade history row to buffer
+     */
+    async addHistoryRow(row) {
+        if (!this.tablesEnsured.history) {
+            await ensureTradeHistorySnapshotsTable(this.logger);
+            this.tablesEnsured.history = true;
+        }
+        this.historyBuffer.push(row);
+    }
+    
+    /**
+     * Add social post row to buffer
+     */
+    async addSocialRow(row) {
+        if (!this.tablesEnsured.social) {
+            await ensureSocialPostSnapshotsTable(this.logger);
+            this.tablesEnsured.social = true;
+        }
+        this.socialBuffer.push(row);
+    }
+    
+    /**
+     * Flush a buffer to BigQuery using load job
+     * Uses lessons from backfill: createLoadJob, temp files, proper polling
+     */
+    async _flushBuffer(buffer, tableId, tableName) {
+        if (buffer.length === 0) return 0;
+        
+        const rows = [...buffer]; // Copy buffer
+        buffer.length = 0; // Clear buffer
+        
+        try {
+            const dataset = await getOrCreateDataset(this.datasetId, this.logger);
+            const table = dataset.table(tableId);
+            
+            // Write to temporary file (load jobs require a file, not in-memory data)
+            const tempFile = path.join(os.tmpdir(), `bigquery_${tableId}_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+            const ndjson = rows.map(r => JSON.stringify(r)).join('\n');
+            
+            try {
+                fs.writeFileSync(tempFile, ndjson, 'utf8');
+                
+                // Create load job (FREE) - using createLoadJob as learned from backfill
+                const [job] = await table.createLoadJob(tempFile, {
+                    sourceFormat: 'NEWLINE_DELIMITED_JSON',
+                    writeDisposition: 'WRITE_APPEND',
+                    autodetect: false // Use existing table schema
+                });
+                
+                // Wait for job to complete using polling (as learned from backfill)
+                let jobMetadata;
+                const maxAttempts = 60; // 5 minutes max (5 second intervals)
+                const pollInterval = 5000; // 5 seconds
+                
+                for (let attempt = 0; attempt < maxAttempts; attempt++) {
+                    [jobMetadata] = await job.getMetadata();
+                    const state = jobMetadata.status?.state;
+                    
+                    if (state === 'DONE') {
+                        break;
+                    }
+                    
+                    if (state === 'PENDING' || state === 'RUNNING') {
+                        // Wait before next poll
+                        await new Promise(resolve => setTimeout(resolve, pollInterval));
+                    } else {
+                        throw new Error(`Unexpected job state: ${state}`);
+                    }
+                }
+                
+                // Check if we timed out
+                if (jobMetadata.status?.state !== 'DONE') {
+                    throw new Error(`Load job did not complete within ${maxAttempts * pollInterval / 1000} seconds`);
+                }
+                
+                // Check for errors
+                if (jobMetadata.status?.errorResult) {
+                    throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
+                }
+                
+                const rowsLoaded = jobMetadata.statistics?.load?.outputRows || rows.length;
+                
+                if (this.logger) {
+                    this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rowsLoaded} ${tableName} rows to BigQuery using LOAD JOB (free)`);
+                }
+                
+                return rowsLoaded;
+            } finally {
+                // Clean up temp file (as learned from backfill)
+                try {
+                    if (fs.existsSync(tempFile)) {
+                        fs.unlinkSync(tempFile);
+                    }
+                } catch (cleanupError) {
+                    if (this.logger) {
+                        this.logger.log('WARN', `[BigQueryBatch] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
+                    }
+                }
+            }
+        } catch (error) {
+            // Log error but don't throw - allow Firestore writes to continue
+            if (this.logger) {
+                this.logger.log('WARN', `[BigQueryBatch] Failed to flush ${tableName} to BigQuery: ${error.message}`);
+            }
+            // Put rows back in buffer for retry on next flush
+            buffer.push(...rows);
+            return 0;
+        }
+    }
+    
+    /**
+     * Flush all buffers to BigQuery
+     * Called by FirestoreBatchManager.flushBatches()
+     */
+    async flushBatches() {
+        if (process.env.BIGQUERY_ENABLED === 'false') {
+            return; // Skip if BigQuery disabled
+        }
+        
+        const results = await Promise.allSettled([
+            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio'),
+            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history'),
+            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social')
+        ]);
+        
+        const totalFlushed = results
+            .filter(r => r.status === 'fulfilled')
+            .reduce((sum, r) => sum + r.value, 0);
+        
+        if (totalFlushed > 0 && this.logger) {
+            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows to BigQuery`);
+        }
+    }
+    
+    /**
+     * Get buffer sizes (for monitoring)
+     */
+    getBufferSizes() {
+        return {
+            portfolio: this.portfolioBuffer.length,
+            history: this.historyBuffer.length,
+            social: this.socialBuffer.length
+        };
+    }
+}
+
+module.exports = { BigQueryBatchManager };