bulltrackers-module 1.0.753 → 1.0.754

package/functions/task-engine/tests/test-task-engine.sh

@@ -9,7 +9,7 @@
 FUNCTION_NAME="orchestrator-http"
 REGION="europe-west1"
 DATE=$(date +%Y-%m-%d) # Defaults to today
-USER_TYPE="normal"     # Options: normal, speculator, popular_investor
+USER_TYPE="popular_investor"     # Options: normal, speculator, popular_investor
 WINDOWS=1              # 1 window = immediate execution (0s delay)
 
 # --- 1. FETCH URL DYNAMICALLY ---

package/functions/task-engine/utils/bigquery_batch_manager.js

@@ -1,26 +1,17 @@
 /**
  * @fileoverview BigQuery Batch Manager for Task Engine
- * 
- * Batches BigQuery writes and flushes them using MERGE operations.
- * Integrates with FirestoreBatchManager to flush together.
- * 
- * UPDATED: Now uses insertRowsWithMerge to prevent duplicate rows
- * when a user is processed twice on the same day.
- * 
- * Deduplication keys:
- * - portfolio_snapshots: ['date', 'user_id', 'user_type']
- * - trade_history_snapshots: ['date', 'user_id', 'user_type']
- * - social_post_snapshots: ['date', 'user_id', 'user_type']
+ * * FIXED: Switched to APPEND-ONLY (insertRows) to avoid DML quotas.
+ * PREVIOUSLY: Used MERGE, which hit the 1,500 DML/day limit.
  */
 
 const { 
     ensurePortfolioSnapshotsTable,
     ensureTradeHistorySnapshotsTable,
     ensureSocialPostSnapshotsTable,
-    insertRowsWithMerge
+    insertRows // <--- CHANGED: Using direct insert instead of merge
 } = require('../../core/utils/bigquery_utils');
 
-// Deduplication keys for each table type
+// Deduplication keys (Kept for reference, though not used in Append mode)
 const PORTFOLIO_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
 const HISTORY_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
 const SOCIAL_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
@@ -43,9 +34,6 @@ class BigQueryBatchManager {
         };
     }
     
-    /**
-     * Add portfolio row to buffer
-     */
     async addPortfolioRow(row) {
         if (!this.tablesEnsured.portfolio) {
             await ensurePortfolioSnapshotsTable(this.logger);
@@ -54,9 +42,6 @@ class BigQueryBatchManager {
         this.portfolioBuffer.push(row);
     }
     
-    /**
-     * Add trade history row to buffer
-     */
     async addHistoryRow(row) {
         if (!this.tablesEnsured.history) {
             await ensureTradeHistorySnapshotsTable(this.logger);
@@ -65,9 +50,6 @@ class BigQueryBatchManager {
         this.historyBuffer.push(row);
     }
     
-    /**
-     * Add social post row to buffer
-     */
     async addSocialRow(row) {
         if (!this.tablesEnsured.social) {
             await ensureSocialPostSnapshotsTable(this.logger);
@@ -77,55 +59,47 @@ class BigQueryBatchManager {
     }
     
     /**
-     * Flush a buffer to BigQuery using MERGE operation
-     * Uses insertRowsWithMerge to prevent duplicates when a user is processed twice on same day
+     * Flush a buffer to BigQuery using APPEND (Load Job)
+     * FIXED: Removed MERGE logic to bypass DML quotas.
      */
-    async _flushBuffer(buffer, tableId, tableName, dedupKeys) {
+    async _flushBuffer(buffer, tableId, tableName) {
         if (buffer.length === 0) return 0;
         
-        const rows = [...buffer]; // Copy buffer
-        buffer.length = 0; // Clear buffer
+        const rows = [...buffer]; 
+        buffer.length = 0; 
         
         try {
-            // Use insertRowsWithMerge for deduplication
-            // This prevents duplicate rows if a user is processed twice on the same day
-            const rowsInserted = await insertRowsWithMerge(
+            // CHANGED: insertRows uses a LOAD Job with WRITE_APPEND. 
+            // This is FREE and has a 100,000 jobs/day limit.
+            await insertRows(
                 this.datasetId, 
                 tableId, 
                 rows, 
-                dedupKeys, 
                 this.logger
             );
             
             if (this.logger) {
-                this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rows.length} ${tableName} rows to BigQuery using MERGE (${rowsInserted} new, ${rows.length - rowsInserted} updated)`);
+                this.logger.log('INFO', `[BigQueryBatch] ✅ Appended ${rows.length} ${tableName} rows to BigQuery (Load Job)`);
             }
             
             return rows.length;
         } catch (error) {
-            // Log error but don't throw - allow Firestore writes to continue
             if (this.logger) {
                 this.logger.log('WARN', `[BigQueryBatch] Failed to flush ${tableName} to BigQuery: ${error.message}`);
             }
-            // Put rows back in buffer for retry on next flush
+            // Put rows back in buffer for retry
             buffer.push(...rows);
             return 0;
         }
     }
     
-    /**
-     * Flush all buffers to BigQuery
-     * Called by FirestoreBatchManager.flushBatches()
-     */
     async flushBatches() {
-        if (process.env.BIGQUERY_ENABLED === 'false') {
-            return; // Skip if BigQuery disabled
-        }
+        if (process.env.BIGQUERY_ENABLED === 'false') return; 
         
         const results = await Promise.allSettled([
-            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio', PORTFOLIO_DEDUP_KEYS),
-            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history', HISTORY_DEDUP_KEYS),
-            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social', SOCIAL_DEDUP_KEYS)
+            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio'),
+            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history'),
+            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social')
         ]);
         
         const totalFlushed = results
@@ -133,13 +107,10 @@ class BigQueryBatchManager {
             .reduce((sum, r) => sum + r.value, 0);
         
         if (totalFlushed > 0 && this.logger) {
-            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows to BigQuery`);
+            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows.`);
         }
     }
     
-    /**
-     * Get buffer sizes (for monitoring)
-     */
     getBufferSizes() {
         return {
             portfolio: this.portfolioBuffer.length,
@@ -149,4 +120,4 @@ class BigQueryBatchManager {
     }
 }
 
-module.exports = { BigQueryBatchManager };
+module.exports = { BigQueryBatchManager };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -1,7 +1,5 @@
 /** @fileoverview Utility class to manage stateful Firestore write batches.
- * REFACTORED: Added support for Popular Investor and Signed-In User sharding.
- * Mapped new user types to their respective collections.
- * IMPLEMENTS: Round-Robin Sharding for maximum user density per document.
+ * REFACTORED: Increased default batch size to 500 to reduce API calls.
  */ 
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -73,14 +71,16 @@ class FirestoreBatchManager {
         return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; 
     }
 
-   async _scheduleFlush() { // Added async
-        // LOWER THIS LIMIT: 400 is too high for PI history. Use 50.
-        const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 50;
+   async _scheduleFlush() { 
+        // OPTIMIZATION: Increased default to 500. 
+        // Firestore limit is 500 writes. Since we shard users into buckets, 
+        // 500 users results in much fewer than 500 writes (likely <10 writes).
+        const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 500;
         const totalOps = this._estimateBatchSize();
         
         if (totalOps >= maxBatch) {  
             this.logger.log('INFO', `[BATCH] Hit limit (${totalOps} >= ${maxBatch}). Flushing...`);
-            await this.flushBatches(); // Added await
+            await this.flushBatches(); 
             return; 
         }
     }
@@ -135,7 +135,7 @@ class FirestoreBatchManager {
             this.usernameMapUpdates[shardId] = {}; 
         }
         this.usernameMapUpdates[shardId][cidStr] = { username };
-        this._scheduleFlush(); // TODO Should we wait this? Would need to be async and update any call to it to await the function call. Not sure best here. Leave for now.
+        this._scheduleFlush(); 
     }
 
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.753",
+  "version": "1.0.754",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [