bulltrackers-module 1.0.752 → 1.0.754

package/functions/orchestrator/index.js

@@ -16,6 +16,9 @@ const QUEUE_NAME = process.env.ORCHESTRATOR_QUEUE || 'task-engine-queue';
 const LOCATION = process.env.GCP_REGION || 'europe-west1'; 
 const PROJECT = process.env.GCP_PROJECT_ID;
 
+// --- FEATURE FLAG: Disable Normal/Speculator Users ---
+const ENABLE_LEGACY_USERS = process.env.ENABLE_LEGACY_USERS === 'true';
+
 /**
  * ENTRY POINT: HTTP Handler for Workflow Interaction
  */
@@ -33,6 +36,14 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
                 throw new Error("Missing userType or date for PLAN action");
             }
             
+            // --- NEW: Block Legacy Users if Disabled ---
+            if ((userType === 'normal' || userType === 'speculator') && !ENABLE_LEGACY_USERS) {
+                const msg = `[Orchestrator] SKIPPING PLAN for '${userType}': ENABLE_LEGACY_USERS is false.`;
+                logger.log('WARN', msg);
+                // Return 200 to prevent retry loops in workflows
+                return res.status(200).send({ status: 'skipped', message: msg });
+            }
+
             // Determine self-URL for callback (Cloud Task needs to call this function back)
             // We use the env var passed by GCF (FUNCTION_URI) or construct it manually
             const orchestratorUrl = orchestratorUrlOverride || 
@@ -47,6 +58,14 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
             if (!planId || !windowId) {
                 throw new Error("Missing planId or windowId for EXECUTE_WINDOW action");
             }
+            
+            // --- NEW: Block Legacy Users if Disabled (Double Check) ---
+            if ((userType === 'normal' || userType === 'speculator') && !ENABLE_LEGACY_USERS) {
+                 const msg = `[Orchestrator] SKIPPING EXECUTE_WINDOW for '${userType}': ENABLE_LEGACY_USERS is false.`;
+                 logger.log('WARN', msg);
+                 return res.status(200).send({ status: 'skipped', message: msg });
+            }
+
             const result = await executeUpdateWindow(planId, windowId, userType, config, dependencies);
             res.status(200).send(result);
 
@@ -227,8 +246,13 @@ async function runDiscoveryOrchestrator(config, deps) {
     const { logger, firestoreUtils } = deps;
     logger.log('INFO', '🚀 Discovery Orchestrator triggered...');
     await firestoreUtils.resetProxyLocks(deps, config);
-    if (isUserTypeEnabled('normal', config.enabledUserTypes)) await runDiscovery('normal', config.discoveryConfig.normal, config, deps);
-    if (isUserTypeEnabled('speculator', config.enabledUserTypes)) await runDiscovery('speculator', config.discoveryConfig.speculator, config, deps);
+    
+    if (ENABLE_LEGACY_USERS) {
+        if (isUserTypeEnabled('normal', config.enabledUserTypes)) await runDiscovery('normal', config.discoveryConfig.normal, config, deps);
+        if (isUserTypeEnabled('speculator', config.enabledUserTypes)) await runDiscovery('speculator', config.discoveryConfig.speculator, config, deps);
+    } else {
+        logger.log('INFO', 'Discovery skipped for legacy users (normal/speculator) because ENABLE_LEGACY_USERS is false.');
+    }
 }
 
 async function runUpdateOrchestrator(config, deps) {
@@ -237,8 +261,13 @@ async function runUpdateOrchestrator(config, deps) {
     await firestoreUtils.resetProxyLocks(deps, config);
     const enabledTypes = config.enabledUserTypes || [];
 
-    if (isUserTypeEnabled('normal', enabledTypes)) await runUpdates('normal', config.updateConfig, config, deps);
-    if (isUserTypeEnabled('speculator', enabledTypes)) await runUpdates('speculator', config.updateConfig, config, deps);
+    if (ENABLE_LEGACY_USERS) {
+        if (isUserTypeEnabled('normal', enabledTypes)) await runUpdates('normal', config.updateConfig, config, deps);
+        if (isUserTypeEnabled('speculator', enabledTypes)) await runUpdates('speculator', config.updateConfig, config, deps);
+    } else {
+        logger.log('INFO', 'Updates skipped for legacy users (normal/speculator) because ENABLE_LEGACY_USERS is false.');
+    }
+
     if (isUserTypeEnabled('popular_investor', enabledTypes)) {
         const piConfig = { ...config.updateConfig, popularInvestorRankingsCollection: config.updateConfig.popularInvestorRankingsCollection || 'popular_investor_rankings' };
         await runUpdates('popular_investor', piConfig, config, deps);

package/functions/task-engine/tests/test-task-engine.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# ==============================================================================
+# BULLTRACKERS TASK ENGINE END-TO-END TESTER
+# This script triggers the Orchestrator to plan an immediate execution window.
+# ==============================================================================
+
+# --- CONFIGURATION ---
+FUNCTION_NAME="orchestrator-http"
+REGION="europe-west1"
+DATE=$(date +%Y-%m-%d) # Defaults to today
+USER_TYPE="popular_investor"     # Options: normal, speculator, popular_investor
+WINDOWS=1              # 1 window = immediate execution (0s delay)
+
+# --- 1. FETCH URL DYNAMICALLY ---
+echo "🔍 Fetching URL for function: $FUNCTION_NAME ($REGION)..."
+
+# Try Gen 2 (Cloud Run) URL first
+URL=$(gcloud functions describe $FUNCTION_NAME --region=$REGION --format='value(serviceConfig.uri)' 2>/dev/null)
+
+# Fallback to Gen 1 if empty
+if [ -z "$URL" ]; then
+    URL=$(gcloud functions describe $FUNCTION_NAME --region=$REGION --format='value(httpsTrigger.url)' 2>/dev/null)
+fi
+
+if [ -z "$URL" ]; then
+    echo "❌ Error: Could not find URL for function '$FUNCTION_NAME'. Check if it is deployed."
+    exit 1
+fi
+
+echo "✅ Target URL: $URL"
+
+# --- 2. GET AUTH TOKEN ---
+echo "🔑 Generating Identity Token..."
+TOKEN=$(gcloud auth print-identity-token)
+
+if [ -z "$TOKEN" ]; then
+    echo "❌ Error: Could not generate token. Run 'gcloud auth login' first."
+    exit 1
+fi
+
+# --- 3. SEND REQUEST ---
+echo "🚀 Triggering Plan for $USER_TYPE on $DATE ($WINDOWS window)..."
+
+RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$URL" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"action\": \"PLAN\",
+    \"userType\": \"$USER_TYPE\",
+    \"date\": \"$DATE\",
+    \"windows\": $WINDOWS
+  }")
+
+# --- 4. PARSE RESPONSE ---
+HTTP_BODY=$(echo "$RESPONSE" | head -n -1)
+HTTP_CODE=$(echo "$RESPONSE" | tail -n 1)
+
+if [ "$HTTP_CODE" -eq 200 ]; then
+    echo ""
+    echo "✅ SUCCESS (HTTP 200)"
+    echo "---------------------------------------------------"
+    echo "$HTTP_BODY" | python3 -m json.tool 2>/dev/null || echo "$HTTP_BODY"
+    echo "---------------------------------------------------"
+    echo "👉 Monitor 'task-engine-queue' in Cloud Tasks Console."
+    echo "👉 Check Logs Explorer for 'Orchestrator' and 'Dispatcher'."
+else
+    echo ""
+    echo "❌ FAILED (HTTP $HTTP_CODE)"
+    echo "---------------------------------------------------"
+    echo "$HTTP_BODY"
+    echo "---------------------------------------------------"
+fi

package/functions/task-engine/utils/bigquery_batch_manager.js

@@ -1,26 +1,17 @@
 /**
  * @fileoverview BigQuery Batch Manager for Task Engine
- * 
- * Batches BigQuery writes and flushes them using MERGE operations.
- * Integrates with FirestoreBatchManager to flush together.
- * 
- * UPDATED: Now uses insertRowsWithMerge to prevent duplicate rows
- * when a user is processed twice on the same day.
- * 
- * Deduplication keys:
- * - portfolio_snapshots: ['date', 'user_id', 'user_type']
- * - trade_history_snapshots: ['date', 'user_id', 'user_type']
- * - social_post_snapshots: ['date', 'user_id', 'user_type']
+ * * FIXED: Switched to APPEND-ONLY (insertRows) to avoid DML quotas.
+ * PREVIOUSLY: Used MERGE, which hit the 1,500 DML/day limit.
  */
 
 const { 
     ensurePortfolioSnapshotsTable,
     ensureTradeHistorySnapshotsTable,
     ensureSocialPostSnapshotsTable,
-    insertRowsWithMerge
+    insertRows // <--- CHANGED: Using direct insert instead of merge
 } = require('../../core/utils/bigquery_utils');
 
-// Deduplication keys for each table type
+// Deduplication keys (Kept for reference, though not used in Append mode)
 const PORTFOLIO_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
 const HISTORY_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
 const SOCIAL_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
@@ -43,9 +34,6 @@ class BigQueryBatchManager {
         };
     }
     
-    /**
-     * Add portfolio row to buffer
-     */
     async addPortfolioRow(row) {
         if (!this.tablesEnsured.portfolio) {
             await ensurePortfolioSnapshotsTable(this.logger);
@@ -54,9 +42,6 @@ class BigQueryBatchManager {
         this.portfolioBuffer.push(row);
     }
     
-    /**
-     * Add trade history row to buffer
-     */
     async addHistoryRow(row) {
         if (!this.tablesEnsured.history) {
             await ensureTradeHistorySnapshotsTable(this.logger);
@@ -65,9 +50,6 @@ class BigQueryBatchManager {
         this.historyBuffer.push(row);
     }
     
-    /**
-     * Add social post row to buffer
-     */
     async addSocialRow(row) {
         if (!this.tablesEnsured.social) {
             await ensureSocialPostSnapshotsTable(this.logger);
@@ -77,55 +59,47 @@ class BigQueryBatchManager {
     }
     
     /**
-     * Flush a buffer to BigQuery using MERGE operation
-     * Uses insertRowsWithMerge to prevent duplicates when a user is processed twice on same day
+     * Flush a buffer to BigQuery using APPEND (Load Job)
+     * FIXED: Removed MERGE logic to bypass DML quotas.
      */
-    async _flushBuffer(buffer, tableId, tableName, dedupKeys) {
+    async _flushBuffer(buffer, tableId, tableName) {
         if (buffer.length === 0) return 0;
         
-        const rows = [...buffer]; // Copy buffer
-        buffer.length = 0; // Clear buffer
+        const rows = [...buffer]; 
+        buffer.length = 0; 
         
         try {
-            // Use insertRowsWithMerge for deduplication
-            // This prevents duplicate rows if a user is processed twice on the same day
-            const rowsInserted = await insertRowsWithMerge(
+            // CHANGED: insertRows uses a LOAD Job with WRITE_APPEND. 
+            // This is FREE and has a 100,000 jobs/day limit.
+            await insertRows(
                 this.datasetId, 
                 tableId, 
                 rows, 
-                dedupKeys, 
                 this.logger
             );
             
             if (this.logger) {
-                this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rows.length} ${tableName} rows to BigQuery using MERGE (${rowsInserted} new, ${rows.length - rowsInserted} updated)`);
+                this.logger.log('INFO', `[BigQueryBatch] ✅ Appended ${rows.length} ${tableName} rows to BigQuery (Load Job)`);
             }
             
             return rows.length;
         } catch (error) {
-            // Log error but don't throw - allow Firestore writes to continue
             if (this.logger) {
                 this.logger.log('WARN', `[BigQueryBatch] Failed to flush ${tableName} to BigQuery: ${error.message}`);
             }
-            // Put rows back in buffer for retry on next flush
+            // Put rows back in buffer for retry
             buffer.push(...rows);
             return 0;
         }
     }
     
-    /**
-     * Flush all buffers to BigQuery
-     * Called by FirestoreBatchManager.flushBatches()
-     */
     async flushBatches() {
-        if (process.env.BIGQUERY_ENABLED === 'false') {
-            return; // Skip if BigQuery disabled
-        }
+        if (process.env.BIGQUERY_ENABLED === 'false') return; 
         
         const results = await Promise.allSettled([
-            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio', PORTFOLIO_DEDUP_KEYS),
-            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history', HISTORY_DEDUP_KEYS),
-            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social', SOCIAL_DEDUP_KEYS)
+            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio'),
+            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history'),
+            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social')
         ]);
         
         const totalFlushed = results
@@ -133,13 +107,10 @@ class BigQueryBatchManager {
             .reduce((sum, r) => sum + r.value, 0);
         
         if (totalFlushed > 0 && this.logger) {
-            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows to BigQuery`);
+            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows.`);
         }
     }
     
-    /**
-     * Get buffer sizes (for monitoring)
-     */
     getBufferSizes() {
         return {
             portfolio: this.portfolioBuffer.length,
@@ -149,4 +120,4 @@ class BigQueryBatchManager {
     }
 }
 
-module.exports = { BigQueryBatchManager };
+module.exports = { BigQueryBatchManager };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -1,7 +1,5 @@
 /** @fileoverview Utility class to manage stateful Firestore write batches.
- * REFACTORED: Added support for Popular Investor and Signed-In User sharding.
- * Mapped new user types to their respective collections.
- * IMPLEMENTS: Round-Robin Sharding for maximum user density per document.
+ * REFACTORED: Increased default batch size to 500 to reduce API calls.
  */ 
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -73,14 +71,16 @@ class FirestoreBatchManager {
         return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; 
     }
 
-   async _scheduleFlush() { // Added async
-        // LOWER THIS LIMIT: 400 is too high for PI history. Use 50.
-        const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 50;
+   async _scheduleFlush() { 
+        // OPTIMIZATION: Increased default to 500. 
+        // Firestore limit is 500 writes. Since we shard users into buckets, 
+        // 500 users results in much fewer than 500 writes (likely <10 writes).
+        const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 500;
         const totalOps = this._estimateBatchSize();
         
         if (totalOps >= maxBatch) {  
             this.logger.log('INFO', `[BATCH] Hit limit (${totalOps} >= ${maxBatch}). Flushing...`);
-            await this.flushBatches(); // Added await
+            await this.flushBatches(); 
             return; 
         }
     }
@@ -135,7 +135,7 @@ class FirestoreBatchManager {
             this.usernameMapUpdates[shardId] = {}; 
         }
         this.usernameMapUpdates[shardId][cidStr] = { username };
-        this._scheduleFlush(); // TODO Should we wait this? Would need to be async and update any call to it to await the function call. Not sure best here. Leave for now.
+        this._scheduleFlush(); 
     }
 
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.752",
+  "version": "1.0.754",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [