bulltrackers-module 1.0.750 → 1.0.752

package/functions/computation-system-v2/handlers/adminTest.js

@@ -31,7 +31,10 @@ async function adminTestHandler(req, res) {
             limit = 10,
             force = true,           // Default to force for testing
             useWorkerPool,          // Override: true/false/undefined (use config)
-            dryRun = false
+            dryRun = false,
+            // [NEW] Accept override parameters for infrastructure testing
+            outputTable,            // e.g., "computation_results_test"
+            workerUrl               // e.g., "https://...cloudfunctions.net/worker-test"
         } = req.body || {};
 
         console.log(`[AdminTest] Action: ${action}, Computation: ${computation}, Date: ${date}`);
@@ -93,38 +96,59 @@ async function adminTestHandler(req, res) {
                     });
                 }
 
-                // Log worker pool override if specified
-                if (useWorkerPool !== undefined) {
-                    console.log(`[AdminTest] Worker pool override: ${useWorkerPool ? 'ENABLED' : 'DISABLED'}`);
+                // Prepare Config Overrides
+                // We create this here so we can log what's happening
+                const baseConfig = system.config;
+                let customConfig = null;
+
+                // Check if we need to modify the configuration
+                if (outputTable || useWorkerPool !== undefined || workerUrl) {
+                    customConfig = { ...baseConfig }; // Shallow copy
+
+                    // 1. Override Output Table
+                    if (outputTable) {
+                        console.log(`[AdminTest] ⚠️ Redirecting output to: ${outputTable}`);
+                        // Fix for TS Error: Ensure baseConfig.resultStore exists before spreading
+                        const baseStore = baseConfig.resultStore;
+                        if (!baseStore) {
+                            throw new Error("System config is missing 'resultStore'");
+                        }
+                        
+                        customConfig.resultStore = {
+                            ...baseStore,
+                            table: outputTable
+                        };
+                    }
+
+                    // 2. Override Worker Pool Settings
+                    if (useWorkerPool !== undefined || workerUrl) {
+                        const basePool = baseConfig.workerPool || {};
+                        customConfig.workerPool = {
+                            ...basePool,
+                            // If useWorkerPool is explicitly provided, use it. Otherwise keep existing.
+                            enabled: useWorkerPool !== undefined ? useWorkerPool : basePool.enabled,
+                            // If workerUrl provided, override it.
+                            workerUrl: workerUrl || basePool.workerUrl
+                        };
+                        
+                        // Log the override state
+                        console.log(`[AdminTest] Worker Pool Override: Enabled=${customConfig.workerPool.enabled}, URL=${customConfig.workerPool.workerUrl || 'default'}`);
+                    }
                 }
 
                 console.log(`[AdminTest] Running ${computation} for ${date}...`);
                 console.log(`[AdminTest] Options: force=${force}, dryRun=${dryRun}, entityIds=${entityIds?.join(',') || 'all'}`);
 
-                // Prepare run options
                 const runOptions = {
                     date,
                     computation,
                     entityIds: entityIds || null,
                     dryRun,
                     force,
-                    useWorkerPool
+                    useWorkerPool,
+                    config: customConfig // Inject the modified config
                 };
 
-                // [FIX] If forcing worker pool, inject config to enable it dynamically
-                // This prevents Orchestrator from ignoring the request if default config has enabled: false
-                if (useWorkerPool) {
-                    const baseConfig = system.config; 
-                    runOptions.config = {
-                        ...baseConfig,
-                        workerPool: {
-                            ...(baseConfig.workerPool || {}),
-                            enabled: true 
-                        }
-                    };
-                    console.log('[AdminTest] Injecting config to FORCE enable RemoteTaskRunner');
-                }
-
                 const result = await system.runComputation(runOptions);
 
                 const duration = Date.now() - startTime;
@@ -169,27 +193,39 @@ async function adminTestHandler(req, res) {
 
                 console.log(`[AdminTest] Running LIMITED test: ${sampleEntities.length} entities`);
 
+                // Prepare Config Overrides (Same logic as 'run')
+                const baseConfig = system.config;
+                let customConfig = null;
+
+                if (outputTable || useWorkerPool !== undefined || workerUrl) {
+                    customConfig = { ...baseConfig };
+                    
+                    if (outputTable) {
+                        const baseStore = baseConfig.resultStore;
+                        if (!baseStore) throw new Error("System config missing 'resultStore'");
+                        customConfig.resultStore = { ...baseStore, table: outputTable };
+                    }
+
+                    if (useWorkerPool !== undefined || workerUrl) {
+                        const basePool = baseConfig.workerPool || {};
+                        customConfig.workerPool = {
+                            ...basePool,
+                            enabled: useWorkerPool !== undefined ? useWorkerPool : basePool.enabled,
+                            workerUrl: workerUrl || basePool.workerUrl
+                        };
+                    }
+                }
+
                 const runOptions = {
                     date,
                     computation,
                     entityIds: sampleEntities,
                     dryRun,
                     force,
-                    useWorkerPool
+                    useWorkerPool,
+                    config: customConfig
                 };
 
-                // Apply the same fix for run_limited if needed
-                if (useWorkerPool) {
-                    const baseConfig = system.config; 
-                    runOptions.config = {
-                        ...baseConfig,
-                        workerPool: {
-                            ...(baseConfig.workerPool || {}),
-                            enabled: true 
-                        }
-                    };
-                }
-
                 const result = await system.runComputation(runOptions);
 
                 const duration = Date.now() - startTime;

package/functions/computation-system-v2/scripts/sh.md

@@ -0,0 +1,115 @@
+Here is the comprehensive documentation for the **Admin Test Endpoint (`compute-admin-test`)**. This guide details the supported actions, available overrides, and their specific impacts on your infrastructure.
+
+### **Overview**
+
+The Admin Test Endpoint is a privileged tool designed for **safe, isolated testing** of computations in production. It allows you to trigger runs manually, bypass scheduling locks, and—most importantly—**divert execution to test infrastructure** (custom tables or worker pools) to prevent polluting production data.
+
+---
+
+### **Global Overrides (Infrastructure Testing)**
+
+These parameters can be applied to **`run`** and **`run_limited`** actions to redirect the execution flow.
+
+| Parameter | Type | Description | Impact |
+| --- | --- | --- | --- |
+| **`outputTable`** | `string` | The BigQuery table where results will be written. | **Data Diversion**: Instead of writing to `computation_results_v3`, the Orchestrator writes to this table. Useful for validating logic without affecting production dashboards. |
+| **`workerUrl`** | `string` | The URL of the worker Cloud Function to invoke. | **Traffic Diversion**: If the worker pool is used, the Orchestrator will send HTTP requests to this URL instead of the production worker. |
+| **`useWorkerPool`** | `boolean` | Force enable/disable the remote worker pool. | **Execution Strategy**: <br>
+
+<br>`true`: Forces remote execution (even for small batches).<br>
+
+<br>`false`: Forces local execution (inside the Admin function).<br>
+
+<br>`undefined`: Uses system default config. |
+
+---
+
+### **Supported Actions**
+
+#### **1. `status**`
+
+Returns the current system manifest, listing all registered computations and their schedules.
+
+* **Configurables:** None.
+* **Data Written:** None.
+* **Impact:** Read-only. Low impact.
+* **Cloud Functions:** `compute-admin-test` (Local).
+
+#### **2. `analyze**`
+
+Runs the "Scheduler Logic" for a specific date to determine what *would* run, without actually running it. Checks dependencies, hash changes, and locks.
+
+* **Configurables:**
+* `date` (YYYY-MM-DD): The target date to analyze.
+
+
+* **Data Written:** None.
+* **Impact:** Read-only. Low impact.
+* **Cloud Functions:** `compute-admin-test` (Local).
+
+#### **3. `run**`
+
+Executes a full computation for the specified date. This is the primary tool for manual triggers and backfills.
+
+* **Configurables:**
+* `computation` (Required): Name of the computation (e.g., `UserPortfolioSummary`).
+* `date`: Target date (Default: Today).
+* `entityIds` (Array): Run only for these specific entities.
+* `force` (Boolean): If `true`, runs even if the code/data hasn't changed (Default: `true` for testing).
+* `dryRun` (Boolean): If `true`, computes results but **does not write to BigQuery**.
+* *Plus Global Overrides (`outputTable`, `workerUrl`, `useWorkerPool`)*.
+
+
+* **Data Written:**
+* **Default:** Writes to `config.resultStore.table` (Production).
+* **With `outputTable`:** Writes to the specified custom table.
+* **With `dryRun`:** No data written.
+
+
+* **Impact:** High. Can trigger heavy BigQuery queries and write large amounts of data.
+* **Cloud Functions:**
+* **Local Mode:** `compute-admin-test` processes all logic.
+* **Worker Pool Mode:** `compute-admin-test` acts as the Orchestrator; `computation-worker` (or `workerUrl`) executes the logic.
+
+
+
+#### **4. `run_limited**`
+
+A safer version of `run` that automatically fetches a small sample of entities (e.g., 5 random users) and runs the computation only for them.
+
+* **Configurables:**
+* `computation` (Required): Name of computation.
+* `limit` (Integer): Number of entities to test (Default: 10).
+* *Plus Global Overrides (`outputTable`, `workerUrl`, `useWorkerPool`)*.
+
+
+* **Data Written:** Same behavior as `run`, but only for the sampled entities.
+* **Impact:** Moderate. Executes real logic but on a strictly limited scope.
+* **Cloud Functions:** Same as `run`.
+
+#### **5. `test_worker**`
+
+Directly tests the **logic** of a worker execution locally within the Admin function. This bypasses the Orchestrator's batching and storage logic, simulating exactly what happens inside a single worker instance.
+
+* **Configurables:**
+* `computation` (Required): Name of computation.
+* `entityIds` (Required Array): Must provide at least one ID to test.
+* `date`: Target date.
+
+
+* **Data Written:** **None**. The result is returned directly in the HTTP response body for inspection.
+* **Impact:** Low. Fetches data for 1 entity and runs logic in-memory.
+* **Cloud Functions:** `compute-admin-test` (Local only).
+
+---
+
+### **Summary of Data Flow**
+
+| Scenario | Execution Location | Data Destination |
+| --- | --- | --- |
+| **Standard Run** | `compute-admin-test` (Local) | `computation_results_v3` (Prod) |
+| **Standard Run + Worker Pool** | `computation-worker` (Remote) | `computation_results_v3` (Prod) |
+| **Run + `outputTable**` | `compute-admin-test` (Local) | **`YOUR_CUSTOM_TABLE`** |
+| **Run + `workerUrl**` | **`worker-test`** (Remote) | `computation_results_v3` (Prod) |
+| **Run + `dryRun**` | `compute-admin-test` (Local) | *None* |
+| **Run + `outputTable` + `workerUrl**` | **`worker-test`** (Remote) | **`YOUR_CUSTOM_TABLE`** |

package/functions/computation-system-v2/scripts/test-admin.sh

@@ -0,0 +1,223 @@
+#!/bin/bash
+
+# ==============================================================================
+# Admin Test Endpoint Interactive Tool
+# Automates the admin.md testing procedures
+# ==============================================================================
+
+# Default Settings
+REGION="europe-west1"
+FUNCTION_NAME="compute-admin-test"
+DEFAULT_DATE=$(date +%Y-%m-%d)
+
+# Colors
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+echo -e "${BLUE}=================================================${NC}"
+echo -e "${BLUE}   BullTrackers Admin Test Interactive Tool      ${NC}"
+echo -e "${BLUE}=================================================${NC}"
+
+# ------------------------------------------------------------------------------
+# 1. Setup & Auth
+# ------------------------------------------------------------------------------
+echo -e "\n${YELLOW}[Setup] Retrieving Function URL and Auth Token...${NC}"
+
+# Get Function URL
+FUNCTION_URL=$(gcloud functions describe $FUNCTION_NAME \
+    --region=$REGION \
+    --format="value(serviceConfig.uri)" 2>/dev/null)
+
+if [ -z "$FUNCTION_URL" ]; then
+    echo -e "${RED}[Error] Could not retrieve Function URL for '$FUNCTION_NAME' in '$REGION'.${NC}"
+    echo "Please ensure you are authenticated (gcloud auth login) and the function is deployed."
+    exit 1
+fi
+
+# Get Identity Token
+TOKEN=$(gcloud auth print-identity-token 2>/dev/null)
+
+if [ -z "$TOKEN" ]; then
+    echo -e "${RED}[Error] Could not generate Identity Token.${NC}"
+    echo "Please run: gcloud auth login"
+    exit 1
+fi
+
+echo -e "${GREEN}[Success] Target: $FUNCTION_URL${NC}"
+
+# ------------------------------------------------------------------------------
+# 2. Helper Functions
+# ------------------------------------------------------------------------------
+
+# Function to prompt for variables with defaults
+ask_var() {
+    local prompt="$1"
+    local default="$2"
+    local var_name="$3"
+    
+    echo -ne "$prompt [${GREEN}$default${NC}]: "
+    read input
+    if [ -z "$input" ]; then
+        eval $var_name="$default"
+    else
+        eval $var_name="$input"
+    fi
+}
+
+# Function to execute the curl command
+run_test() {
+    local json_payload="$1"
+    
+    echo -e "\n${YELLOW}[Executing] Sending Request...${NC}"
+    echo -e "Payload: $json_payload"
+    
+    curl -X POST "$FUNCTION_URL" \
+        -H "Authorization: Bearer $TOKEN" \
+        -H "Content-Type: application/json" \
+        -d "$json_payload"
+        
+    echo -e "\n${BLUE}[Done]${NC}"
+}
+
+# ------------------------------------------------------------------------------
+# 3. Interactive Menu
+# ------------------------------------------------------------------------------
+
+while true; do
+    echo -e "\n${BLUE}Select an Action:${NC}"
+    echo "1) System Status (status)"
+    echo "2) Analyze Schedule (analyze)"
+    echo "3) Run Full Computation (run)"
+    echo "4) Run Limited Test (run_limited - safer)"
+    echo "5) Test Specific Entities (run with entityIds)"
+    echo "6) Test Worker Logic Directly (test_worker)"
+    echo "7) Test Worker Pool Offloading (run with useWorkerPool)"
+    echo "8) Advanced Infrastructure Test (Custom Table/Worker)"
+    echo "q) Quit"
+    
+    read -p "Enter choice: " choice
+    
+    case $choice in
+        1)
+            # Status
+            run_test '{"action": "status"}'
+            ;;
+            
+        2)
+            # Analyze
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            run_test "{\"action\": \"analyze\", \"date\": \"$TARGET_DATE\"}"
+            ;;
+            
+        3)
+            # Run Full
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            
+            # Confirm dangerous action
+            echo -e "${RED}WARNING: This will run '$COMP_NAME' for ALL entities.${NC}"
+            read -p "Are you sure? (y/N): " confirm
+            if [[ $confirm == "y" || $confirm == "Y" ]]; then
+                run_test "{\"action\": \"run\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"force\": true}"
+            else
+                echo "Cancelled."
+            fi
+            ;;
+            
+        4)
+            # Run Limited
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            ask_var "Enter Limit (Num Entities)" "5" "LIMIT_NUM"
+            
+            run_test "{\"action\": \"run_limited\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"limit\": $LIMIT_NUM}"
+            ;;
+            
+        5)
+            # Test Entities
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            ask_var "Enter Entity IDs (comma separated, no spaces)" "user-123,user-456" "RAW_IDS"
+            
+            # Format IDs into JSON array
+            IFS=',' read -r -a id_array <<< "$RAW_IDS"
+            JSON_IDS=$(printf '"%s",' "${id_array[@]}")
+            JSON_IDS="[${JSON_IDS%,}]"
+            
+            run_test "{\"action\": \"run\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"entityIds\": $JSON_IDS, \"force\": true}"
+            ;;
+            
+        6)
+            # Test Worker Logic
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            ask_var "Enter Single Entity ID" "user-123" "ENTITY_ID"
+            
+            # Worker test expects array of entityIds
+            run_test "{\"action\": \"test_worker\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"entityIds\": [\"$ENTITY_ID\"]}"
+            ;;
+            
+        7)
+            # Test Worker Pool
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            
+            echo -e "${YELLOW}Running with Worker Pool enabled...${NC}"
+            run_test "{\"action\": \"run\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"useWorkerPool\": true, \"force\": true}"
+            ;;
+
+        8)
+            # Advanced Infrastructure Test (Custom Table & Worker)
+            echo -e "${YELLOW}--- Infrastructure Integration Test ---${NC}"
+            ask_var "Enter Computation Name" "UserPortfolioSummary" "COMP_NAME"
+            ask_var "Enter Date" "$DEFAULT_DATE" "TARGET_DATE"
+            ask_var "Enter Output Table Name" "computation_results_test" "OUT_TABLE"
+            ask_var "Enter Custom Worker URL (optional)" "" "WORKER_URL"
+            ask_var "Use Worker Pool? (true/false)" "true" "USE_POOL"
+            ask_var "Entity Limit (0 for Full Run)" "10" "LIMIT_NUM"
+
+            # Determine Action (run vs run_limited)
+            if [ "$LIMIT_NUM" -gt 0 ]; then
+                ACTION="run_limited"
+                LIMIT_PART=", \"limit\": $LIMIT_NUM"
+            else
+                ACTION="run"
+                LIMIT_PART=", \"force\": true"
+            fi
+
+            # Build Optional JSON Parts
+            TABLE_PART=""
+            if [ ! -z "$OUT_TABLE" ]; then
+                TABLE_PART=", \"outputTable\": \"$OUT_TABLE\""
+            fi
+
+            WORKER_PART=""
+            if [ ! -z "$WORKER_URL" ]; then
+                WORKER_PART=", \"workerUrl\": \"$WORKER_URL\""
+            fi
+
+            echo -e "${YELLOW}Running ${ACTION} on ${OUT_TABLE}...${NC}"
+            
+            # Construct Payload
+            # We use loose concatenation here which works fine for JSON string building in bash
+            JSON="{\"action\": \"$ACTION\", \"computation\": \"$COMP_NAME\", \"date\": \"$TARGET_DATE\", \"useWorkerPool\": $USE_POOL $LIMIT_PART $TABLE_PART $WORKER_PART}"
+            
+            run_test "$JSON"
+            ;;
+            
+        q|Q)
+            echo "Exiting."
+            exit 0
+            ;;
+            
+        *)
+            echo -e "${RED}Invalid option.${NC}"
+            ;;
+    esac
+    
+    echo -e "\n-------------------------------------------------"
+    read -p "Press Enter to continue..."
+done

package/index.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Main entry point for the Bulltrackers shared module.
  * CLEANED: Removed legacy V1 Dispatcher/Computation references.
+ * UPDATED: Re-integrated Dispatcher (Task Throttler).
  */
 
 // Core utilities
@@ -21,7 +22,6 @@ const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = requir
 const { getUpdateTargets, dispatchUpdates }                             = require('./functions/orchestrator/helpers/update_helpers');
 
 // --- COMPUTATION SYSTEM V2 (The new standard) ---
-// We import the WHOLE package now, which includes handlers AND ManifestBuilder
 const computationSystemV2 = require('./functions/computation-system-v2/index');
 
 // Task Engine
@@ -30,6 +30,9 @@ const { handleDiscover }                                  = require('./functions
 const { handleVerify }                                    = require('./functions/task-engine/helpers/verify_helpers');
 const { handleUpdate }                                    = require('./functions/task-engine/helpers/update_helpers');
 
+// Dispatcher (Task Throttler)
+const { handleRequest: dispatcherRequest }                = require('./functions/dispatcher/index');
+
 const { createApiV2App } = require('./functions/api-v2/index');
 
 // Maintenance & Backfills
@@ -76,7 +79,9 @@ const orchestrator = {
   dispatchUpdates,
 };
 
-// [REMOVED] Legacy 'dispatcher' pipe. It is replaced by computationSystemV2.
+const dispatcher = {
+  handleRequest: dispatcherRequest
+};
 
 const taskEngine = {
   handleRequest: taskRequest,
@@ -85,7 +90,7 @@ const taskEngine = {
   handleUpdate,
 };
 
-const computationSystem = computationSystemV2; // Direct mapping
+const computationSystem = computationSystemV2; 
 
 const api = {
   createApiV2App, 
@@ -119,5 +124,5 @@ const alertSystem = {
 };
 
 module.exports = {
-  pipe: { core, orchestrator, taskEngine, computationSystem, api, maintenance, proxy, alertSystem },
+  pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy, alertSystem },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.750",
+  "version": "1.0.752",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [