bulltrackers-module 1.0.736 → 1.0.737

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -246,10 +246,12 @@ module.exports = {
     execution: {
         // Max concurrent entity processing (per-entity computations)
         // Higher = faster but more memory. Tune based on your Cloud Function memory.
-        entityConcurrency: 20,
+        entityConcurrency: 50,
         
         // Batch size for BigQuery inserts
         insertBatchSize: 500,
+
+        fetchBatchSize: 30000,
         
         // Memory safety: max entities to load for a dependency
         // If a dependency has more entities than this, use getDependency(name, entityId) instead
@@ -361,7 +363,7 @@ module.exports = {
         // Max concurrent worker invocations
         // Higher = faster but more network/GCS load
         // Recommended: 100-200 for production
-        concurrency: parseInt(process.env.WORKER_CONCURRENCY || '100', 10),
+        concurrency: 100,
         
         // Worker invocation timeout (ms)
         // Should be slightly less than worker function timeout
@@ -382,6 +384,8 @@ module.exports = {
         // Useful for testing specific computations
         forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS 
             ? process.env.WORKER_FORCE_COMPUTATIONS.split(',') 
-            : []
+            : [],
+        
+        minEntitiesForOffload: 100,
     }
 };

package/functions/computation-system-v2/docs/architecture.md

@@ -0,0 +1,59 @@
+```mermaid
+graph TD
+    Root((System))
+
+    %% Subgraph: Scheduling & Control
+    subgraph Control_Plane [Control Plane]
+        Cron((Timer)) -->|Every Minute| Scheduler[Scheduler Handler]
+        Scheduler -->|Find Due & Zombies| StateRepo[(State DB)]
+        Scheduler -->|Dispatch Task| CloudTasks[Cloud Tasks Queue]
+        CloudTasks -->|HTTP POST w/ Backoff| Dispatcher[Dispatcher Handler]
+        Dispatcher -->|Run Computation| Orchestrator[Orchestrator]
+        Orchestrator -->|Return Status| Dispatcher
+        Dispatcher -.->|Blocked| Return503[503 Retry]
+        Return503 -.-> CloudTasks
+        Dispatcher -.->|Success / Skipped| Return200[200 OK]
+    end
+
+    %% Subgraph: Execution
+    subgraph Execution_Core [Execution Core]
+        Orchestrator --> Manifest[Manifest Builder]
+        Orchestrator -->|Check Hashes & Deps| StateRepo
+        Orchestrator -->|Fetch Data| BigQuery[(BigQuery)]
+        Orchestrator --> ExecMode{Mode?}
+
+        ExecMode -->|Global / Light| LocalExec[Local Execution]
+        LocalExec --> Logic[Computation Logic]
+        Logic --> LocalExec
+
+        ExecMode -->|Per-Entity / Heavy| RemoteRunner[Remote Task Runner]
+        RemoteRunner -->|Upload Context| GCS[(Cloud Storage)]
+        RemoteRunner --> Worker[Worker Handler]
+        Worker -->|Download Context| GCS
+        Worker -->|Execute| Logic
+        Worker -->|Return Result| RemoteRunner
+    end
+
+    %% Subgraph: Persistence
+    subgraph Persistence [Persistence Layer]
+        LocalExec -->|Commit Results| StateRepo
+        RemoteRunner -->|Commit Batch| StateRepo
+    end
+
+    %% Single-root anchoring (critical)
+    Root --> Cron
+    Root -.-> Orchestrator
+    Root -.-> LocalExec
+    Root -.-> RemoteRunner
+
+    %% Styling
+    classDef plain fill:#ffffff,stroke:#333,stroke-width:1px;
+    classDef db fill:#e1f5fe,stroke:#01579b,stroke-width:2px;
+    classDef logic fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px;
+    classDef queue fill:#fff9c4,stroke:#fbc02d,stroke-width:2px;
+
+    class Cron,Scheduler,Dispatcher,Orchestrator,Manifest,LocalExec,RemoteRunner,Worker plain;
+    class StateRepo,BigQuery,GCS db;
+    class Logic logic;
+    class CloudTasks queue;
+```

package/functions/computation-system-v2/index.js

@@ -19,10 +19,12 @@ const { ManifestBuilder } = require('./framework/core/Manifest');
 const { Computation } = require('./framework/core/Computation');
 
 // Add computations to config
+// These are loaded from computation-system-v2/computations folder
 config.computations = [
     require('./computations/UserPortfolioSummary'),
     require('./computations/PopularInvestorProfileMetrics'),
     require('./computations/PopularInvestorRiskAssessment'),
+    require('./computations/PopularInvestorRiskMetrics'),
     // Add more computations here as they're migrated
 ];
 

package/functions/computation-system-v2/test/test-full-pipeline.js

@@ -0,0 +1,227 @@
+/**
+ * @fileoverview Full Pipeline Integration Test
+ * * Simulates the entire flow from Dispatcher -> Orchestrator -> Worker Pool -> Storage.
+ * * Verifies that:
+ * 1. The Dispatcher BLOCKS computations with missing mandatory data (RunAnalyzer).
+ * 2. Runnable computations are sent to the Worker Pool (if configured).
+ * 3. Results are stored in the TEST table (not production).
+ * * * USAGE:
+ * node test/test-full-pipeline.js --date 2026-01-24
+ */
+
+process.env.NODE_ENV = 'test';
+process.env.WORKER_LOCAL_MODE = 'true'; // Simulate workers locally
+process.env.WORKER_POOL_ENABLED = 'true';
+
+const fs = require('fs');
+const path = require('path');
+const { Orchestrator } = require('../framework/execution/Orchestrator');
+const { TestConfigBuilder } = require('./run-pipeline-test'); 
+const prodConfig = require('../config/bulltrackers.config');
+
+// ============================================================================
+// TEST RUNNER
+// ============================================================================
+
+async function runPipelineTest() {
+    const args = parseArgs();
+    console.log('\n╔════════════════════════════════════════════════════════════╗');
+    console.log('║           FULL PIPELINE INTEGRATION TEST                   ║');
+    console.log('╚════════════════════════════════════════════════════════════╝');
+    console.log(`📅 Target Date: ${args.date}`);
+    console.log(`🧪 Test Table:  computation_results_test`);
+    console.log(`👷 Worker Pool: ENABLED (Local Simulation)\n`);
+
+    // 0. DYNAMICALLY LOAD COMPUTATIONS
+    // This fixes the "Initialized with 0 computations" error
+    const computationsDir = path.join(__dirname, '../computations');
+    const loadedComputations = fs.readdirSync(computationsDir)
+        .filter(f => f.endsWith('.js'))
+        .map(f => require(path.join(computationsDir, f)));
+    
+    // Inject into config
+    prodConfig.computations = loadedComputations;
+    console.log(`📦 Auto-discovered ${prodConfig.computations.length} computations from /computations directory`);
+
+    // 1. CONFIGURE TEST ENVIRONMENT
+    const builder = new TestConfigBuilder(prodConfig, {
+        runId: `test-${Date.now()}`,
+        date: args.date,
+        testBucket: 'bulltrackers-computation-staging', // <--- FIX: Use your real bucket
+        batchSize: 1000,
+        concurrency: 2
+    });
+    
+    const testConfig = builder.build();
+    
+    // Explicitly enable worker pool in the test config
+    testConfig.workerPool = {
+        ...prodConfig.workerPool,
+        enabled: true,
+        localMode: true,
+        minEntitiesForOffload: 0 // Force everything to worker pool for testing
+    };
+
+    // 2. INITIALIZE ORCHESTRATOR
+    const orchestrator = new Orchestrator(testConfig, console);
+    await orchestrator.initialize();
+
+    // -------------------------------------------------------------------------
+    // PHASE 1: DISPATCHER VERIFICATION (Pre-Flight Check)
+    // -------------------------------------------------------------------------
+    console.log('🔍 PHASE 1: DISPATCHER ANALYSIS (The Gatekeeper)');
+    console.log('   Verifying that missing data BLOCKS execution...');
+    
+    const analysis = await orchestrator.analyze({ date: args.date });
+    
+    printAnalysisTable(analysis);
+
+    // Validation: Ensure nothing "Impossible" or "Blocked" is in the runnable list
+    const badRunnables = analysis.runnable.filter(r => 
+        analysis.blocked.find(b => b.name === r.name) || 
+        analysis.impossible.find(i => i.name === r.name)
+    );
+
+    if (badRunnables.length > 0) {
+        console.error('❌ CRITICAL FAILURE: Dispatcher marked blocked tasks as runnable!');
+        process.exit(1);
+    }
+    console.log('✅ Dispatcher logic validated. Blocked tasks will NOT run.\n');
+
+    // -------------------------------------------------------------------------
+    // PHASE 2: EXECUTION (Worker Pool & Storage)
+    // -------------------------------------------------------------------------
+    console.log('🚀 PHASE 2: PIPELINE EXECUTION');
+    console.log('   Running only valid tasks via Worker Pool...');
+
+    // We intercept storage to verify writes without polluting real DB (optional if using test table)
+    const storageInterceptor = new TestStorageInterceptor(orchestrator.storageManager);
+    orchestrator.storageManager = storageInterceptor;
+
+    // Run!
+    const result = await orchestrator.execute({ 
+        date: args.date,
+        dryRun: false // We want to test the full "write" path to the test table
+    });
+
+    // -------------------------------------------------------------------------
+    // PHASE 3: VERIFICATION & REPORTING
+    // -------------------------------------------------------------------------
+    console.log('\n📊 PHASE 3: FINAL REPORT');
+    
+    // 1. Did Blocked Tasks Run?
+    const blockedRan = result.completed.filter(c => 
+        analysis.blocked.find(b => b.name === c.name)
+    );
+    
+    if (blockedRan.length > 0) {
+        console.error(`❌ FAILURE: The following BLOCKED tasks executed anyway: ${blockedRan.map(c => c.name).join(', ')}`);
+    } else {
+        console.log('✅ SUCCESS: No blocked tasks were executed.');
+    }
+
+    // 2. Did Runnable Tasks Succeed?
+    const runnableNames = analysis.runnable.map(r => r.name);
+    const successfulRunnables = result.completed.filter(c => runnableNames.includes(c.name));
+    
+    if (successfulRunnables.length > 0) {
+        console.log(`✅ SUCCESS: ${successfulRunnables.length} runnable tasks completed successfully.`);
+    } else if (runnableNames.length > 0) {
+        console.warn('⚠️ WARNING: Runnable tasks existed but none completed (check errors below).');
+    } else {
+        console.log('ℹ️  No runnable tasks found (this is expected if data is missing).');
+    }
+
+    // 3. Storage Verification
+    const writes = storageInterceptor.getSummary();
+    console.log(`💾 Storage: Written ${writes.totalEntities} entity results to ${testConfig.resultStore.table}`);
+
+    if (result.summary.errors > 0) {
+        console.log('\n❌ EXECUTION ERRORS:');
+        result.errors.forEach(e => console.log(`   - ${e.name}: ${e.error}`));
+    }
+}
+
+// ============================================================================
+// HELPERS
+// ============================================================================
+
+function printAnalysisTable(analysis) {
+    console.log('\n   ┌──────────────────────────────┬──────────────┬──────────────────────────────────────────┐');
+    console.log('   │ Computation                  │ Status       │ Reason                                   │');
+    console.log('   ├──────────────────────────────┼──────────────┼──────────────────────────────────────────┤');
+    
+    const all = [
+        ...analysis.runnable.map(r => ({ ...r, status: 'RUNNABLE' })),
+        ...analysis.blocked.map(r => ({ ...r, status: 'BLOCKED' })),
+        ...analysis.impossible.map(r => ({ ...r, status: 'IMPOSSIBLE' })),
+        ...analysis.reRuns.map(r => ({ ...r, status: 'RERUN' })),
+        ...analysis.skipped.map(r => ({ ...r, status: 'SKIPPED' }))
+    ];
+
+    all.forEach(row => {
+        const name = row.name.padEnd(28).slice(0, 28);
+        const status = row.status.padEnd(12);
+        const reason = (row.reason || 'Ready to run').padEnd(40).slice(0, 40);
+        
+        let color = '\x1b[37m'; // White
+        if (row.status === 'BLOCKED') color = '\x1b[31m'; // Red
+        if (row.status === 'RUNNABLE') color = '\x1b[32m'; // Green
+        
+        console.log(`   │ ${color}${name}\x1b[0m │ ${color}${status}\x1b[0m │ ${reason} │`);
+    });
+    console.log('   └──────────────────────────────┴──────────────┴──────────────────────────────────────────┘\n');
+}
+
+/**
+ * Simple Storage Interceptor to verify writes
+ */
+class TestStorageInterceptor {
+    constructor(realStorage) {
+        this.realStorage = realStorage;
+        this.writes = [];
+        
+        // Proxy methods
+        return new Proxy(this, {
+            get(target, prop) {
+                if (prop in target) return target[prop];
+                if (typeof target.realStorage[prop] === 'function') {
+                    return target.realStorage[prop].bind(target.realStorage);
+                }
+                return target.realStorage[prop];
+            }
+        });
+    }
+
+    async commitResults(date, entry, results, depHashes) {
+        this.writes.push({
+            date,
+            computation: entry.name,
+            count: Object.keys(results).length
+        });
+        // Pass through to real storage (which is pointing to test table)
+        return this.realStorage.commitResults(date, entry, results, depHashes);
+    }
+
+    getSummary() {
+        return {
+            totalWrites: this.writes.length,
+            totalEntities: this.writes.reduce((sum, w) => sum + w.count, 0),
+            details: this.writes
+        };
+    }
+}
+
+function parseArgs() {
+    const args = process.argv.slice(2);
+    let date = new Date();
+    date.setDate(date.getDate() - 1); // Default to yesterday
+    
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--date') date = new Date(args[++i]);
+    }
+    
+    return { date: date.toISOString().slice(0, 10) };
+}
+
+runPipelineTest().catch(console.error);

package/functions/computation-system-v2/test/test-worker-pool.js

@@ -1,494 +1,266 @@
 /**
- * @fileoverview Worker Pool Test Harness
- * 
- * Tests the serverless worker pool functionality in LOCAL MODE.
- * No GCS or HTTP required - workers run in-process.
- * 
- * USAGE:
- *   node test-worker-pool.js [computation-name]
- * 
- * EXAMPLES:
- *   node test-worker-pool.js                           # Test with mock data
- *   node test-worker-pool.js UserPortfolioSummary      # Test specific computation
- * 
- * FEATURES:
- *   - Tests worker handler directly (no HTTP)
- *   - Tests RemoteTaskRunner in local mode
- *   - Tests full Orchestrator with worker pool
- *   - Measures performance and validates results
+ * @fileoverview Worker Pool Integration Test (Real Data)
+ * * Tests the Worker Pool pipeline using REAL BigQuery data.
+ * * WHAT IT DOES:
+ * 1. Connects to BigQuery using your local credentials.
+ * 2. Fetches REAL data for the specified date/computation.
+ * 3. Simulates the Orchestrator's data packaging.
+ * 4. Runs the worker logic locally (via RemoteTaskRunner localMode).
+ * * USAGE:
+ * node test/test-worker-pool.js --date 2026-01-24 --computation UserPortfolioSummary
+ * * FLAGS:
+ * --date YYYY-MM-DD       (Required) Target date
+ * --computation Name      (Default: UserPortfolioSummary)
+ * --limit N               (Default: 5) Number of entities to test
+ * --entities id1,id2      (Optional) Specific entities to test
  */
 
 const path = require('path');
+const { RemoteTaskRunner } = require('../framework/execution/RemoteTaskRunner');
+const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
+const { QueryBuilder } = require('../framework/data/QueryBuilder');
+const { DataFetcher } = require('../framework/data/DataFetcher');
+const config = require('../config/bulltrackers.config');
 
-// Set local mode BEFORE importing modules
+// Force Local Mode for the Worker Pool (runs logic in-process but uses the Runner's pipeline)
 process.env.WORKER_LOCAL_MODE = 'true';
 process.env.WORKER_POOL_ENABLED = 'true';
 
-// Import modules
-const { executeLocal, loadComputation, workerHandler } = require('../handlers/worker');
-const { RemoteTaskRunner } = require('../framework/execution/RemoteTaskRunner');
-
 // ============================================================================
-// MOCK DATA GENERATORS
+// HELPER: Mini Orchestrator (Data Loading)
 // ============================================================================
 
-/**
- * Generate mock portfolio data for testing
- */
-function generateMockPortfolioData(userId, positionCount = 5) {
-    const positions = [];
-    const instruments = ['AAPL', 'GOOGL', 'MSFT', 'AMZN', 'TSLA', 'META', 'NVDA', 'BRK.B', 'JPM', 'V'];
-    
-    for (let i = 0; i < positionCount; i++) {
-        const investedAmount = Math.random() * 10000 + 1000;
-        const profitPercent = (Math.random() * 100 - 20); // -20% to +80%
+class TestContext {
+    constructor() {
+        this.logger = console;
         
-        positions.push({
-            InstrumentID: 1000 + i,
-            Ticker: instruments[i % instruments.length],
-            Direction: Math.random() > 0.1 ? 'Buy' : 'Sell',
-            InvestedAmount: investedAmount,
-            Value: investedAmount * (1 + profitPercent / 100),
-            Profit: investedAmount * profitPercent / 100,
-            ProfitPercent: profitPercent,
-            OpenedDate: new Date(Date.now() - Math.random() * 365 * 24 * 60 * 60 * 1000).toISOString()
-        });
+        // Initialize Framework Data Layer
+        this.schemaRegistry = new SchemaRegistry(config.bigquery, this.logger);
+        this.queryBuilder = new QueryBuilder(config.bigquery, this.schemaRegistry, this.logger);
+        this.dataFetcher = new DataFetcher({ ...config.bigquery, tables: config.tables }, this.queryBuilder, this.logger);
     }
 
-    return {
-        Positions: positions,
-        TotalEquity: positions.reduce((sum, p) => sum + p.Value, 0),
-        TotalInvestedAmount: positions.reduce((sum, p) => sum + p.InvestedAmount, 0)
-    };
-}
-
-/**
- * Generate mock asset prices
- */
-function generateMockAssetPrices() {
-    return {
-        AAPL: { close: 175.50, change: 1.2 },
-        GOOGL: { close: 142.30, change: -0.5 },
-        MSFT: { close: 378.90, change: 0.8 },
-        AMZN: { close: 178.25, change: 2.1 },
-        TSLA: { close: 248.50, change: -1.5 },
-        META: { close: 505.75, change: 1.8 },
-        NVDA: { close: 875.20, change: 3.2 },
-        'BRK.B': { close: 410.30, change: 0.3 },
-        JPM: { close: 195.80, change: 0.9 },
-        V: { close: 275.40, change: 0.6 }
-    };
+    async initialize() {
+        console.log('🔌 Connecting to BigQuery and initializing Schema Registry...');
+        // We only warm cache for tables we'll likely need to save time
+        // In a real run, we might scan the computation's requirements
+    }
 }
 
 // ============================================================================
-// TEST CASES
+// MAIN RUNNER
 // ============================================================================
 
-/**
- * Test 1: Direct worker execution (executeLocal)
- */
-async function testDirectWorkerExecution() {
-    console.log('\n📦 TEST 1: Direct Worker Execution (executeLocal)');
-    console.log('=' .repeat(60));
-    
-    const entityId = 'user-test-001';
-    const date = new Date().toISOString().slice(0, 10);
-    
-    const contextPackage = {
-        entityData: {
-            'portfolio_snapshots': generateMockPortfolioData(entityId, 5),
-            'asset_prices': generateMockAssetPrices()
-        },
-        references: {},
-        dependencies: {},
-        config: {}
-    };
+async function runRealDataTest() {
+    const args = parseArgs();
+    console.log('\n╔════════════════════════════════════════════════════════════╗');
+    console.log('║       WORKER POOL INTEGRATION TEST (REAL DATA)            ║');
+    console.log('╚════════════════════════════════════════════════════════════╝');
+    console.log(`📅 Date:        ${args.date}`);
+    console.log(`🧮 Computation: ${args.computation}`);
+    console.log(`📉 Entity Limit:${args.limit}`);
+    console.log(`🔧 Local Mode:  ENABLED\n`);
 
-    const startTime = Date.now();
-    
+    const ctx = new TestContext();
+    await ctx.initialize();
+
+    // 1. Load the Computation Class
+    const computationPath = path.join(__dirname, `../computations/${args.computation}`);
+    let ComputationClass;
     try {
-        const result = await executeLocal({
-            computationName: 'UserPortfolioSummary',
-            entityId,
-            date,
-            contextPackage
-        });
-        
-        const duration = Date.now() - startTime;
-        
-        console.log(`✅ Worker executed successfully in ${duration}ms`);
-        console.log(`   Entity ID: ${result.entityId}`);
-        console.log(`   Result:`, JSON.stringify(result.result, null, 2));
-        
-        // Validate result structure
-        if (result.result) {
-            console.log(`   ✓ Has userId: ${!!result.result.userId}`);
-            console.log(`   ✓ Has totalValue: ${!!result.result.totalValue}`);
-            console.log(`   ✓ Has positionCount: ${result.result.positionCount}`);
-        }
-        
-        return { success: true, duration, result: result.result };
-    } catch (error) {
-        console.log(`❌ Worker execution failed: ${error.message}`);
-        console.log(error.stack);
-        return { success: false, error: error.message };
+        ComputationClass = require(computationPath);
+    } catch (e) {
+        throw new Error(`Could not load computation at ${computationPath}: ${e.message}`);
     }
-}
 
-/**
- * Test 2: Worker HTTP handler simulation
- */
-async function testWorkerHttpHandler() {
-    console.log('\n📦 TEST 2: Worker HTTP Handler (workerHandler)');
-    console.log('='.repeat(60));
-    
-    const entityId = 'user-http-001';
-    const date = new Date().toISOString().slice(0, 10);
-    
-    // Create mock request/response objects
-    const mockReq = {
-        body: {
-            computationName: 'UserPortfolioSummary',
-            entityId,
-            date,
-            localContext: {
-                entityData: {
-                    'portfolio_snapshots': generateMockPortfolioData(entityId, 3),
-                    'asset_prices': generateMockAssetPrices()
-                },
-                references: {},
-                dependencies: {},
-                config: {}
-            }
-        }
-    };
-    
-    let responseData = null;
-    let responseStatus = null;
-    
-    const mockRes = {
-        status: (code) => {
-            responseStatus = code;
-            return mockRes;
-        },
-        json: (data) => {
-            responseData = data;
-            return mockRes;
-        }
+    const compConfig = ComputationClass.getConfig();
+    const manifestEntry = {
+        name: compConfig.name.toLowerCase(),
+        originalName: compConfig.name,
+        type: compConfig.type,
+        requires: compConfig.requires || {},
+        dependencies: compConfig.dependencies || [],
+        hash: 'test-hash-123'
     };
 
-    const startTime = Date.now();
-    
-    try {
-        await workerHandler(mockReq, mockRes);
-        
-        const duration = Date.now() - startTime;
-        
-        console.log(`✅ HTTP handler completed in ${duration}ms`);
-        console.log(`   Status: ${responseStatus}`);
-        console.log(`   Response status: ${responseData?.status}`);
-        
-        if (responseData?.result) {
-            console.log(`   Total Value: $${responseData.result.totalValue?.toFixed(2)}`);
-            console.log(`   Positions: ${responseData.result.positionCount}`);
-        }
-        
-        return { success: responseStatus === 200, duration, response: responseData };
-    } catch (error) {
-        console.log(`❌ HTTP handler failed: ${error.message}`);
-        return { success: false, error: error.message };
+    console.log(`📦 Loaded ${manifestEntry.originalName}`);
+    console.log(`   Requires: ${Object.keys(manifestEntry.requires).join(', ')}`);
+
+    // 2. Determine Driver Table (for batching)
+    const driverTable = getDriverTable(manifestEntry.requires, config);
+    if (!driverTable) {
+        throw new Error(`Could not determine driver table (entityField) for ${args.computation}. Is it a global computation?`);
     }
-}
+    const driverEntityField = config.tables[driverTable].entityField;
+    console.log(`   Driver Table: ${driverTable} (${driverEntityField})`);
 
-/**
- * Test 3: RemoteTaskRunner in local mode
- */
-async function testRemoteTaskRunnerLocal() {
-    console.log('\n📦 TEST 3: RemoteTaskRunner (Local Mode)');
-    console.log('='.repeat(60));
-    
-    const config = {
-        workerPool: {
-            enabled: true,
-            localMode: true,
-            concurrency: 10
-        }
-    };
-    
-    const runner = new RemoteTaskRunner(config, console);
-    
-    // Create mock manifest entry
-    const entry = {
-        name: 'userportfoliosummary',
-        originalName: 'UserPortfolioSummary',
-        type: 'per-entity'
-    };
-    
-    const dateStr = new Date().toISOString().slice(0, 10);
-    const baseContext = {
-        references: {},
-        config: {}
-    };
-    
-    // Generate test entities
-    const entityIds = ['user-batch-001', 'user-batch-002', 'user-batch-003', 'user-batch-004', 'user-batch-005'];
-    const entityDataMap = new Map();
+    // 3. Fetch Data (The "Heavy Lifting" usually done by Orchestrator)
+    console.log('\n📥 Fetching REAL data from BigQuery...');
     
-    for (const entityId of entityIds) {
-        entityDataMap.set(entityId, {
-            'portfolio_snapshots': generateMockPortfolioData(entityId, Math.floor(Math.random() * 8) + 2),
-            'asset_prices': generateMockAssetPrices()
+    // Split requirements
+    const { batchRequires, globalRequires } = splitRequirements(manifestEntry.requires, driverTable, config);
+
+    // Fetch Global Data (Shared)
+    let globalData = {};
+    if (Object.keys(globalRequires).length > 0) {
+        console.log(`   Fetching global requirements: ${Object.keys(globalRequires).join(', ')}...`);
+        globalData = await ctx.dataFetcher.fetchForComputation(globalRequires, args.date);
+    }
+
+    // Fetch Batch Data (Per-Entity)
+    console.log(`   Fetching batch requirements: ${Object.keys(batchRequires).join(', ')}...`);
+    
+    // Create a specific entity filter if requested
+    if (args.entities.length > 0) {
+        // Inject where clause for specific entities
+        Object.values(batchRequires).forEach(req => {
+            req.where = req.where || {};
+            req.where[driverEntityField] = args.entities; // This relies on DataFetcher supporting array-based WHERE
         });
     }
-    
-    const depResults = {};
 
-    const startTime = Date.now();
+    // We fetch one large batch for the test
+    const batchStream = ctx.dataFetcher.fetchComputationBatched(batchRequires, args.date, args.limit);
     
-    try {
-        const { results, errors } = await runner.runBatch(
-            entry,
-            dateStr,
-            baseContext,
-            entityIds,
-            entityDataMap,
-            depResults
-        );
-        
-        const duration = Date.now() - startTime;
-        
-        console.log(`✅ Batch completed in ${duration}ms`);
-        console.log(`   Entities processed: ${Object.keys(results).length}/${entityIds.length}`);
-        console.log(`   Errors: ${errors.length}`);
-        console.log(`   Throughput: ${(entityIds.length / (duration / 1000)).toFixed(2)} entities/sec`);
-        
-        // Show sample results
-        const firstResult = Object.values(results)[0];
-        if (firstResult) {
-            console.log(`   Sample result - Total Value: $${firstResult.totalValue?.toFixed(2)}`);
-        }
-        
-        if (errors.length > 0) {
-            console.log(`   First error: ${errors[0].error}`);
-        }
-        
-        return { success: errors.length === 0, duration, resultCount: Object.keys(results).length };
-    } catch (error) {
-        console.log(`❌ RemoteTaskRunner failed: ${error.message}`);
-        console.log(error.stack);
-        return { success: false, error: error.message };
+    // Get the first batch
+    const { value: batch, done } = await batchStream.next();
+    
+    if (done || !batch || batch.entityIds.length === 0) {
+        console.warn('⚠️ No data found for this date/computation.');
+        return;
     }
-}
 
-/**
- * Test 4: Performance test with larger batch
- */
-async function testPerformance() {
-    console.log('\n📦 TEST 4: Performance Test (Large Batch)');
-    console.log('='.repeat(60));
-    
-    const config = {
+    console.log(`✅ Data fetched. Processing ${batch.entityIds.length} entities: ${batch.entityIds.join(', ')}`);
+
+    // 4. Initialize RemoteTaskRunner
+    // We configured WORKER_LOCAL_MODE=true, so this will run logic in-process
+    // but crucially, it will go through the _buildContextPackage logic.
+    const runner = new RemoteTaskRunner({
         workerPool: {
             enabled: true,
-            localMode: true,
-            concurrency: 50  // Higher concurrency for perf test
+            localMode: true, // Run in-process
+            concurrency: 5
+        }
+    }, console);
+
+    // Prepare Context
+    const baseContext = {
+        references: {}, // TODO: Load ref data if needed
+        config: {
+            project: config.bigquery.projectId,
+            tables: config.tables
         }
     };
-    
-    const runner = new RemoteTaskRunner(config, console);
-    
-    const entry = {
-        name: 'userportfoliosummary',
-        originalName: 'UserPortfolioSummary',
-        type: 'per-entity'
-    };
-    
-    const dateStr = new Date().toISOString().slice(0, 10);
-    const baseContext = { references: {}, config: {} };
-    
-    // Generate 100 test entities
-    const entityCount = 100;
-    const entityIds = Array.from({ length: entityCount }, (_, i) => `user-perf-${i.toString().padStart(4, '0')}`);
+
+    // Prepare Data Maps
     const entityDataMap = new Map();
-    
-    console.log(`   Generating ${entityCount} test entities...`);
-    
-    for (const entityId of entityIds) {
-        entityDataMap.set(entityId, {
-            'portfolio_snapshots': generateMockPortfolioData(entityId, Math.floor(Math.random() * 10) + 1),
-            'asset_prices': generateMockAssetPrices()
-        });
+    const combinedData = { ...batch.data, ...globalData };
+
+    for (const entityId of batch.entityIds) {
+        const entityData = filterDataForEntity(combinedData, entityId, driverEntityField, config);
+        entityDataMap.set(entityId, entityData);
     }
+
+    // 5. Execute via Worker Pool Logic
+    console.log('\n🚀 Executing via RemoteTaskRunner (Local Simulation)...');
     
-    console.log(`   Running batch...`);
     const startTime = Date.now();
     
-    try {
-        const { results, errors } = await runner.runBatch(
-            entry,
-            dateStr,
-            baseContext,
-            entityIds,
-            entityDataMap,
-            {}
-        );
-        
-        const duration = Date.now() - startTime;
-        const throughput = (entityCount / (duration / 1000)).toFixed(2);
-        
-        console.log(`\n   📊 PERFORMANCE RESULTS:`);
-        console.log(`   ─────────────────────────`);
-        console.log(`   Total entities:     ${entityCount}`);
-        console.log(`   Successful:         ${Object.keys(results).length}`);
-        console.log(`   Failed:             ${errors.length}`);
-        console.log(`   Total duration:     ${duration}ms`);
-        console.log(`   Throughput:         ${throughput} entities/sec`);
-        console.log(`   Avg per entity:     ${(duration / entityCount).toFixed(2)}ms`);
-        
-        return { 
-            success: errors.length === 0, 
-            duration, 
-            throughput: parseFloat(throughput),
-            resultCount: Object.keys(results).length 
-        };
-    } catch (error) {
-        console.log(`❌ Performance test failed: ${error.message}`);
-        return { success: false, error: error.message };
+    const { results, errors } = await runner.runBatch(
+        manifestEntry,
+        args.date,
+        baseContext,
+        batch.entityIds,
+        entityDataMap,
+        {} // Mock dependencies for now
+    );
+
+    const duration = Date.now() - startTime;
+
+    // 6. Report Results
+    console.log('\n╔════════════════════════════════════════════════════════════╗');
+    console.log('║                     EXECUTION REPORT                       ║');
+    console.log('╚════════════════════════════════════════════════════════════╝');
+    console.log(`⏱️  Duration: ${duration}ms`);
+    console.log(`✅ Success:  ${Object.keys(results).length}`);
+    console.log(`❌ Errors:   ${errors.length}`);
+
+    if (Object.keys(results).length > 0) {
+        console.log('\n🔍 Sample Result (First Entity):');
+        const sampleId = Object.keys(results)[0];
+        console.log(`   Entity: ${sampleId}`);
+        console.log(JSON.stringify(results[sampleId], null, 2));
     }
-}
 
-/**
- * Test 5: Error handling
- */
-async function testErrorHandling() {
-    console.log('\n📦 TEST 5: Error Handling');
-    console.log('='.repeat(60));
-    
-    // Test 5a: Unknown computation
-    console.log('\n   5a. Unknown computation name:');
-    try {
-        const result = await executeLocal({
-            computationName: 'NonExistentComputation',
-            entityId: 'test',
-            date: '2026-01-25',
-            contextPackage: {}
-        });
-        console.log(`   ❌ Should have thrown error, got: ${JSON.stringify(result)}`);
-    } catch (error) {
-        console.log(`   ✅ Correctly threw error: ${error.message}`);
+    if (errors.length > 0) {
+        console.log('\n❌ Errors:');
+        errors.forEach(e => console.log(`   [${e.entityId}] ${e.error}`));
     }
-    
-    // Test 5b: Missing data
-    console.log('\n   5b. Missing required data:');
-    try {
-        const result = await executeLocal({
-            computationName: 'UserPortfolioSummary',
-            entityId: 'test-missing-data',
-            date: '2026-01-25',
-            contextPackage: {
-                entityData: {},  // No portfolio data
-                references: {},
-                dependencies: {}
-            }
-        });
-        console.log(`   ✅ Handled gracefully: ${result.result === null ? 'null result' : JSON.stringify(result.result)}`);
-    } catch (error) {
-        console.log(`   ⚠️ Error (may be expected): ${error.message}`);
+}
+
+// ============================================================================
+// UTILS (Duplicated from Orchestrator logic)
+// ============================================================================
+
+function getDriverTable(requires, config) {
+    for (const name of Object.keys(requires)) {
+        const conf = config.tables[name];
+        if (conf && conf.entityField) return name;
     }
-    
-    return { success: true };
+    return null; 
 }
 
-/**
- * Test 6: Load all computations
- */
-async function testLoadComputations() {
-    console.log('\n📦 TEST 6: Load All Computations');
-    console.log('='.repeat(60));
-    
-    const computationNames = [
-        'UserPortfolioSummary',
-        'PopularInvestorProfileMetrics',
-        'PopularInvestorRiskAssessment',
-        'PopularInvestorRiskMetrics'
-    ];
-    
-    const results = [];
-    
-    for (const name of computationNames) {
-        try {
-            const cls = loadComputation(name);
-            if (cls) {
-                console.log(`   ✅ ${name}: Loaded successfully`);
-                results.push({ name, success: true });
-            } else {
-                console.log(`   ❌ ${name}: Failed to load (returned null)`);
-                results.push({ name, success: false });
-            }
-        } catch (error) {
-            console.log(`   ❌ ${name}: ${error.message}`);
-            results.push({ name, success: false, error: error.message });
+function splitRequirements(requires, driverTable, config) {
+    const batchRequires = {};
+    const globalRequires = {};
+    const driverConfig = config.tables[driverTable];
+    const driverEntityField = driverConfig ? driverConfig.entityField : null;
+    
+    for (const [name, spec] of Object.entries(requires)) {
+        const conf = config.tables[name];
+        if (conf && conf.entityField === driverEntityField) {
+            batchRequires[name] = spec;
+        } else {
+            globalRequires[name] = spec;
         }
     }
-    
-    const successCount = results.filter(r => r.success).length;
-    console.log(`\n   Summary: ${successCount}/${computationNames.length} loaded successfully`);
-    
-    return { success: successCount === computationNames.length, results };
+    return { batchRequires, globalRequires };
 }
 
-// ============================================================================
-// MAIN TEST RUNNER
-// ============================================================================
+function filterDataForEntity(data, id, driverEntityField, config) {
+    const out = {};
+    Object.entries(data).forEach(([tbl, d]) => {
+        const conf = config.tables[tbl] || {};
+        if (conf.entityField === driverEntityField && d && !Array.isArray(d)) {
+             out[tbl] = d[id] || null;
+        } else {
+             out[tbl] = d; 
+        }
+    });
+    return out;
+}
 
-async function runAllTests() {
-    console.log('\n');
-    console.log('╔════════════════════════════════════════════════════════════╗');
-    console.log('║           WORKER POOL LOCAL TEST HARNESS                  ║');
-    console.log('╚════════════════════════════════════════════════════════════╝');
-    console.log(`\n📅 Date: ${new Date().toISOString()}`);
-    console.log(`🔧 Mode: LOCAL (WORKER_LOCAL_MODE=true)`);
-    console.log(`🔧 Worker Pool: ENABLED (WORKER_POOL_ENABLED=true)`);
-    
-    const results = {};
-    
-    // Run all tests
-    results.directExecution = await testDirectWorkerExecution();
-    results.httpHandler = await testWorkerHttpHandler();
-    results.remoteTaskRunner = await testRemoteTaskRunnerLocal();
-    results.performance = await testPerformance();
-    results.errorHandling = await testErrorHandling();
-    results.loadComputations = await testLoadComputations();
-    
-    // Summary
-    console.log('\n');
-    console.log('╔════════════════════════════════════════════════════════════╗');
-    console.log('║                      TEST SUMMARY                          ║');
-    console.log('╚════════════════════════════════════════════════════════════╝');
-    
-    let passCount = 0;
-    let failCount = 0;
-    
-    for (const [name, result] of Object.entries(results)) {
-        const icon = result.success ? '✅' : '❌';
-        const status = result.success ? 'PASS' : 'FAIL';
-        console.log(`${icon} ${name}: ${status}`);
-        if (result.success) passCount++;
-        else failCount++;
-    }
-    
-    console.log(`\n📊 Results: ${passCount} passed, ${failCount} failed`);
-    
-    if (results.performance && results.performance.throughput) {
-        console.log(`📈 Performance: ${results.performance.throughput} entities/sec`);
+function parseArgs() {
+    const args = process.argv.slice(2);
+    const options = {
+        date: new Date().toISOString().slice(0, 10),
+        computation: 'UserPortfolioSummary',
+        limit: 5,
+        entities: []
+    };
+
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === '--date') options.date = args[++i];
+        if (arg === '--computation') options.computation = args[++i];
+        if (arg === '--limit') options.limit = parseInt(args[++i], 10);
+        if (arg === '--entities') options.entities = args[++i].split(',');
     }
-    
-    // Exit with appropriate code
-    process.exit(failCount > 0 ? 1 : 0);
+    return options;
 }
 
-// Run tests
-runAllTests().catch(err => {
-    console.error('Fatal test error:', err);
+// Run
+runRealDataTest().catch(e => {
+    console.error('\n💥 Fatal Error:', e);
     process.exit(1);
-});
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.736",
+  "version": "1.0.737",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [