bulltrackers-module 1.0.736 → 1.0.738

package/functions/computation-system-v2/test/test-full-pipeline.js

@@ -0,0 +1,227 @@
+/**
+ * @fileoverview Full Pipeline Integration Test
+ * * Simulates the entire flow from Dispatcher -> Orchestrator -> Worker Pool -> Storage.
+ * * Verifies that:
+ * 1. The Dispatcher BLOCKS computations with missing mandatory data (RunAnalyzer).
+ * 2. Runnable computations are sent to the Worker Pool (if configured).
+ * 3. Results are stored in the TEST table (not production).
+ * * * USAGE:
+ * node test/test-full-pipeline.js --date 2026-01-24
+ */
+
+process.env.NODE_ENV = 'test';
+process.env.WORKER_LOCAL_MODE = 'true'; // Simulate workers locally
+process.env.WORKER_POOL_ENABLED = 'true';
+
+const fs = require('fs');
+const path = require('path');
+const { Orchestrator } = require('../framework/execution/Orchestrator');
+const { TestConfigBuilder } = require('./run-pipeline-test'); 
+const prodConfig = require('../config/bulltrackers.config');
+
+// ============================================================================
+// TEST RUNNER
+// ============================================================================
+
+async function runPipelineTest() {
+    const args = parseArgs();
+    console.log('\n╔════════════════════════════════════════════════════════════╗');
+    console.log('║           FULL PIPELINE INTEGRATION TEST                   ║');
+    console.log('╚════════════════════════════════════════════════════════════╝');
+    console.log(`📅 Target Date: ${args.date}`);
+    console.log(`🧪 Test Table:  computation_results_test`);
+    console.log(`👷 Worker Pool: ENABLED (Local Simulation)\n`);
+
+    // 0. DYNAMICALLY LOAD COMPUTATIONS
+    // This fixes the "Initialized with 0 computations" error
+    const computationsDir = path.join(__dirname, '../computations');
+    const loadedComputations = fs.readdirSync(computationsDir)
+        .filter(f => f.endsWith('.js'))
+        .map(f => require(path.join(computationsDir, f)));
+    
+    // Inject into config
+    prodConfig.computations = loadedComputations;
+    console.log(`📦 Auto-discovered ${prodConfig.computations.length} computations from /computations directory`);
+
+    // 1. CONFIGURE TEST ENVIRONMENT
+    const builder = new TestConfigBuilder(prodConfig, {
+        runId: `test-${Date.now()}`,
+        date: args.date,
+        testBucket: 'bulltrackers-computation-staging', // <--- FIX: Use your real bucket
+        batchSize: 1000,
+        concurrency: 2
+    });
+    
+    const testConfig = builder.build();
+    
+    // Explicitly enable worker pool in the test config
+    testConfig.workerPool = {
+        ...prodConfig.workerPool,
+        enabled: true,
+        localMode: true,
+        minEntitiesForOffload: 0 // Force everything to worker pool for testing
+    };
+
+    // 2. INITIALIZE ORCHESTRATOR
+    const orchestrator = new Orchestrator(testConfig, console);
+    await orchestrator.initialize();
+
+    // -------------------------------------------------------------------------
+    // PHASE 1: DISPATCHER VERIFICATION (Pre-Flight Check)
+    // -------------------------------------------------------------------------
+    console.log('🔍 PHASE 1: DISPATCHER ANALYSIS (The Gatekeeper)');
+    console.log('   Verifying that missing data BLOCKS execution...');
+    
+    const analysis = await orchestrator.analyze({ date: args.date });
+    
+    printAnalysisTable(analysis);
+
+    // Validation: Ensure nothing "Impossible" or "Blocked" is in the runnable list
+    const badRunnables = analysis.runnable.filter(r => 
+        analysis.blocked.find(b => b.name === r.name) || 
+        analysis.impossible.find(i => i.name === r.name)
+    );
+
+    if (badRunnables.length > 0) {
+        console.error('❌ CRITICAL FAILURE: Dispatcher marked blocked tasks as runnable!');
+        process.exit(1);
+    }
+    console.log('✅ Dispatcher logic validated. Blocked tasks will NOT run.\n');
+
+    // -------------------------------------------------------------------------
+    // PHASE 2: EXECUTION (Worker Pool & Storage)
+    // -------------------------------------------------------------------------
+    console.log('🚀 PHASE 2: PIPELINE EXECUTION');
+    console.log('   Running only valid tasks via Worker Pool...');
+
+    // We intercept storage to verify writes without polluting real DB (optional if using test table)
+    const storageInterceptor = new TestStorageInterceptor(orchestrator.storageManager);
+    orchestrator.storageManager = storageInterceptor;
+
+    // Run!
+    const result = await orchestrator.execute({ 
+        date: args.date,
+        dryRun: false // We want to test the full "write" path to the test table
+    });
+
+    // -------------------------------------------------------------------------
+    // PHASE 3: VERIFICATION & REPORTING
+    // -------------------------------------------------------------------------
+    console.log('\n📊 PHASE 3: FINAL REPORT');
+    
+    // 1. Did Blocked Tasks Run?
+    const blockedRan = result.completed.filter(c => 
+        analysis.blocked.find(b => b.name === c.name)
+    );
+    
+    if (blockedRan.length > 0) {
+        console.error(`❌ FAILURE: The following BLOCKED tasks executed anyway: ${blockedRan.map(c => c.name).join(', ')}`);
+    } else {
+        console.log('✅ SUCCESS: No blocked tasks were executed.');
+    }
+
+    // 2. Did Runnable Tasks Succeed?
+    const runnableNames = analysis.runnable.map(r => r.name);
+    const successfulRunnables = result.completed.filter(c => runnableNames.includes(c.name));
+    
+    if (successfulRunnables.length > 0) {
+        console.log(`✅ SUCCESS: ${successfulRunnables.length} runnable tasks completed successfully.`);
+    } else if (runnableNames.length > 0) {
+        console.warn('⚠️ WARNING: Runnable tasks existed but none completed (check errors below).');
+    } else {
+        console.log('ℹ️  No runnable tasks found (this is expected if data is missing).');
+    }
+
+    // 3. Storage Verification
+    const writes = storageInterceptor.getSummary();
+    console.log(`💾 Storage: Written ${writes.totalEntities} entity results to ${testConfig.resultStore.table}`);
+
+    if (result.summary.errors > 0) {
+        console.log('\n❌ EXECUTION ERRORS:');
+        result.errors.forEach(e => console.log(`   - ${e.name}: ${e.error}`));
+    }
+}
+
+// ============================================================================
+// HELPERS
+// ============================================================================
+
+function printAnalysisTable(analysis) {
+    console.log('\n   ┌──────────────────────────────┬──────────────┬──────────────────────────────────────────┐');
+    console.log('   │ Computation                  │ Status       │ Reason                                   │');
+    console.log('   ├──────────────────────────────┼──────────────┼──────────────────────────────────────────┤');
+    
+    const all = [
+        ...analysis.runnable.map(r => ({ ...r, status: 'RUNNABLE' })),
+        ...analysis.blocked.map(r => ({ ...r, status: 'BLOCKED' })),
+        ...analysis.impossible.map(r => ({ ...r, status: 'IMPOSSIBLE' })),
+        ...analysis.reRuns.map(r => ({ ...r, status: 'RERUN' })),
+        ...analysis.skipped.map(r => ({ ...r, status: 'SKIPPED' }))
+    ];
+
+    all.forEach(row => {
+        const name = row.name.padEnd(28).slice(0, 28);
+        const status = row.status.padEnd(12);
+        const reason = (row.reason || 'Ready to run').padEnd(40).slice(0, 40);
+        
+        let color = '\x1b[37m'; // White
+        if (row.status === 'BLOCKED') color = '\x1b[31m'; // Red
+        if (row.status === 'RUNNABLE') color = '\x1b[32m'; // Green
+        
+        console.log(`   │ ${color}${name}\x1b[0m │ ${color}${status}\x1b[0m │ ${reason} │`);
+    });
+    console.log('   └──────────────────────────────┴──────────────┴──────────────────────────────────────────┘\n');
+}
+
+/**
+ * Simple Storage Interceptor to verify writes
+ */
+class TestStorageInterceptor {
+    constructor(realStorage) {
+        this.realStorage = realStorage;
+        this.writes = [];
+        
+        // Proxy methods
+        return new Proxy(this, {
+            get(target, prop) {
+                if (prop in target) return target[prop];
+                if (typeof target.realStorage[prop] === 'function') {
+                    return target.realStorage[prop].bind(target.realStorage);
+                }
+                return target.realStorage[prop];
+            }
+        });
+    }
+
+    async commitResults(date, entry, results, depHashes) {
+        this.writes.push({
+            date,
+            computation: entry.name,
+            count: Object.keys(results).length
+        });
+        // Pass through to real storage (which is pointing to test table)
+        return this.realStorage.commitResults(date, entry, results, depHashes);
+    }
+
+    getSummary() {
+        return {
+            totalWrites: this.writes.length,
+            totalEntities: this.writes.reduce((sum, w) => sum + w.count, 0),
+            details: this.writes
+        };
+    }
+}
+
+function parseArgs() {
+    const args = process.argv.slice(2);
+    let date = new Date();
+    date.setDate(date.getDate() - 1); // Default to yesterday
+    
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--date') date = new Date(args[++i]);
+    }
+    
+    return { date: date.toISOString().slice(0, 10) };
+}
+
+runPipelineTest().catch(console.error);