bulltrackers-module 1.0.735 → 1.0.737

package/functions/computation-system-v2/test/run-pipeline-test.js

@@ -0,0 +1,554 @@
+/**
+ * @fileoverview Production Pipeline Test Runner
+ * * Tests the ENTIRE production pipeline with:
+ * - Cost tracking and hard limits
+ * - Test tables/buckets (isolated from production)
+ * - Entity filtering (test subset of users)
+ * - Lineage tracking
+ * - Performance profiling
+ * * Run: node test/run-pipeline-test.js --date 2026-01-24 --entities user1,user2 --max-cost 5 --batch-size 1000
+ * * * UPDATE: Sets NODE_ENV='test' and force:true to bypass Checkpointer locks AND "Up-To-Date" checks.
+ */
+
+// FORCE TEST ENVIRONMENT
+process.env.NODE_ENV = 'test';
+
+const path = require('path');
+const fs = require('fs');
+const { Orchestrator } = require('../framework/execution/Orchestrator');
+const { StorageManager } = require('../framework/storage/StorageManager');
+const { CostTracker } = require('../framework/cost/CostTracker');
+const { ComputationProfiler } = require('../framework/monitoring/Profiler');
+
+// ============================================================================
+// TEST CONFIGURATION BUILDER
+// ============================================================================
+
+class TestConfigBuilder {
+    constructor(productionConfig, testOptions) {
+        this.prodConfig = productionConfig;
+        this.testOptions = testOptions;
+    }
+
+    build() {
+        // Clone production config
+        const testConfig = JSON.parse(JSON.stringify(this.prodConfig));
+
+        // Restore non-serializable objects (Classes and Functions)
+        testConfig.computations = this.prodConfig.computations;
+        testConfig.rules = this.prodConfig.rules;
+        
+        // Override GCS to use test bucket
+        const prodBucket = testConfig.gcs?.bucket;
+        testConfig.gcs = {
+            bucket: this.testOptions.testBucket || `${prodBucket}-test`,
+            prefix: 'test-staging'
+        };
+
+        // Override result storage to use test table
+        testConfig.resultStore = {
+            table: 'test_computation_results',
+            partitionField: 'date',
+            clusterFields: ['computation_name', 'category']
+        };
+
+        // Add test-specific metadata
+        testConfig.testMode = {
+            enabled: true,
+            runId: this.testOptions.runId || `test-${Date.now()}`,
+            targetEntities: this.testOptions.entities || null,
+            maxCostUSD: this.testOptions.maxCost || 10,
+            dateOverride: this.testOptions.date
+        };
+
+        // Override execution settings for testing
+        testConfig.execution = {
+            ...testConfig.execution,
+            entityConcurrency: this.testOptions.concurrency || 5, // Default 5 or user provided
+            batchSize: this.testOptions.batchSize || 100 // Default 100 or user provided
+        };
+
+        // FIX: Explicitly enable lock bypass in config
+        testConfig.bypassLocks = true;
+
+        return testConfig;
+    }
+}
+
+// ============================================================================
+// RUNTIME INTERCEPTORS (Proxy Pattern)
+// ============================================================================
+
+/**
+ * Intercepts StorageManager to redirect writes to test tables
+ */
+class TestStorageInterceptor {
+    constructor(storageManager, testConfig) {
+        this.storage = storageManager;
+        this.testConfig = testConfig;
+        this.capturedWrites = [];
+    }
+
+    // Wrap commitResults to capture what would be written
+    async commitResults(date, entry, results, depHashes) {
+        this.capturedWrites.push({
+            date,
+            computation: entry.name,
+            entityCount: Object.keys(results).length,
+            timestamp: new Date().toISOString()
+        });
+
+        // Actually write to test storage
+        return this.storage.commitResults(date, entry, results, depHashes);
+    }
+
+    async finalizeResults(date, entry) {
+        return this.storage.finalizeResults(date, entry);
+    }
+
+    // Proxy other methods
+    async initCheckpoint(...args) { return this.storage.initCheckpoint(...args); }
+    async updateCheckpoint(...args) { return this.storage.updateCheckpoint(...args); }
+    async completeCheckpoint(...args) { return this.storage.completeCheckpoint(...args); }
+    async getLatestCheckpoint(...args) { return this.storage.getLatestCheckpoint(...args); }
+    async savePerformanceReport(...args) { return this.storage.savePerformanceReport(...args); }
+
+    getSummary() {
+        return {
+            totalWrites: this.capturedWrites.length,
+            totalEntities: this.capturedWrites.reduce((sum, w) => sum + w.entityCount, 0),
+            byComputation: this.capturedWrites.reduce((acc, w) => {
+                acc[w.computation] = (acc[w.computation] || 0) + w.entityCount;
+                return acc;
+            }, {})
+        };
+    }
+}
+
+/**
+ * Intercepts CostTracker to enforce hard cost limits
+ */
+class TestCostEnforcer {
+    constructor(costTracker, maxCostUSD) {
+        this.tracker = costTracker;
+        this.maxCostUSD = maxCostUSD;
+        this.currentCostUSD = 0;
+        this.costByComputation = {};
+    }
+
+    async trackCost(computationName, date, bytesProcessed) {
+        // Calculate cost (same as production)
+        const COST_PER_TB = 5.0;
+        const BYTES_PER_TB = 1099511627776;
+        const cost = (bytesProcessed / BYTES_PER_TB) * COST_PER_TB;
+
+        this.currentCostUSD += cost;
+        this.costByComputation[computationName] = (this.costByComputation[computationName] || 0) + cost;
+
+        // HARD LIMIT ENFORCEMENT
+        if (this.currentCostUSD > this.maxCostUSD) {
+            throw new Error(
+                `🚨 COST LIMIT EXCEEDED: $${this.currentCostUSD.toFixed(4)} > $${this.maxCostUSD}. ` +
+                `Last computation: ${computationName}. Test aborted.`
+            );
+        }
+
+        // Still track in production tracker (to test table)
+        await this.tracker.trackCost(computationName, date, bytesProcessed);
+
+        return cost;
+    }
+
+    getSummary() {
+        return {
+            totalCostUSD: this.currentCostUSD,
+            maxCostUSD: this.maxCostUSD,
+            utilizationPct: (this.currentCostUSD / this.maxCostUSD) * 100,
+            byComputation: Object.entries(this.costByComputation)
+                .map(([name, cost]) => ({ name, cost }))
+                .sort((a, b) => b.cost - a.cost)
+        };
+    }
+}
+
+// ============================================================================
+// LINEAGE TRACKER
+// ============================================================================
+
+class TestLineageTracker {
+    constructor(manifest) {
+        this.manifest = manifest;
+        this.executionOrder = [];
+        this.dependencyGraph = new Map();
+    }
+
+    recordExecution(computationName, pass, status, duration) {
+        this.executionOrder.push({
+            name: computationName,
+            pass,
+            status,
+            duration,
+            timestamp: new Date().toISOString()
+        });
+    }
+
+    buildGraph() {
+        for (const entry of this.manifest) {
+            this.dependencyGraph.set(entry.name, {
+                name: entry.originalName,
+                pass: entry.pass,
+                dependencies: entry.dependencies,
+                category: entry.category,
+                type: entry.type
+            });
+        }
+    }
+
+    generateReport() {
+        this.buildGraph();
+
+        const report = {
+            totalComputations: this.manifest.length,
+            executionOrder: this.executionOrder,
+            dependencyGraph: Array.from(this.dependencyGraph.entries()).map(([name, data]) => ({
+                computation: data.name,
+                pass: data.pass,
+                dependencies: data.dependencies.map(dep => {
+                    const depData = this.dependencyGraph.get(dep);
+                    return depData ? depData.name : dep;
+                }),
+                category: data.category,
+                type: data.type
+            })),
+            passSummary: this._summarizeByPass()
+        };
+
+        return report;
+    }
+
+    _summarizeByPass() {
+        const byPass = {};
+        for (const entry of this.manifest) {
+            if (!byPass[entry.pass]) byPass[entry.pass] = [];
+            byPass[entry.pass].push(entry.originalName);
+        }
+        return byPass;
+    }
+
+    printGraph() {
+        console.log('\n📊 COMPUTATION DEPENDENCY GRAPH\n');
+        
+        const byPass = this._summarizeByPass();
+        const passNumbers = Object.keys(byPass).map(Number).sort((a, b) => a - b);
+
+        for (const pass of passNumbers) {
+            console.log(`Pass ${pass}:`);
+            for (const compName of byPass[pass]) {
+                const entry = this.manifest.find(e => e.originalName === compName);
+                const deps = entry.dependencies.length > 0 
+                    ? ` (depends on: ${entry.dependencies.join(', ')})`
+                    : '';
+                console.log(`  • ${compName}${deps}`);
+            }
+            console.log('');
+        }
+    }
+}
+
+// ============================================================================
+// TEST RUNNER
+// ============================================================================
+
+class PipelineTestRunner {
+    constructor(options) {
+        this.options = options;
+        this.results = {
+            runId: null,
+            startTime: null,
+            endTime: null,
+            status: 'pending',
+            computations: [],
+            cost: null,
+            storage: null,
+            lineage: null,
+            performance: []
+        };
+    }
+
+    async run() {
+        console.log('╔════════════════════════════════════════════════════════════╗');
+        console.log('║         PRODUCTION PIPELINE FULL INTEGRATION TEST          ║');
+        console.log('╚════════════════════════════════════════════════════════════╝\n');
+
+        this.results.startTime = new Date().toISOString();
+        this.results.runId = `test-${Date.now()}`;
+
+        try {
+            // 1. Load production config
+            const prodConfig = this._loadProductionConfig();
+            
+            // 2. Build test config
+            const testOptions = {
+                runId: this.results.runId,
+                date: this.options.date,
+                entities: this.options.entities,
+                maxCost: this.options.maxCost || 10,
+                testDataset: this.options.testDataset,
+                testBucket: this.options.testBucket,
+                batchSize: this.options.batchSize,
+                concurrency: this.options.concurrency
+            };
+
+            const builder = new TestConfigBuilder(prodConfig, testOptions);
+            const testConfig = builder.build();
+
+            console.log('🔧 Test Configuration:');
+            console.log(`   Run ID: ${testConfig.testMode.runId}`);
+            console.log(`   Date: ${testConfig.testMode.dateOverride}`);
+            console.log(`   Entities: ${testConfig.testMode.targetEntities || 'ALL'}`);
+            console.log(`   Max Cost: $${testConfig.testMode.maxCostUSD}`);
+            console.log(`   Test Dataset: ${testConfig.bigquery.dataset}`);
+            console.log(`   Test Bucket: ${testConfig.gcs.bucket}`);
+            console.log(`   Batch Size: ${testConfig.execution.batchSize}`);
+            console.log(`   Concurrency: ${testConfig.execution.entityConcurrency}`);
+            console.log(`   Bypass Locks: ${testConfig.bypassLocks ? 'YES' : 'NO'}\n`);
+
+            // 3. Initialize orchestrator (PRODUCTION CODE)
+            const orchestrator = new Orchestrator(testConfig, console);
+
+            // PATCH: Monkey-patch _executeComputation to debug silent errors
+            const originalExecuteComp = orchestrator._executeComputation;
+            orchestrator._executeComputation = async function(entry, ...args) {
+                try {
+                    return await originalExecuteComp.call(this, entry, ...args);
+                } catch (e) {
+                    console.error(JSON.stringify(e, null, 2));
+                    if (e.stack) console.error(e.stack);
+                    throw e; // Re-throw to be caught by the standard handler
+                }
+            };
+
+            await orchestrator.initialize();
+
+            console.log(`📦 Loaded ${orchestrator.manifest.length} computations\n`);
+
+            // 4. Setup interceptors
+            const storageInterceptor = new TestStorageInterceptor(
+                orchestrator.storageManager,
+                testConfig
+            );
+            const costEnforcer = new TestCostEnforcer(
+                orchestrator.storageManager.costTracker || { trackCost: async () => {} },
+                testConfig.testMode.maxCostUSD
+            );
+            const lineageTracker = new TestLineageTracker(orchestrator.manifest);
+
+            // Inject interceptors via proxy
+            // FIX: Use .bind() to ensure methods don't lose 'this' context
+            orchestrator.storageManager = new Proxy(storageInterceptor, {
+                get: (target, prop) => {
+                    // 1. Check interceptor first
+                    if (prop in target) return target[prop];
+                    
+                    // 2. Fallback to original manager
+                    const originalManager = target.storage;
+                    const value = originalManager[prop];
+                    
+                    // 3. CRITICAL: Bind functions to original manager to preserve 'this'
+                    if (typeof value === 'function') {
+                        return value.bind(originalManager);
+                    }
+                    return value;
+                }
+            });
+
+            // Print dependency graph
+            lineageTracker.printGraph();
+
+            // 5. Execute pipeline (PRODUCTION CODE)
+            console.log('🚀 Starting pipeline execution...\n');
+
+            const execResult = await orchestrator.execute({
+                date: testConfig.testMode.dateOverride,
+                entities: testConfig.testMode.targetEntities,
+                dryRun: false, // Actually write to test tables
+                force: true // FIX: FORCE LOCK BYPASS & FORCE RE-RUN OF VALID COMPUTATIONS
+            });
+
+            // 6. Collect results
+            this.results.computations = execResult.completed.map(c => ({
+                name: c.name,
+                status: c.status,
+                resultCount: c.resultCount,
+                duration: c.duration
+            }));
+
+            // FIX: Correctly report failure if errors occurred
+            if (execResult.summary.errors > 0 || (execResult.errors && execResult.errors.length > 0)) {
+                 this.results.status = 'failed';
+                 this.results.errors = execResult.errors;
+                 console.error(`\n❌ Pipeline finished with ${execResult.summary.errors} errors.`);
+            } else if (execResult.summary.completed === 0 && orchestrator.manifest.length > 0) {
+                 this.results.status = 'failed';
+                 console.error('\n❌ Pipeline finished but 0 computations completed.');
+            } else {
+                 this.results.status = 'success';
+            }
+
+            this.results.cost = costEnforcer.getSummary();
+            this.results.storage = storageInterceptor.getSummary();
+            this.results.lineage = lineageTracker.generateReport();
+            this.results.endTime = new Date().toISOString();
+
+            // 7. Print summary
+            this._printSummary();
+
+            // 8. Save report
+            this._saveReport();
+
+            return this.results;
+
+        } catch (error) {
+            this.results.status = 'failed';
+            this.results.error = error.message;
+            this.results.endTime = new Date().toISOString();
+
+            console.error('\n❌ TEST FAILED:', error); // Log full error object
+            if (error.stack) console.error(error.stack);
+
+            this._saveReport();
+            throw error;
+        }
+    }
+
+    _loadProductionConfig() {
+        // Load real production config
+        const prodConfig = require('../config/bulltrackers.config');
+
+        // Dynamically load ALL computations from directory
+        const computationsDir = path.join(__dirname, '../computations');
+        const files = fs.readdirSync(computationsDir).filter(f => f.endsWith('.js'));
+
+        prodConfig.computations = files.map(f => {
+            const ComputationClass = require(path.join(computationsDir, f));
+            return ComputationClass;
+        });
+
+        console.log(`✅ Loaded ${prodConfig.computations.length} computations from directory\n`);
+
+        return prodConfig;
+    }
+
+    _printSummary() {
+        console.log('\n╔════════════════════════════════════════════════════════════╗');
+        console.log('║                      TEST SUMMARY                           ║');
+        console.log('╚════════════════════════════════════════════════════════════╝\n');
+
+        const duration = new Date(this.results.endTime) - new Date(this.results.startTime);
+
+        console.log('⏱️  Duration:', Math.round(duration / 1000), 'seconds');
+        
+        // Colorize status
+        const statusIcon = this.results.status === 'success' ? '✅' : '❌';
+        console.log(`${statusIcon} Status:`, this.results.status.toUpperCase());
+        console.log('');
+
+        // Computations
+        console.log('📊 Computations:');
+        console.log(`   Completed: ${this.results.computations.length}`);
+        for (const comp of this.results.computations) {
+            console.log(`   • ${comp.name}: ${comp.resultCount} entities (${comp.duration}ms)`);
+        }
+        console.log('');
+
+        // Cost
+        console.log('💰 Cost Analysis:');
+        console.log(`   Total: $${this.results.cost.totalCostUSD.toFixed(6)}`);
+        console.log(`   Budget: $${this.results.cost.maxCostUSD}`);
+        console.log(`   Utilization: ${this.results.cost.utilizationPct.toFixed(1)}%`);
+        console.log('   By computation:');
+        for (const { name, cost } of this.results.cost.byComputation.slice(0, 5)) {
+            console.log(`     - ${name}: $${cost.toFixed(6)}`);
+        }
+        console.log('');
+
+        // Storage
+        console.log('💾 Storage:');
+        console.log(`   Total writes: ${this.results.storage.totalWrites}`);
+        console.log(`   Total entities: ${this.results.storage.totalEntities}`);
+        console.log('');
+
+        // Lineage
+        console.log('🔗 Lineage:');
+        console.log(`   Total passes: ${Object.keys(this.results.lineage.passSummary).length}`);
+        console.log(`   Execution order verified: ✅`);
+    }
+
+    _saveReport() {
+        const reportPath = path.join(__dirname, `test-report-${this.results.runId}.json`);
+        fs.writeFileSync(reportPath, JSON.stringify(this.results, null, 2));
+        console.log(`\n📄 Full report saved: ${reportPath}`);
+    }
+}
+
+// ============================================================================
+// CLI INTERFACE
+// ============================================================================
+
+async function main() {
+    const args = process.argv.slice(2);
+    const options = {
+        date: null,
+        entities: null,
+        maxCost: 10,
+        testDataset: null,
+        testBucket: null,
+        batchSize: null,
+        concurrency: null
+    };
+
+    // Parse CLI arguments
+    for (let i = 0; i < args.length; i += 2) {
+        const key = args[i].replace(/^--/, '');
+        const value = args[i + 1];
+
+        if (key === 'entities') {
+            options.entities = value.split(',');
+        } else if (key === 'max-cost') {
+            options.maxCost = parseFloat(value);
+        } else if (key === 'batch-size') {
+            options.batchSize = parseInt(value, 10);
+        } else if (key === 'concurrency') {
+            options.concurrency = parseInt(value, 10);
+        } else {
+            options[key] = value;
+        }
+    }
+
+    // Validate required options
+    if (!options.date) {
+        console.error('❌ Missing required option: --date');
+        console.log('\nUsage: node test/run-pipeline-test.js --date YYYY-MM-DD [OPTIONS]\n');
+        console.log('Options:');
+        console.log('  --date           Target date (required)');
+        console.log('  --entities       Comma-separated entity IDs (optional, tests all if omitted)');
+        console.log('  --max-cost       Maximum cost in USD (default: 10)');
+        console.log('  --batch-size     Batch size for data fetching (default: 100)');
+        console.log('  --concurrency    Entity processing concurrency (default: 5)');
+        console.log('  --test-dataset   Override test dataset name');
+        console.log('  --test-bucket    Override test bucket name');
+        process.exit(1);
+    }
+
+    const runner = new PipelineTestRunner(options);
+    await runner.run();
+}
+
+if (require.main === module) {
+    main().catch(e => {
+        console.error('\n💥 Fatal error:', e);
+        process.exit(1);
+    });
+}
+
+module.exports = { PipelineTestRunner, TestConfigBuilder };