bulltrackers-module 1.0.765 → 1.0.768

package/functions/computation-system-v2/test/run-pipeline-test.js

@@ -1,554 +0,0 @@
-/**
- * @fileoverview Production Pipeline Test Runner
- * * Tests the ENTIRE production pipeline with:
- * - Cost tracking and hard limits
- * - Test tables/buckets (isolated from production)
- * - Entity filtering (test subset of users)
- * - Lineage tracking
- * - Performance profiling
- * * Run: node test/run-pipeline-test.js --date 2026-01-24 --entities user1,user2 --max-cost 5 --batch-size 1000
- * * * UPDATE: Sets NODE_ENV='test' and force:true to bypass Checkpointer locks AND "Up-To-Date" checks.
- */
-
-// FORCE TEST ENVIRONMENT
-process.env.NODE_ENV = 'test';
-
-const path = require('path');
-const fs = require('fs');
-const { Orchestrator } = require('../framework/execution/Orchestrator');
-const { StorageManager } = require('../framework/storage/StorageManager');
-const { CostTracker } = require('../framework/cost/CostTracker');
-const { ComputationProfiler } = require('../framework/monitoring/Profiler');
-
-// ============================================================================
-// TEST CONFIGURATION BUILDER
-// ============================================================================
-
-class TestConfigBuilder {
-    constructor(productionConfig, testOptions) {
-        this.prodConfig = productionConfig;
-        this.testOptions = testOptions;
-    }
-
-    build() {
-        // Clone production config
-        const testConfig = JSON.parse(JSON.stringify(this.prodConfig));
-
-        // Restore non-serializable objects (Classes and Functions)
-        testConfig.computations = this.prodConfig.computations;
-        testConfig.rules = this.prodConfig.rules;
-        
-        // Override GCS to use test bucket
-        const prodBucket = testConfig.gcs?.bucket;
-        testConfig.gcs = {
-            bucket: this.testOptions.testBucket || `${prodBucket}-test`,
-            prefix: 'test-staging'
-        };
-
-        // Override result storage to use test table
-        testConfig.resultStore = {
-            table: 'test_computation_results',
-            partitionField: 'date',
-            clusterFields: ['computation_name', 'category']
-        };
-
-        // Add test-specific metadata
-        testConfig.testMode = {
-            enabled: true,
-            runId: this.testOptions.runId || `test-${Date.now()}`,
-            targetEntities: this.testOptions.entities || null,
-            maxCostUSD: this.testOptions.maxCost || 10,
-            dateOverride: this.testOptions.date
-        };
-
-        // Override execution settings for testing
-        testConfig.execution = {
-            ...testConfig.execution,
-            entityConcurrency: this.testOptions.concurrency || 5, // Default 5 or user provided
-            batchSize: this.testOptions.batchSize || 100 // Default 100 or user provided
-        };
-
-        // FIX: Explicitly enable lock bypass in config
-        testConfig.bypassLocks = true;
-
-        return testConfig;
-    }
-}
-
-// ============================================================================
-// RUNTIME INTERCEPTORS (Proxy Pattern)
-// ============================================================================
-
-/**
- * Intercepts StorageManager to redirect writes to test tables
- */
-class TestStorageInterceptor {
-    constructor(storageManager, testConfig) {
-        this.storage = storageManager;
-        this.testConfig = testConfig;
-        this.capturedWrites = [];
-    }
-
-    // Wrap commitResults to capture what would be written
-    async commitResults(date, entry, results, depHashes) {
-        this.capturedWrites.push({
-            date,
-            computation: entry.name,
-            entityCount: Object.keys(results).length,
-            timestamp: new Date().toISOString()
-        });
-
-        // Actually write to test storage
-        return this.storage.commitResults(date, entry, results, depHashes);
-    }
-
-    async finalizeResults(date, entry) {
-        return this.storage.finalizeResults(date, entry);
-    }
-
-    // Proxy other methods
-    async initCheckpoint(...args) { return this.storage.initCheckpoint(...args); }
-    async updateCheckpoint(...args) { return this.storage.updateCheckpoint(...args); }
-    async completeCheckpoint(...args) { return this.storage.completeCheckpoint(...args); }
-    async getLatestCheckpoint(...args) { return this.storage.getLatestCheckpoint(...args); }
-    async savePerformanceReport(...args) { return this.storage.savePerformanceReport(...args); }
-
-    getSummary() {
-        return {
-            totalWrites: this.capturedWrites.length,
-            totalEntities: this.capturedWrites.reduce((sum, w) => sum + w.entityCount, 0),
-            byComputation: this.capturedWrites.reduce((acc, w) => {
-                acc[w.computation] = (acc[w.computation] || 0) + w.entityCount;
-                return acc;
-            }, {})
-        };
-    }
-}
-
-/**
- * Intercepts CostTracker to enforce hard cost limits
- */
-class TestCostEnforcer {
-    constructor(costTracker, maxCostUSD) {
-        this.tracker = costTracker;
-        this.maxCostUSD = maxCostUSD;
-        this.currentCostUSD = 0;
-        this.costByComputation = {};
-    }
-
-    async trackCost(computationName, date, bytesProcessed) {
-        // Calculate cost (same as production)
-        const COST_PER_TB = 5.0;
-        const BYTES_PER_TB = 1099511627776;
-        const cost = (bytesProcessed / BYTES_PER_TB) * COST_PER_TB;
-
-        this.currentCostUSD += cost;
-        this.costByComputation[computationName] = (this.costByComputation[computationName] || 0) + cost;
-
-        // HARD LIMIT ENFORCEMENT
-        if (this.currentCostUSD > this.maxCostUSD) {
-            throw new Error(
-                `🚨 COST LIMIT EXCEEDED: $${this.currentCostUSD.toFixed(4)} > $${this.maxCostUSD}. ` +
-                `Last computation: ${computationName}. Test aborted.`
-            );
-        }
-
-        // Still track in production tracker (to test table)
-        await this.tracker.trackCost(computationName, date, bytesProcessed);
-
-        return cost;
-    }
-
-    getSummary() {
-        return {
-            totalCostUSD: this.currentCostUSD,
-            maxCostUSD: this.maxCostUSD,
-            utilizationPct: (this.currentCostUSD / this.maxCostUSD) * 100,
-            byComputation: Object.entries(this.costByComputation)
-                .map(([name, cost]) => ({ name, cost }))
-                .sort((a, b) => b.cost - a.cost)
-        };
-    }
-}
-
-// ============================================================================
-// LINEAGE TRACKER
-// ============================================================================
-
-class TestLineageTracker {
-    constructor(manifest) {
-        this.manifest = manifest;
-        this.executionOrder = [];
-        this.dependencyGraph = new Map();
-    }
-
-    recordExecution(computationName, pass, status, duration) {
-        this.executionOrder.push({
-            name: computationName,
-            pass,
-            status,
-            duration,
-            timestamp: new Date().toISOString()
-        });
-    }
-
-    buildGraph() {
-        for (const entry of this.manifest) {
-            this.dependencyGraph.set(entry.name, {
-                name: entry.originalName,
-                pass: entry.pass,
-                dependencies: entry.dependencies,
-                category: entry.category,
-                type: entry.type
-            });
-        }
-    }
-
-    generateReport() {
-        this.buildGraph();
-
-        const report = {
-            totalComputations: this.manifest.length,
-            executionOrder: this.executionOrder,
-            dependencyGraph: Array.from(this.dependencyGraph.entries()).map(([name, data]) => ({
-                computation: data.name,
-                pass: data.pass,
-                dependencies: data.dependencies.map(dep => {
-                    const depData = this.dependencyGraph.get(dep);
-                    return depData ? depData.name : dep;
-                }),
-                category: data.category,
-                type: data.type
-            })),
-            passSummary: this._summarizeByPass()
-        };
-
-        return report;
-    }
-
-    _summarizeByPass() {
-        const byPass = {};
-        for (const entry of this.manifest) {
-            if (!byPass[entry.pass]) byPass[entry.pass] = [];
-            byPass[entry.pass].push(entry.originalName);
-        }
-        return byPass;
-    }
-
-    printGraph() {
-        console.log('\n📊 COMPUTATION DEPENDENCY GRAPH\n');
-        
-        const byPass = this._summarizeByPass();
-        const passNumbers = Object.keys(byPass).map(Number).sort((a, b) => a - b);
-
-        for (const pass of passNumbers) {
-            console.log(`Pass ${pass}:`);
-            for (const compName of byPass[pass]) {
-                const entry = this.manifest.find(e => e.originalName === compName);
-                const deps = entry.dependencies.length > 0 
-                    ? ` (depends on: ${entry.dependencies.join(', ')})`
-                    : '';
-                console.log(`  • ${compName}${deps}`);
-            }
-            console.log('');
-        }
-    }
-}
-
-// ============================================================================
-// TEST RUNNER
-// ============================================================================
-
-class PipelineTestRunner {
-    constructor(options) {
-        this.options = options;
-        this.results = {
-            runId: null,
-            startTime: null,
-            endTime: null,
-            status: 'pending',
-            computations: [],
-            cost: null,
-            storage: null,
-            lineage: null,
-            performance: []
-        };
-    }
-
-    async run() {
-        console.log('╔════════════════════════════════════════════════════════════╗');
-        console.log('║         PRODUCTION PIPELINE FULL INTEGRATION TEST          ║');
-        console.log('╚════════════════════════════════════════════════════════════╝\n');
-
-        this.results.startTime = new Date().toISOString();
-        this.results.runId = `test-${Date.now()}`;
-
-        try {
-            // 1. Load production config
-            const prodConfig = this._loadProductionConfig();
-            
-            // 2. Build test config
-            const testOptions = {
-                runId: this.results.runId,
-                date: this.options.date,
-                entities: this.options.entities,
-                maxCost: this.options.maxCost || 10,
-                testDataset: this.options.testDataset,
-                testBucket: this.options.testBucket,
-                batchSize: this.options.batchSize,
-                concurrency: this.options.concurrency
-            };
-
-            const builder = new TestConfigBuilder(prodConfig, testOptions);
-            const testConfig = builder.build();
-
-            console.log('🔧 Test Configuration:');
-            console.log(`   Run ID: ${testConfig.testMode.runId}`);
-            console.log(`   Date: ${testConfig.testMode.dateOverride}`);
-            console.log(`   Entities: ${testConfig.testMode.targetEntities || 'ALL'}`);
-            console.log(`   Max Cost: $${testConfig.testMode.maxCostUSD}`);
-            console.log(`   Test Dataset: ${testConfig.bigquery.dataset}`);
-            console.log(`   Test Bucket: ${testConfig.gcs.bucket}`);
-            console.log(`   Batch Size: ${testConfig.execution.batchSize}`);
-            console.log(`   Concurrency: ${testConfig.execution.entityConcurrency}`);
-            console.log(`   Bypass Locks: ${testConfig.bypassLocks ? 'YES' : 'NO'}\n`);
-
-            // 3. Initialize orchestrator (PRODUCTION CODE)
-            const orchestrator = new Orchestrator(testConfig, console);
-
-            // PATCH: Monkey-patch _executeComputation to debug silent errors
-            const originalExecuteComp = orchestrator._executeComputation;
-            orchestrator._executeComputation = async function(entry, ...args) {
-                try {
-                    return await originalExecuteComp.call(this, entry, ...args);
-                } catch (e) {
-                    console.error(JSON.stringify(e, null, 2));
-                    if (e.stack) console.error(e.stack);
-                    throw e; // Re-throw to be caught by the standard handler
-                }
-            };
-
-            await orchestrator.initialize();
-
-            console.log(`📦 Loaded ${orchestrator.manifest.length} computations\n`);
-
-            // 4. Setup interceptors
-            const storageInterceptor = new TestStorageInterceptor(
-                orchestrator.storageManager,
-                testConfig
-            );
-            const costEnforcer = new TestCostEnforcer(
-                orchestrator.storageManager.costTracker || { trackCost: async () => {} },
-                testConfig.testMode.maxCostUSD
-            );
-            const lineageTracker = new TestLineageTracker(orchestrator.manifest);
-
-            // Inject interceptors via proxy
-            // FIX: Use .bind() to ensure methods don't lose 'this' context
-            orchestrator.storageManager = new Proxy(storageInterceptor, {
-                get: (target, prop) => {
-                    // 1. Check interceptor first
-                    if (prop in target) return target[prop];
-                    
-                    // 2. Fallback to original manager
-                    const originalManager = target.storage;
-                    const value = originalManager[prop];
-                    
-                    // 3. CRITICAL: Bind functions to original manager to preserve 'this'
-                    if (typeof value === 'function') {
-                        return value.bind(originalManager);
-                    }
-                    return value;
-                }
-            });
-
-            // Print dependency graph
-            lineageTracker.printGraph();
-
-            // 5. Execute pipeline (PRODUCTION CODE)
-            console.log('🚀 Starting pipeline execution...\n');
-
-            const execResult = await orchestrator.execute({
-                date: testConfig.testMode.dateOverride,
-                entities: testConfig.testMode.targetEntities,
-                dryRun: false, // Actually write to test tables
-                force: true // FIX: FORCE LOCK BYPASS & FORCE RE-RUN OF VALID COMPUTATIONS
-            });
-
-            // 6. Collect results
-            this.results.computations = execResult.completed.map(c => ({
-                name: c.name,
-                status: c.status,
-                resultCount: c.resultCount,
-                duration: c.duration
-            }));
-
-            // FIX: Correctly report failure if errors occurred
-            if (execResult.summary.errors > 0 || (execResult.errors && execResult.errors.length > 0)) {
-                 this.results.status = 'failed';
-                 this.results.errors = execResult.errors;
-                 console.error(`\n❌ Pipeline finished with ${execResult.summary.errors} errors.`);
-            } else if (execResult.summary.completed === 0 && orchestrator.manifest.length > 0) {
-                 this.results.status = 'failed';
-                 console.error('\n❌ Pipeline finished but 0 computations completed.');
-            } else {
-                 this.results.status = 'success';
-            }
-
-            this.results.cost = costEnforcer.getSummary();
-            this.results.storage = storageInterceptor.getSummary();
-            this.results.lineage = lineageTracker.generateReport();
-            this.results.endTime = new Date().toISOString();
-
-            // 7. Print summary
-            this._printSummary();
-
-            // 8. Save report
-            this._saveReport();
-
-            return this.results;
-
-        } catch (error) {
-            this.results.status = 'failed';
-            this.results.error = error.message;
-            this.results.endTime = new Date().toISOString();
-
-            console.error('\n❌ TEST FAILED:', error); // Log full error object
-            if (error.stack) console.error(error.stack);
-
-            this._saveReport();
-            throw error;
-        }
-    }
-
-    _loadProductionConfig() {
-        // Load real production config
-        const prodConfig = require('../config/bulltrackers.config');
-
-        // Dynamically load ALL computations from directory
-        const computationsDir = path.join(__dirname, '../computations');
-        const files = fs.readdirSync(computationsDir).filter(f => f.endsWith('.js'));
-
-        prodConfig.computations = files.map(f => {
-            const ComputationClass = require(path.join(computationsDir, f));
-            return ComputationClass;
-        });
-
-        console.log(`✅ Loaded ${prodConfig.computations.length} computations from directory\n`);
-
-        return prodConfig;
-    }
-
-    _printSummary() {
-        console.log('\n╔════════════════════════════════════════════════════════════╗');
-        console.log('║                      TEST SUMMARY                           ║');
-        console.log('╚════════════════════════════════════════════════════════════╝\n');
-
-        const duration = new Date(this.results.endTime) - new Date(this.results.startTime);
-
-        console.log('⏱️  Duration:', Math.round(duration / 1000), 'seconds');
-        
-        // Colorize status
-        const statusIcon = this.results.status === 'success' ? '✅' : '❌';
-        console.log(`${statusIcon} Status:`, this.results.status.toUpperCase());
-        console.log('');
-
-        // Computations
-        console.log('📊 Computations:');
-        console.log(`   Completed: ${this.results.computations.length}`);
-        for (const comp of this.results.computations) {
-            console.log(`   • ${comp.name}: ${comp.resultCount} entities (${comp.duration}ms)`);
-        }
-        console.log('');
-
-        // Cost
-        console.log('💰 Cost Analysis:');
-        console.log(`   Total: $${this.results.cost.totalCostUSD.toFixed(6)}`);
-        console.log(`   Budget: $${this.results.cost.maxCostUSD}`);
-        console.log(`   Utilization: ${this.results.cost.utilizationPct.toFixed(1)}%`);
-        console.log('   By computation:');
-        for (const { name, cost } of this.results.cost.byComputation.slice(0, 5)) {
-            console.log(`     - ${name}: $${cost.toFixed(6)}`);
-        }
-        console.log('');
-
-        // Storage
-        console.log('💾 Storage:');
-        console.log(`   Total writes: ${this.results.storage.totalWrites}`);
-        console.log(`   Total entities: ${this.results.storage.totalEntities}`);
-        console.log('');
-
-        // Lineage
-        console.log('🔗 Lineage:');
-        console.log(`   Total passes: ${Object.keys(this.results.lineage.passSummary).length}`);
-        console.log(`   Execution order verified: ✅`);
-    }
-
-    _saveReport() {
-        const reportPath = path.join(__dirname, `test-report-${this.results.runId}.json`);
-        fs.writeFileSync(reportPath, JSON.stringify(this.results, null, 2));
-        console.log(`\n📄 Full report saved: ${reportPath}`);
-    }
-}
-
-// ============================================================================
-// CLI INTERFACE
-// ============================================================================
-
-async function main() {
-    const args = process.argv.slice(2);
-    const options = {
-        date: null,
-        entities: null,
-        maxCost: 10,
-        testDataset: null,
-        testBucket: null,
-        batchSize: null,
-        concurrency: null
-    };
-
-    // Parse CLI arguments
-    for (let i = 0; i < args.length; i += 2) {
-        const key = args[i].replace(/^--/, '');
-        const value = args[i + 1];
-
-        if (key === 'entities') {
-            options.entities = value.split(',');
-        } else if (key === 'max-cost') {
-            options.maxCost = parseFloat(value);
-        } else if (key === 'batch-size') {
-            options.batchSize = parseInt(value, 10);
-        } else if (key === 'concurrency') {
-            options.concurrency = parseInt(value, 10);
-        } else {
-            options[key] = value;
-        }
-    }
-
-    // Validate required options
-    if (!options.date) {
-        console.error('❌ Missing required option: --date');
-        console.log('\nUsage: node test/run-pipeline-test.js --date YYYY-MM-DD [OPTIONS]\n');
-        console.log('Options:');
-        console.log('  --date           Target date (required)');
-        console.log('  --entities       Comma-separated entity IDs (optional, tests all if omitted)');
-        console.log('  --max-cost       Maximum cost in USD (default: 10)');
-        console.log('  --batch-size     Batch size for data fetching (default: 100)');
-        console.log('  --concurrency    Entity processing concurrency (default: 5)');
-        console.log('  --test-dataset   Override test dataset name');
-        console.log('  --test-bucket    Override test bucket name');
-        process.exit(1);
-    }
-
-    const runner = new PipelineTestRunner(options);
-    await runner.run();
-}
-
-if (require.main === module) {
-    main().catch(e => {
-        console.error('\n💥 Fatal error:', e);
-        process.exit(1);
-    });
-}
-
-module.exports = { PipelineTestRunner, TestConfigBuilder };