bulltrackers-module 1.0.765 → 1.0.768

package/functions/computation-system-v2/test/test-full-pipeline.js

@@ -1,227 +0,0 @@
-/**
- * @fileoverview Full Pipeline Integration Test
- * * Simulates the entire flow from Dispatcher -> Orchestrator -> Worker Pool -> Storage.
- * * Verifies that:
- * 1. The Dispatcher BLOCKS computations with missing mandatory data (RunAnalyzer).
- * 2. Runnable computations are sent to the Worker Pool (if configured).
- * 3. Results are stored in the TEST table (not production).
- * * * USAGE:
- * node test/test-full-pipeline.js --date 2026-01-24
- */
-
-process.env.NODE_ENV = 'test';
-process.env.WORKER_LOCAL_MODE = 'true'; // Simulate workers locally
-process.env.WORKER_POOL_ENABLED = 'true';
-
-const fs = require('fs');
-const path = require('path');
-const { Orchestrator } = require('../framework/execution/Orchestrator');
-const { TestConfigBuilder } = require('./run-pipeline-test'); 
-const prodConfig = require('../config/bulltrackers.config');
-
-// ============================================================================
-// TEST RUNNER
-// ============================================================================
-
-async function runPipelineTest() {
-    const args = parseArgs();
-    console.log('\n╔════════════════════════════════════════════════════════════╗');
-    console.log('║           FULL PIPELINE INTEGRATION TEST                   ║');
-    console.log('╚════════════════════════════════════════════════════════════╝');
-    console.log(`📅 Target Date: ${args.date}`);
-    console.log(`🧪 Test Table:  computation_results_test`);
-    console.log(`👷 Worker Pool: ENABLED (Local Simulation)\n`);
-
-    // 0. DYNAMICALLY LOAD COMPUTATIONS
-    // This fixes the "Initialized with 0 computations" error
-    const computationsDir = path.join(__dirname, '../computations');
-    const loadedComputations = fs.readdirSync(computationsDir)
-        .filter(f => f.endsWith('.js'))
-        .map(f => require(path.join(computationsDir, f)));
-    
-    // Inject into config
-    prodConfig.computations = loadedComputations;
-    console.log(`📦 Auto-discovered ${prodConfig.computations.length} computations from /computations directory`);
-
-    // 1. CONFIGURE TEST ENVIRONMENT
-    const builder = new TestConfigBuilder(prodConfig, {
-        runId: `test-${Date.now()}`,
-        date: args.date,
-        testBucket: 'bulltrackers-computation-staging', // <--- FIX: Use your real bucket
-        batchSize: 1000,
-        concurrency: 2
-    });
-    
-    const testConfig = builder.build();
-    
-    // Explicitly enable worker pool in the test config
-    testConfig.workerPool = {
-        ...prodConfig.workerPool,
-        enabled: true,
-        localMode: true,
-        minEntitiesForOffload: 0 // Force everything to worker pool for testing
-    };
-
-    // 2. INITIALIZE ORCHESTRATOR
-    const orchestrator = new Orchestrator(testConfig, console);
-    await orchestrator.initialize();
-
-    // -------------------------------------------------------------------------
-    // PHASE 1: DISPATCHER VERIFICATION (Pre-Flight Check)
-    // -------------------------------------------------------------------------
-    console.log('🔍 PHASE 1: DISPATCHER ANALYSIS (The Gatekeeper)');
-    console.log('   Verifying that missing data BLOCKS execution...');
-    
-    const analysis = await orchestrator.analyze({ date: args.date });
-    
-    printAnalysisTable(analysis);
-
-    // Validation: Ensure nothing "Impossible" or "Blocked" is in the runnable list
-    const badRunnables = analysis.runnable.filter(r => 
-        analysis.blocked.find(b => b.name === r.name) || 
-        analysis.impossible.find(i => i.name === r.name)
-    );
-
-    if (badRunnables.length > 0) {
-        console.error('❌ CRITICAL FAILURE: Dispatcher marked blocked tasks as runnable!');
-        process.exit(1);
-    }
-    console.log('✅ Dispatcher logic validated. Blocked tasks will NOT run.\n');
-
-    // -------------------------------------------------------------------------
-    // PHASE 2: EXECUTION (Worker Pool & Storage)
-    // -------------------------------------------------------------------------
-    console.log('🚀 PHASE 2: PIPELINE EXECUTION');
-    console.log('   Running only valid tasks via Worker Pool...');
-
-    // We intercept storage to verify writes without polluting real DB (optional if using test table)
-    const storageInterceptor = new TestStorageInterceptor(orchestrator.storageManager);
-    orchestrator.storageManager = storageInterceptor;
-
-    // Run!
-    const result = await orchestrator.execute({ 
-        date: args.date,
-        dryRun: false // We want to test the full "write" path to the test table
-    });
-
-    // -------------------------------------------------------------------------
-    // PHASE 3: VERIFICATION & REPORTING
-    // -------------------------------------------------------------------------
-    console.log('\n📊 PHASE 3: FINAL REPORT');
-    
-    // 1. Did Blocked Tasks Run?
-    const blockedRan = result.completed.filter(c => 
-        analysis.blocked.find(b => b.name === c.name)
-    );
-    
-    if (blockedRan.length > 0) {
-        console.error(`❌ FAILURE: The following BLOCKED tasks executed anyway: ${blockedRan.map(c => c.name).join(', ')}`);
-    } else {
-        console.log('✅ SUCCESS: No blocked tasks were executed.');
-    }
-
-    // 2. Did Runnable Tasks Succeed?
-    const runnableNames = analysis.runnable.map(r => r.name);
-    const successfulRunnables = result.completed.filter(c => runnableNames.includes(c.name));
-    
-    if (successfulRunnables.length > 0) {
-        console.log(`✅ SUCCESS: ${successfulRunnables.length} runnable tasks completed successfully.`);
-    } else if (runnableNames.length > 0) {
-        console.warn('⚠️ WARNING: Runnable tasks existed but none completed (check errors below).');
-    } else {
-        console.log('ℹ️  No runnable tasks found (this is expected if data is missing).');
-    }
-
-    // 3. Storage Verification
-    const writes = storageInterceptor.getSummary();
-    console.log(`💾 Storage: Written ${writes.totalEntities} entity results to ${testConfig.resultStore.table}`);
-
-    if (result.summary.errors > 0) {
-        console.log('\n❌ EXECUTION ERRORS:');
-        result.errors.forEach(e => console.log(`   - ${e.name}: ${e.error}`));
-    }
-}
-
-// ============================================================================
-// HELPERS
-// ============================================================================
-
-function printAnalysisTable(analysis) {
-    console.log('\n   ┌──────────────────────────────┬──────────────┬──────────────────────────────────────────┐');
-    console.log('   │ Computation                  │ Status       │ Reason                                   │');
-    console.log('   ├──────────────────────────────┼──────────────┼──────────────────────────────────────────┤');
-    
-    const all = [
-        ...analysis.runnable.map(r => ({ ...r, status: 'RUNNABLE' })),
-        ...analysis.blocked.map(r => ({ ...r, status: 'BLOCKED' })),
-        ...analysis.impossible.map(r => ({ ...r, status: 'IMPOSSIBLE' })),
-        ...analysis.reRuns.map(r => ({ ...r, status: 'RERUN' })),
-        ...analysis.skipped.map(r => ({ ...r, status: 'SKIPPED' }))
-    ];
-
-    all.forEach(row => {
-        const name = row.name.padEnd(28).slice(0, 28);
-        const status = row.status.padEnd(12);
-        const reason = (row.reason || 'Ready to run').padEnd(40).slice(0, 40);
-        
-        let color = '\x1b[37m'; // White
-        if (row.status === 'BLOCKED') color = '\x1b[31m'; // Red
-        if (row.status === 'RUNNABLE') color = '\x1b[32m'; // Green
-        
-        console.log(`   │ ${color}${name}\x1b[0m │ ${color}${status}\x1b[0m │ ${reason} │`);
-    });
-    console.log('   └──────────────────────────────┴──────────────┴──────────────────────────────────────────┘\n');
-}
-
-/**
- * Simple Storage Interceptor to verify writes
- */
-class TestStorageInterceptor {
-    constructor(realStorage) {
-        this.realStorage = realStorage;
-        this.writes = [];
-        
-        // Proxy methods
-        return new Proxy(this, {
-            get(target, prop) {
-                if (prop in target) return target[prop];
-                if (typeof target.realStorage[prop] === 'function') {
-                    return target.realStorage[prop].bind(target.realStorage);
-                }
-                return target.realStorage[prop];
-            }
-        });
-    }
-
-    async commitResults(date, entry, results, depHashes) {
-        this.writes.push({
-            date,
-            computation: entry.name,
-            count: Object.keys(results).length
-        });
-        // Pass through to real storage (which is pointing to test table)
-        return this.realStorage.commitResults(date, entry, results, depHashes);
-    }
-
-    getSummary() {
-        return {
-            totalWrites: this.writes.length,
-            totalEntities: this.writes.reduce((sum, w) => sum + w.count, 0),
-            details: this.writes
-        };
-    }
-}
-
-function parseArgs() {
-    const args = process.argv.slice(2);
-    let date = new Date();
-    date.setDate(date.getDate() - 1); // Default to yesterday
-    
-    for (let i = 0; i < args.length; i++) {
-        if (args[i] === '--date') date = new Date(args[++i]);
-    }
-    
-    return { date: date.toISOString().slice(0, 10) };
-}
-
-runPipelineTest().catch(console.error);

package/functions/computation-system-v2/test/test-worker-pool.js

@@ -1,266 +0,0 @@
-/**
- * @fileoverview Worker Pool Integration Test (Real Data)
- * * Tests the Worker Pool pipeline using REAL BigQuery data.
- * * WHAT IT DOES:
- * 1. Connects to BigQuery using your local credentials.
- * 2. Fetches REAL data for the specified date/computation.
- * 3. Simulates the Orchestrator's data packaging.
- * 4. Runs the worker logic locally (via RemoteTaskRunner localMode).
- * * USAGE:
- * node test/test-worker-pool.js --date 2026-01-24 --computation UserPortfolioSummary
- * * FLAGS:
- * --date YYYY-MM-DD       (Required) Target date
- * --computation Name      (Default: UserPortfolioSummary)
- * --limit N               (Default: 5) Number of entities to test
- * --entities id1,id2      (Optional) Specific entities to test
- */
-
-const path = require('path');
-const { RemoteTaskRunner } = require('../framework/execution/RemoteTaskRunner');
-const { SchemaRegistry } = require('../framework/data/SchemaRegistry');
-const { QueryBuilder } = require('../framework/data/QueryBuilder');
-const { DataFetcher } = require('../framework/data/DataFetcher');
-const config = require('../config/bulltrackers.config');
-
-// Force Local Mode for the Worker Pool (runs logic in-process but uses the Runner's pipeline)
-process.env.WORKER_LOCAL_MODE = 'true';
-process.env.WORKER_POOL_ENABLED = 'true';
-
-// ============================================================================
-// HELPER: Mini Orchestrator (Data Loading)
-// ============================================================================
-
-class TestContext {
-    constructor() {
-        this.logger = console;
-        
-        // Initialize Framework Data Layer
-        this.schemaRegistry = new SchemaRegistry(config.bigquery, this.logger);
-        this.queryBuilder = new QueryBuilder(config.bigquery, this.schemaRegistry, this.logger);
-        this.dataFetcher = new DataFetcher({ ...config.bigquery, tables: config.tables }, this.queryBuilder, this.logger);
-    }
-
-    async initialize() {
-        console.log('🔌 Connecting to BigQuery and initializing Schema Registry...');
-        // We only warm cache for tables we'll likely need to save time
-        // In a real run, we might scan the computation's requirements
-    }
-}
-
-// ============================================================================
-// MAIN RUNNER
-// ============================================================================
-
-async function runRealDataTest() {
-    const args = parseArgs();
-    console.log('\n╔════════════════════════════════════════════════════════════╗');
-    console.log('║       WORKER POOL INTEGRATION TEST (REAL DATA)            ║');
-    console.log('╚════════════════════════════════════════════════════════════╝');
-    console.log(`📅 Date:        ${args.date}`);
-    console.log(`🧮 Computation: ${args.computation}`);
-    console.log(`📉 Entity Limit:${args.limit}`);
-    console.log(`🔧 Local Mode:  ENABLED\n`);
-
-    const ctx = new TestContext();
-    await ctx.initialize();
-
-    // 1. Load the Computation Class
-    const computationPath = path.join(__dirname, `../computations/${args.computation}`);
-    let ComputationClass;
-    try {
-        ComputationClass = require(computationPath);
-    } catch (e) {
-        throw new Error(`Could not load computation at ${computationPath}: ${e.message}`);
-    }
-
-    const compConfig = ComputationClass.getConfig();
-    const manifestEntry = {
-        name: compConfig.name.toLowerCase(),
-        originalName: compConfig.name,
-        type: compConfig.type,
-        requires: compConfig.requires || {},
-        dependencies: compConfig.dependencies || [],
-        hash: 'test-hash-123'
-    };
-
-    console.log(`📦 Loaded ${manifestEntry.originalName}`);
-    console.log(`   Requires: ${Object.keys(manifestEntry.requires).join(', ')}`);
-
-    // 2. Determine Driver Table (for batching)
-    const driverTable = getDriverTable(manifestEntry.requires, config);
-    if (!driverTable) {
-        throw new Error(`Could not determine driver table (entityField) for ${args.computation}. Is it a global computation?`);
-    }
-    const driverEntityField = config.tables[driverTable].entityField;
-    console.log(`   Driver Table: ${driverTable} (${driverEntityField})`);
-
-    // 3. Fetch Data (The "Heavy Lifting" usually done by Orchestrator)
-    console.log('\n📥 Fetching REAL data from BigQuery...');
-    
-    // Split requirements
-    const { batchRequires, globalRequires } = splitRequirements(manifestEntry.requires, driverTable, config);
-
-    // Fetch Global Data (Shared)
-    let globalData = {};
-    if (Object.keys(globalRequires).length > 0) {
-        console.log(`   Fetching global requirements: ${Object.keys(globalRequires).join(', ')}...`);
-        globalData = await ctx.dataFetcher.fetchForComputation(globalRequires, args.date);
-    }
-
-    // Fetch Batch Data (Per-Entity)
-    console.log(`   Fetching batch requirements: ${Object.keys(batchRequires).join(', ')}...`);
-    
-    // Create a specific entity filter if requested
-    if (args.entities.length > 0) {
-        // Inject where clause for specific entities
-        Object.values(batchRequires).forEach(req => {
-            req.where = req.where || {};
-            req.where[driverEntityField] = args.entities; // This relies on DataFetcher supporting array-based WHERE
-        });
-    }
-
-    // We fetch one large batch for the test
-    const batchStream = ctx.dataFetcher.fetchComputationBatched(batchRequires, args.date, args.limit);
-    
-    // Get the first batch
-    const { value: batch, done } = await batchStream.next();
-    
-    if (done || !batch || batch.entityIds.length === 0) {
-        console.warn('⚠️ No data found for this date/computation.');
-        return;
-    }
-
-    console.log(`✅ Data fetched. Processing ${batch.entityIds.length} entities: ${batch.entityIds.join(', ')}`);
-
-    // 4. Initialize RemoteTaskRunner
-    // We configured WORKER_LOCAL_MODE=true, so this will run logic in-process
-    // but crucially, it will go through the _buildContextPackage logic.
-    const runner = new RemoteTaskRunner({
-        workerPool: {
-            enabled: true,
-            localMode: true, // Run in-process
-            concurrency: 5
-        }
-    }, console);
-
-    // Prepare Context
-    const baseContext = {
-        references: {}, // TODO: Load ref data if needed
-        config: {
-            project: config.bigquery.projectId,
-            tables: config.tables
-        }
-    };
-
-    // Prepare Data Maps
-    const entityDataMap = new Map();
-    const combinedData = { ...batch.data, ...globalData };
-
-    for (const entityId of batch.entityIds) {
-        const entityData = filterDataForEntity(combinedData, entityId, driverEntityField, config);
-        entityDataMap.set(entityId, entityData);
-    }
-
-    // 5. Execute via Worker Pool Logic
-    console.log('\n🚀 Executing via RemoteTaskRunner (Local Simulation)...');
-    
-    const startTime = Date.now();
-    
-    const { results, errors } = await runner.runBatch(
-        manifestEntry,
-        args.date,
-        baseContext,
-        batch.entityIds,
-        entityDataMap,
-        {} // Mock dependencies for now
-    );
-
-    const duration = Date.now() - startTime;
-
-    // 6. Report Results
-    console.log('\n╔════════════════════════════════════════════════════════════╗');
-    console.log('║                     EXECUTION REPORT                       ║');
-    console.log('╚════════════════════════════════════════════════════════════╝');
-    console.log(`⏱️  Duration: ${duration}ms`);
-    console.log(`✅ Success:  ${Object.keys(results).length}`);
-    console.log(`❌ Errors:   ${errors.length}`);
-
-    if (Object.keys(results).length > 0) {
-        console.log('\n🔍 Sample Result (First Entity):');
-        const sampleId = Object.keys(results)[0];
-        console.log(`   Entity: ${sampleId}`);
-        console.log(JSON.stringify(results[sampleId], null, 2));
-    }
-
-    if (errors.length > 0) {
-        console.log('\n❌ Errors:');
-        errors.forEach(e => console.log(`   [${e.entityId}] ${e.error}`));
-    }
-}
-
-// ============================================================================
-// UTILS (Duplicated from Orchestrator logic)
-// ============================================================================
-
-function getDriverTable(requires, config) {
-    for (const name of Object.keys(requires)) {
-        const conf = config.tables[name];
-        if (conf && conf.entityField) return name;
-    }
-    return null; 
-}
-
-function splitRequirements(requires, driverTable, config) {
-    const batchRequires = {};
-    const globalRequires = {};
-    const driverConfig = config.tables[driverTable];
-    const driverEntityField = driverConfig ? driverConfig.entityField : null;
-    
-    for (const [name, spec] of Object.entries(requires)) {
-        const conf = config.tables[name];
-        if (conf && conf.entityField === driverEntityField) {
-            batchRequires[name] = spec;
-        } else {
-            globalRequires[name] = spec;
-        }
-    }
-    return { batchRequires, globalRequires };
-}
-
-function filterDataForEntity(data, id, driverEntityField, config) {
-    const out = {};
-    Object.entries(data).forEach(([tbl, d]) => {
-        const conf = config.tables[tbl] || {};
-        if (conf.entityField === driverEntityField && d && !Array.isArray(d)) {
-             out[tbl] = d[id] || null;
-        } else {
-             out[tbl] = d; 
-        }
-    });
-    return out;
-}
-
-function parseArgs() {
-    const args = process.argv.slice(2);
-    const options = {
-        date: new Date().toISOString().slice(0, 10),
-        computation: 'UserPortfolioSummary',
-        limit: 5,
-        entities: []
-    };
-
-    for (let i = 0; i < args.length; i++) {
-        const arg = args[i];
-        if (arg === '--date') options.date = args[++i];
-        if (arg === '--computation') options.computation = args[++i];
-        if (arg === '--limit') options.limit = parseInt(args[++i], 10);
-        if (arg === '--entities') options.entities = args[++i].split(',');
-    }
-    return options;
-}
-
-// Run
-runRealDataTest().catch(e => {
-    console.error('\n💥 Fatal Error:', e);
-    process.exit(1);
-});