bulltrackers-module 1.0.768 → 1.0.769

package/functions/computation-system-v2/devtools/simulation/SimulationEngine.js

@@ -0,0 +1,525 @@
+/**
+ * @fileoverview Simulation Engine
+ * * The core of the local simulation platform.
+ * Wraps the actual Orchestrator and injects mock data/storage layers.
+ * * Key Features:
+ * - 1:1 simulation of DAG execution
+ * - Multi-pass support for dependency chains
+ * - Real-time result inspection
+ * - No modifications to actual DAG code
+ */
+
+const path = require('path');
+
+// Import the actual framework components
+let Orchestrator, ManifestBuilder;
+try {
+    ({ Orchestrator } = require('../../framework/execution/Orchestrator'));
+    ({ ManifestBuilder } = require('../../framework/core/Manifest'));
+} catch (e) {
+    // Handle case where we're being loaded from a different directory
+    console.warn('Could not load framework components directly:', e.message);
+}
+
+// Import our mock components
+const { MockDataFetcher } = require('./MockDataFetcher');
+const { MockStorageManager } = require('./MockStorageManager');
+const { SystemIntrospector } = require('../shared/SystemIntrospector');
+const { MockDataFactory } = require('../shared/MockDataFactory');
+
+// **NEW**: Import MOCK_INSTRUMENTS to populate reference tables
+const { MOCK_INSTRUMENTS } = require('../shared/SchemaTemplates');
+
+// Helper to generate IDs (duplicate of logic in MockDataFactory, but needed here for consistency)
+function generateEntityIds(count) {
+    const ids = [];
+    for (let i = 0; i < count; i++) {
+        // Use realistic PI-like IDs (8 digits)
+        ids.push(String(Math.floor(Math.random() * 40000000) + 10000000));
+    }
+    return ids;
+}
+
+class SimulationEngine {
+    /**
+     * @param {Object} config - The bulltrackers.config.js exports
+     * @param {Object} options
+     * @param {number} [options.entityCount=10] - Number of mock entities
+     * @param {string[]} [options.entityIds] - Specific entity IDs to use
+     * @param {Date|string} [options.targetDate] - Simulation date
+     * @param {Object} [options.preloadedData] - Pre-generated mock data
+     * @param {boolean} [options.verbose=false] - Enable verbose logging
+     */
+    constructor(config, options = {}) {
+        this.config = config;
+        this.options = options;
+        this.entityCount = options.entityCount || 10;
+
+        // FIX: Ensure we have a consistent set of Entity IDs immediately.
+        // If we leave this null, MockDataFactory generates different random IDs for every table, breaking joins.
+        this.entityIds = options.entityIds || generateEntityIds(this.entityCount);
+
+        this.targetDate = options.targetDate || new Date().toISOString().split('T')[0];
+        this.verbose = options.verbose || false;
+
+        // Initialize shared services
+        this.introspector = new SystemIntrospector(config);
+        this.mockDataFactory = new MockDataFactory({ introspector: this.introspector });
+
+        // **KEY FIX**: Preload Global Reference Data (Mappings)
+        // This ensures that ALL mock instruments exist in the ticker_mappings and sector_mappings tables.
+        const preloadedData = options.preloadedData || {};
+
+        // 1. Preload Ticker Mappings (InstrumentID -> Ticker)
+        // We replicate the FULL list of instruments for EVERY entity ID.
+        // This satisfies the MockDataFetcher's sharding logic while acting like a global table.
+        if (!preloadedData['ticker_mappings'] && MOCK_INSTRUMENTS) {
+            const mappingRows = MOCK_INSTRUMENTS.map(instr => ({
+                instrument_id: instr.id,
+                ticker: instr.ticker,
+                last_updated: new Date().toISOString()
+            }));
+
+            preloadedData['ticker_mappings'] = {};
+            for (const id of this.entityIds) {
+                preloadedData['ticker_mappings'][id] = mappingRows;
+            }
+        }
+
+        // 2. Preload Sector Mappings (Ticker -> Sector)
+        if (!preloadedData['sector_mappings'] && MOCK_INSTRUMENTS) {
+            const sectorRows = MOCK_INSTRUMENTS.map(instr => ({
+                symbol: instr.ticker,
+                sector: instr.sector
+            }));
+
+            preloadedData['sector_mappings'] = {};
+            for (const id of this.entityIds) {
+                preloadedData['sector_mappings'][id] = sectorRows;
+            }
+        }
+
+        // Initialize mock components
+        this.mockDataFetcher = new MockDataFetcher({
+            mockDataFactory: this.mockDataFactory,
+            introspector: this.introspector,
+            preloadedData: preloadedData,
+            entityCount: this.entityCount,
+            entityIds: this.entityIds // Pass the consistent IDs here
+        });
+
+        this.mockStorageManager = new MockStorageManager({
+            config
+        });
+
+        // Orchestrator will be created lazily
+        this._orchestrator = null;
+        this._manifest = null;
+
+        // Simulation state
+        this.simulationResults = {};
+        this.executionLog = [];
+    }
+
+    /**
+     * Initialize the simulation engine.
+     * Creates an Orchestrator with mocked services.
+     */
+    async initialize() {
+        if (this._orchestrator) return;
+
+        this._log('Initializing simulation engine...');
+
+        // Create a custom logger that captures output
+        const logger = this._createLogger();
+
+        // Create the real Orchestrator
+        this._orchestrator = new Orchestrator(this.config, logger);
+
+        // **KEY INJECTION**: Replace the data/storage components with mocks
+        this._orchestrator.dataFetcher = this.mockDataFetcher;
+        this._orchestrator.storageManager = this.mockStorageManager;
+
+        // **MOCK SchemaRegistry**: Prevent BigQuery connections during simulation
+        this._orchestrator.schemaRegistry = {
+            warmCache: async () => { this._log('Mock SchemaRegistry: warmCache skipped'); },
+            getSchema: (table) => ({ columns: [] }),
+            hasTable: () => true
+        };
+
+        // **MOCK StateRepository**: Prevent BigQuery state lookups
+        this._orchestrator.stateRepository = {
+            getDailyStatus: async () => new Map(),
+            getRunDates: async () => [],
+            updateStatusCache: () => { },
+            getResult: async () => ({}),
+            getEntityResult: async () => null,
+            getBatchEntityResults: async () => ({}),
+            cacheResult: () => { },
+            _log: () => { }
+        };
+
+        // Initialize (builds manifest etc)
+        await this._orchestrator.initialize();
+        this._manifest = this._orchestrator.manifest;
+
+        this._log(`Initialized with ${this._manifest.length} computations`);
+    }
+
+    /**
+     * Run a single computation in simulation mode.
+     * * @param {string} computationName - Name of the computation to simulate
+     * @param {Object} options
+     * @param {string[]} [options.entityIds] - Specific entities to process
+     * @param {string} [options.targetDate] - Override target date
+     * @returns {Object} Simulation result with metrics
+     */
+    async runComputation(computationName, options = {}) {
+        await this.initialize();
+
+        const targetDate = options.targetDate || this.targetDate;
+
+        // Use the consistent IDs unless overridden
+        const entityIds = options.entityIds || this.entityIds;
+
+        this._log(`Running simulation: ${computationName} for ${targetDate}`);
+
+        // Find the manifest entry (case-insensitive)
+        const normalizedName = computationName.toLowerCase();
+        const entry = this._manifest.find(e =>
+            e.name === computationName ||
+            e.name.toLowerCase() === normalizedName
+        );
+        if (!entry) {
+            throw new Error(`Computation not found: ${computationName}`);
+        }
+
+        // Load dependency results into mock DataFetcher
+        await this._loadDependencyResults(entry, targetDate);
+
+        // Configure entity IDs if specified (update the fetcher)
+        if (entityIds) {
+            this.mockDataFetcher.entityIds = entityIds;
+        }
+
+        // Run the computation through the real Orchestrator
+        const startTime = Date.now();
+        const startMemory = process.memoryUsage().heapUsed;
+
+        try {
+            await this._orchestrator.runSingle(entry, targetDate, {
+                entityIds,
+                dryRun: false,
+                forceLocal: true,  // Disable remote workers in simulation
+                allowPartialCommit: true // Enable result capture for partial simulations
+            });
+
+            const endTime = Date.now();
+            const endMemory = process.memoryUsage().heapUsed;
+
+            // Gather results
+            const results = this.mockStorageManager.getResults(computationName);
+            const stats = {
+                executionTimeMs: endTime - startTime,
+                memoryUsedBytes: endMemory - startMemory,
+                entitiesProcessed: Object.keys(results).length,
+                dataFetcher: this.mockDataFetcher.getStats(),
+                storage: this.mockStorageManager.getStats()
+            };
+
+            // Store in simulation results for dependent computations
+            this.simulationResults[computationName] = {
+                results,
+                stats,
+                entry,
+                date: targetDate
+            };
+
+            // Log execution
+            this.executionLog.push({
+                computation: computationName,
+                date: targetDate,
+                success: true,
+                stats,
+                timestamp: new Date().toISOString()
+            });
+
+            this._log(`✓ ${computationName} completed in ${stats.executionTimeMs}ms, ${stats.entitiesProcessed} entities`);
+
+            return {
+                success: true,
+                results,
+                stats,
+                passLevel: this.introspector.getPassLevel(computationName),
+                dependencies: entry.dependencies || []
+            };
+
+        } catch (error) {
+            this.executionLog.push({
+                computation: computationName,
+                date: targetDate,
+                success: false,
+                error: error.message,
+                stack: error.stack,
+                timestamp: new Date().toISOString()
+            });
+
+            this._log(`✗ ${computationName} failed: ${error.message}`);
+            if (this.verbose) {
+                console.error(error.stack);
+            }
+
+            return {
+                success: false,
+                error: error.message,
+                stack: error.stack,
+                stack: error.stack
+            };
+        }
+    }
+
+    /**
+     * Run a full simulation of the DAG up to a specific computation.
+     * Executes all dependencies in topological order first.
+     * * @param {string} computationName - Target computation
+     * @param {Object} options
+     * @returns {Object} Full simulation result
+     */
+    async runWithDependencies(computationName, options = {}) {
+        await this.initialize();
+
+        const targetDate = options.targetDate || this.targetDate;
+        const executionOrder = this._getExecutionOrder(computationName);
+
+        this._log(`Running ${computationName} with ${executionOrder.length} total computations`);
+
+        const results = [];
+
+        for (const compName of executionOrder) {
+            const result = await this.runComputation(compName, { targetDate, ...options });
+            results.push({ name: compName, ...result });
+
+            if (!result.success) {
+                this._log(`Pipeline stopped due to failure in ${compName}`);
+                break;
+            }
+        }
+
+        // Get the target computation result
+        const targetResult = results.find(r => r.name === computationName);
+
+        return {
+            targetComputation: computationName,
+            executionOrder,
+            results,
+            targetResult,
+            totalTimeMs: results.reduce((sum, r) => sum + (r.stats?.executionTimeMs || 0), 0)
+        };
+    }
+
+    /**
+     * Get DAG analysis for a computation.
+     * * @param {string} computationName
+     * @returns {Object} DAG info
+     */
+    getComputationInfo(computationName) {
+        const passLevel = this.introspector.getPassLevel(computationName);
+        const dependencies = this.introspector.getDependencyGraph().get(computationName) || [];
+        const config = this.introspector.getComputationConfig(computationName);
+
+        // Find dependents (computations that depend on this one)
+        const dependents = [];
+        for (const [name, deps] of this.introspector.getDependencyGraph()) {
+            if (deps.includes(computationName)) {
+                dependents.push(name);
+            }
+        }
+
+        return {
+            name: computationName,
+            passLevel,
+            totalPasses: this.introspector.getTotalPasses(),
+            dependencies,
+            dependents,
+            config,
+            validation: config ? this.introspector.validateComputationConfig(config) : null
+        };
+    }
+
+    /**
+     * Validate a computation without running it.
+     * * @param {string} computationName
+     * @returns {Object} Validation result
+     */
+    validateComputation(computationName) {
+        const config = this.introspector.getComputationConfig(computationName);
+        if (!config) {
+            return { valid: false, errors: [`Computation not found: ${computationName}`] };
+        }
+
+        return this.introspector.validateComputationConfig(config);
+    }
+
+    /**
+     * Get sample results for an entity.
+     * * @param {string} computationName
+     * @param {string} entityId
+     * @returns {Object|null}
+     */
+    getResult(computationName, entityId) {
+        const results = this.mockStorageManager.getResults(computationName);
+        return results[entityId] || null;
+    }
+
+    /**
+     * Get all results for a computation.
+     * * @param {string} computationName
+     * @returns {Object}
+     */
+    getResults(computationName) {
+        return this.mockStorageManager.getResults(computationName);
+    }
+
+    /**
+     * Compare two simulation runs.
+     * * @param {Object} result1 - First simulation result
+     * @param {Object} result2 - Second simulation result
+     * @returns {Object} Diff
+     */
+    compareResults(result1, result2) {
+        const diff = {
+            added: [],
+            removed: [],
+            changed: [],
+            unchanged: []
+        };
+
+        const entities1 = new Set(Object.keys(result1.results || {}));
+        const entities2 = new Set(Object.keys(result2.results || {}));
+
+        // Find added/removed
+        for (const id of entities2) {
+            if (!entities1.has(id)) diff.added.push(id);
+        }
+        for (const id of entities1) {
+            if (!entities2.has(id)) diff.removed.push(id);
+        }
+
+        // Find changed/unchanged
+        for (const id of entities1) {
+            if (entities2.has(id)) {
+                const r1 = JSON.stringify(result1.results[id]);
+                const r2 = JSON.stringify(result2.results[id]);
+                if (r1 === r2) {
+                    diff.unchanged.push(id);
+                } else {
+                    diff.changed.push({
+                        entityId: id,
+                        before: result1.results[id],
+                        after: result2.results[id]
+                    });
+                }
+            }
+        }
+
+        return diff;
+    }
+
+    /**
+     * Reset simulation state.
+     */
+    reset() {
+        this.mockDataFetcher.resetStats();
+        this.mockStorageManager.clear();
+        this.simulationResults = {};
+        this.executionLog = [];
+    }
+
+    /**
+     * Get execution log.
+     */
+    getExecutionLog() {
+        return [...this.executionLog];
+    }
+
+    /**
+     * Get system statistics.
+     */
+    getSystemStats() {
+        return this.introspector.getStats();
+    }
+
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+
+    /**
+     * Load dependency results into mock fetcher for multi-pass simulation.
+     */
+    async _loadDependencyResults(entry, targetDate) {
+        if (!entry.dependencies || entry.dependencies.length === 0) return;
+
+        for (const depName of entry.dependencies) {
+            // Check if we've already simulated this dependency
+            if (this.simulationResults[depName]) {
+                const depResults = this.simulationResults[depName].results;
+                this.mockDataFetcher.addComputationResults(depName, depResults);
+                this._log(`Loaded cached results for dependency: ${depName}`);
+            } else {
+                // Dependency hasn't been run yet - need to run it first
+                this._log(`Dependency ${depName} not yet simulated, running...`);
+                await this.runComputation(depName, { targetDate });
+
+                // Now load its results
+                const depResults = this.mockStorageManager.getResultsForDependency(depName);
+                this.mockDataFetcher.addComputationResults(depName, depResults);
+            }
+        }
+    }
+
+    /**
+     * Get execution order for a computation (topological sort).
+     */
+    _getExecutionOrder(computationName) {
+        const graph = this.introspector.getDependencyGraph();
+        const order = [];
+        const visited = new Set();
+
+        const visit = (name) => {
+            if (visited.has(name)) return;
+            visited.add(name);
+
+            const deps = graph.get(name) || [];
+            for (const dep of deps) {
+                visit(dep);
+            }
+
+            order.push(name);
+        };
+
+        visit(computationName);
+        return order;
+    }
+
+    /**
+     * Create a logger that captures output.
+     */
+    _createLogger() {
+        const verbose = this.verbose;
+        return {
+            log: (...args) => verbose && console.log('[SIM]', ...args),
+            info: (...args) => verbose && console.log('[SIM]', ...args),
+            warn: (...args) => console.warn('[SIM:WARN]', ...args),
+            error: (...args) => console.error('[SIM:ERROR]', ...args),
+            debug: (...args) => verbose && console.debug('[SIM:DEBUG]', ...args)
+        };
+    }
+
+    _log(message) {
+        if (this.verbose) {
+            console.log(`[SimulationEngine] ${message}`);
+        }
+    }
+}
+
+module.exports = { SimulationEngine };