bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/devtools/simulation/DAGAnalyzer.js

@@ -0,0 +1,243 @@
+/**
+ * @fileoverview DAG Analyzer
+ * 
+ * Analyzes the computation DAG for visualization and debugging.
+ * Provides pass levels, dependency chains, and execution order info.
+ */
+
+class DAGAnalyzer {
+    /**
+     * @param {Object} introspector - SystemIntrospector instance
+     */
+    constructor(introspector) {
+        this.introspector = introspector;
+    }
+
+    /**
+     * Get full DAG structure for visualization.
+     * @returns {Object} { nodes: [], edges: [] }
+     */
+    getVisualizationData() {
+        const graph = this.introspector.getDependencyGraph();
+        const levels = this.introspector.getPassLevels();
+        const configs = this.introspector.getAllComputationConfigs();
+
+        const nodes = [];
+        const edges = [];
+
+        for (const [name, deps] of graph) {
+            const config = configs.get(name);
+            nodes.push({
+                id: name,
+                label: name,
+                level: levels.get(name) || 1,
+                type: config?.type || 'unknown',
+                category: config?.category || 'unknown',
+                isHistorical: config?.isHistorical || false
+            });
+
+            for (const dep of deps) {
+                edges.push({
+                    source: dep,
+                    target: name,
+                    id: `${dep}->${name}`
+                });
+            }
+        }
+
+        return { nodes, edges };
+    }
+
+    /**
+     * Get execution plan for a target computation.
+     * @param {string} targetComputation
+     * @returns {Object} Execution plan
+     */
+    getExecutionPlan(targetComputation) {
+        const graph = this.introspector.getDependencyGraph();
+        const levels = this.introspector.getPassLevels();
+
+        const order = [];
+        const visited = new Set();
+
+        const visit = (name) => {
+            if (visited.has(name)) return;
+            visited.add(name);
+
+            const deps = graph.get(name) || [];
+            for (const dep of deps) {
+                visit(dep);
+            }
+
+            order.push(name);
+        };
+
+        visit(targetComputation);
+
+        // Group by pass level
+        const byPass = {};
+        for (const name of order) {
+            const level = levels.get(name) || 1;
+            if (!byPass[level]) byPass[level] = [];
+            byPass[level].push(name);
+        }
+
+        return {
+            target: targetComputation,
+            totalComputations: order.length,
+            executionOrder: order,
+            byPassLevel: byPass,
+            estimatedPasses: Object.keys(byPass).length
+        };
+    }
+
+    /**
+     * Find all paths from source to target computation.
+     * @param {string} source
+     * @param {string} target
+     * @returns {string[][]} Array of paths
+     */
+    findPaths(source, target) {
+        const graph = this.introspector.getDependencyGraph();
+        const paths = [];
+
+        const dfs = (current, path) => {
+            if (current === target) {
+                paths.push([...path, current]);
+                return;
+            }
+
+            const deps = graph.get(current) || [];
+            for (const dep of deps) {
+                if (!path.includes(dep)) {
+                    dfs(dep, [...path, current]);
+                }
+            }
+        };
+
+        dfs(source, []);
+        return paths;
+    }
+
+    /**
+     * Get impact analysis - what would be affected if a computation changes.
+     * @param {string} computationName
+     * @returns {Object}
+     */
+    getImpactAnalysis(computationName) {
+        const graph = this.introspector.getDependencyGraph();
+        const affected = new Set();
+
+        // Find all computations that depend (directly or indirectly) on this one
+        const findDependents = (name) => {
+            for (const [candidate, deps] of graph) {
+                if (deps.includes(name) && !affected.has(candidate)) {
+                    affected.add(candidate);
+                    findDependents(candidate);
+                }
+            }
+        };
+
+        findDependents(computationName);
+
+        return {
+            computation: computationName,
+            directDependents: [...graph.entries()]
+                .filter(([_, deps]) => deps.includes(computationName))
+                .map(([name]) => name),
+            allAffected: Array.from(affected),
+            impactScore: affected.size
+        };
+    }
+
+    /**
+     * Detect cycles in the DAG (should always be empty for valid DAG).
+     * @returns {string[][]} Any cycles found
+     */
+    detectCycles() {
+        const graph = this.introspector.getDependencyGraph();
+        const cycles = [];
+        const visited = new Set();
+        const recursionStack = new Set();
+
+        const dfs = (node, path) => {
+            if (recursionStack.has(node)) {
+                const cycleStart = path.indexOf(node);
+                cycles.push(path.slice(cycleStart));
+                return;
+            }
+            if (visited.has(node)) return;
+
+            visited.add(node);
+            recursionStack.add(node);
+
+            const deps = graph.get(node) || [];
+            for (const dep of deps) {
+                dfs(dep, [...path, node]);
+            }
+
+            recursionStack.delete(node);
+        };
+
+        for (const node of graph.keys()) {
+            dfs(node, []);
+        }
+
+        return cycles;
+    }
+
+    /**
+     * Get summary statistics about the DAG.
+     * @returns {Object}
+     */
+    getSummary() {
+        const graph = this.introspector.getDependencyGraph();
+        const levels = this.introspector.getPassLevels();
+
+        let totalEdges = 0;
+        let maxInDegree = 0;
+        let maxOutDegree = 0;
+
+        const outDegree = new Map();
+        const inDegree = new Map();
+
+        for (const [name, deps] of graph) {
+            outDegree.set(name, deps.length);
+            totalEdges += deps.length;
+            maxOutDegree = Math.max(maxOutDegree, deps.length);
+
+            for (const dep of deps) {
+                inDegree.set(dep, (inDegree.get(dep) || 0) + 1);
+            }
+        }
+
+        for (const [_, count] of inDegree) {
+            maxInDegree = Math.max(maxInDegree, count);
+        }
+
+        // Find entry points (no dependencies)
+        const entryPoints = [...graph.entries()]
+            .filter(([_, deps]) => deps.length === 0)
+            .map(([name]) => name);
+
+        // Find exit points (no dependents)
+        const allDeps = new Set();
+        for (const deps of graph.values()) {
+            for (const dep of deps) allDeps.add(dep);
+        }
+        const exitPoints = [...graph.keys()].filter(name => !allDeps.has(name) || true);
+
+        return {
+            totalNodes: graph.size,
+            totalEdges,
+            totalPasses: this.introspector.getTotalPasses(),
+            maxInDegree,
+            maxOutDegree,
+            entryPointCount: entryPoints.length,
+            entryPoints: entryPoints.slice(0, 5), // First 5
+            cycles: this.detectCycles()
+        };
+    }
+}
+
+module.exports = { DAGAnalyzer };

package/functions/computation-system-v2/devtools/simulation/MockDataFetcher.js

@@ -0,0 +1,306 @@
+/**
+ * @fileoverview Mock Data Fetcher
+ * * Drop-in replacement for DataFetcher that returns mock data.
+ * Designed to be injected into the real Orchestrator for 1:1 simulation.
+ */
+
+const { MockDataFactory } = require('../shared/MockDataFactory');
+
+class MockDataFetcher {
+    /**
+     * @param {Object} options
+     * @param {Object} [options.mockDataFactory] - Custom MockDataFactory instance
+     * @param {Object} [options.introspector] - SystemIntrospector for metadata
+     * @param {Object} [options.preloadedData] - Pre-generated data { tableName: { entityId: rows[] } }
+     * @param {Object} [options.computationResults] - Results from earlier passes
+     * @param {number} [options.entityCount=10] - Default entity count
+     * @param {string[]} [options.entityIds] - Specific entity IDs to generate
+     */
+    constructor(options = {}) {
+        this.mockDataFactory = options.mockDataFactory || new MockDataFactory(options);
+        this.introspector = options.introspector || null;
+        this.preloadedData = options.preloadedData || {};
+        this.computationResults = options.computationResults || {};
+        this.entityCount = options.entityCount || 10;
+        this.entityIds = options.entityIds || null;
+
+        // Stats tracking (mimics real DataFetcher)
+        this.stats = {
+            queriesExecuted: 0,
+            rowsReturned: 0,
+            tablesAccessed: new Set(),
+            errors: []
+        };
+
+        // Query log for inspection
+        this.queryLog = [];
+    }
+
+    /**
+     * Main entry point - matches DataFetcher.fetchForComputation signature.
+     * * @param {Object} requires - Computation requirements
+     * @param {Date|string} targetDate - Target date
+     * @param {string[]|null} entities - Specific entities to fetch
+     * @returns {Object} Data keyed by table name
+     */
+    async fetchForComputation(requires, targetDate, entities = null) {
+        const results = {};
+        const errors = [];
+
+        const effectiveEntities = entities || this.entityIds || null;
+
+        for (const [key, spec] of Object.entries(requires)) {
+            try {
+                // Handle metric type (computation dependency)
+                if (spec.type === 'metric') {
+                    results[key] = await this._fetchMetric(spec, targetDate, effectiveEntities);
+                    continue;
+                }
+
+                // Log the query
+                this.queryLog.push({
+                    table: key,
+                    targetDate,
+                    lookback: spec.lookback || 0,
+                    entities: effectiveEntities,
+                    timestamp: new Date().toISOString()
+                });
+
+                // Fetch mock data
+                const data = await this.fetch({
+                    table: key,
+                    targetDate,
+                    lookback: spec.lookback || 0,
+                    mandatory: spec.mandatory || false,
+                    fields: spec.fields || null,
+                    entities: effectiveEntities
+                });
+
+                results[key] = data;
+                this.stats.tablesAccessed.add(key);
+
+                // Check mandatory
+                if (spec.mandatory && this._isEmpty(data)) {
+                    errors.push({ table: key, reason: 'MANDATORY_MISSING' });
+                }
+
+            } catch (e) {
+                if (spec.mandatory) {
+                    errors.push({ table: key, reason: e.message });
+                }
+                results[key] = null;
+            }
+        }
+
+        if (errors.length > 0) {
+            const msg = errors.map(e => `${e.table}: ${e.reason}`).join(', ');
+            this.stats.errors.push(msg);
+            throw new Error(`[MockDataFetcher] Missing mandatory data: ${msg}`);
+        }
+
+        return results;
+    }
+
+    /**
+     * Low-level fetch - matches DataFetcher.fetch signature.
+     * * @param {Object} options
+     * @returns {Object} Data keyed by entity ID
+     */
+    async fetch(options) {
+        // Destructure 'lookback' and 'targetDate' from options
+        const { table, targetDate, lookback = 0, entities = null, fields = null } = options;
+
+        this.stats.queriesExecuted++;
+
+        // Check for preloaded data first
+        if (this.preloadedData[table]) {
+            const preloaded = this.preloadedData[table];
+            if (entities) {
+                const filtered = {};
+                for (const id of entities) {
+                    if (preloaded[id]) {
+                        filtered[id] = preloaded[id];
+                        this.stats.rowsReturned += preloaded[id].length;
+                    }
+                }
+                return filtered;
+            }
+            this.stats.rowsReturned += Object.values(preloaded).reduce((sum, rows) => sum + rows.length, 0);
+            return preloaded;
+        }
+
+        // Generate mock data
+        try {
+            const entityIds = entities || this.entityIds;
+            const { byEntity } = this.mockDataFactory.generate(table, {
+                entityCount: this.entityCount,
+                entityIds,
+                // FIX: Use 'lookback' and 'targetDate' variables, not 'daysBack'/'asOfDate'
+                daysBack: lookback || 0,
+                asOfDate: targetDate
+            });
+
+            // Convert Map to Object
+            const result = {};
+            for (const [id, rows] of byEntity) {
+                result[id] = rows;
+                this.stats.rowsReturned += rows.length;
+            }
+
+            return result;
+        } catch (e) {
+            // Log error to see why generation failed (optional but helpful)
+            console.error(`[MockDataFetcher] Generation failed for ${table}:`, e.message);
+            // Table not in templates - return empty data
+            return {};
+        }
+    }
+
+    /**
+     * Fetch metric (computation result from earlier pass).
+     */
+    async _fetchMetric(spec, targetDate, entities) {
+        const computationName = spec.computation;
+        if (!computationName) {
+            throw new Error('Metric spec missing "computation" field');
+        }
+
+        const cached = this.computationResults[computationName];
+        if (!cached) {
+            // Generate empty metric result
+            return {};
+        }
+
+        // Filter by entities if specified
+        if (entities) {
+            const filtered = {};
+            for (const id of entities) {
+                if (cached[id]) {
+                    filtered[id] = cached[id];
+                }
+            }
+            return filtered;
+        }
+
+        return cached;
+    }
+
+    /**
+     * Batched fetch - for streaming computations.
+     * Returns an async iterator that yields { data, entityIds } batches.
+     * NOTE: This is NOT async because Orchestrator calls it without await.
+     */
+    fetchComputationBatched(requires, targetDate, batchSize = 1000) {
+        const self = this;
+
+        // Return an async iterable that yields batches in correct format
+        return {
+            [Symbol.asyncIterator]() {
+                let yielded = false;
+                return {
+                    async next() {
+                        if (yielded) {
+                            return { done: true };
+                        }
+                        yielded = true;
+
+                        // Fetch data inside the iterator
+                        const allData = await self.fetchForComputation(requires, targetDate, self.entityIds);
+
+                        // Get all entity IDs from the first table that has data
+                        const firstTableWithData = Object.keys(allData).find(k => allData[k] && Object.keys(allData[k]).length > 0);
+                        const entityIds = firstTableWithData ? Object.keys(allData[firstTableWithData]) : (self.entityIds || []);
+
+                        return {
+                            done: false,
+                            value: { data: allData, entityIds }
+                        };
+                    }
+                };
+            }
+        };
+    }
+
+    /**
+     * Check data availability.
+     * Returns { canRun: boolean, missing: string[], available: { table: { available, rowCount } } }
+     */
+    async checkAvailability(requires, targetDate) {
+        // In mock mode, all data is always available
+        const available = {};
+        for (const table of Object.keys(requires)) {
+            available[table] = { available: true, rowCount: this.entityCount };
+        }
+        return {
+            canRun: true,
+            missing: [],
+            available
+        };
+    }
+
+    /**
+     * Check if data exists for a table.
+     */
+    async hasData(table, targetDate) {
+        return true; // Mock always has data
+    }
+
+    /**
+     * Get fetch statistics.
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            tablesAccessed: Array.from(this.stats.tablesAccessed)
+        };
+    }
+
+    /**
+     * Reset statistics.
+     */
+    resetStats() {
+        this.stats = {
+            queriesExecuted: 0,
+            rowsReturned: 0,
+            tablesAccessed: new Set(),
+            errors: []
+        };
+        this.queryLog = [];
+    }
+
+    /**
+     * Clear cache (no-op for mock).
+     */
+    clearCache() {
+        // No cache in mock
+    }
+
+    /**
+     * Add computation results for dependency simulation.
+     * * @param {string} computationName
+     * @param {Object} results - { entityId: result }
+     */
+    addComputationResults(computationName, results) {
+        this.computationResults[computationName] = results;
+    }
+
+    /**
+     * Get query log for inspection.
+     */
+    getQueryLog() {
+        return [...this.queryLog];
+    }
+
+    // =========================================================================
+    // HELPERS
+    // =========================================================================
+
+    _isEmpty(data) {
+        if (!data) return true;
+        if (Array.isArray(data)) return data.length === 0;
+        if (typeof data === 'object') return Object.keys(data).length === 0;
+        return false;
+    }
+}
+
+module.exports = { MockDataFetcher };