bulltrackers-module 1.0.768 → 1.0.770

package/functions/computation-system-v2/devtools/shared/SystemIntrospector.js

@@ -0,0 +1,517 @@
+/**
+ * @fileoverview System Introspector
+ * 
+ * Central intelligence layer for the developer tooling platform.
+ * Extracts and exposes knowledge about the computation system:
+ * - Available tables and their schemas
+ * - Business rule modules and functions
+ * - System constraints (lookback limits, valid types)
+ * - Computation dependency graph
+ * 
+ * Used by: VS Code Extension, Simulation Engine, Builder
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+class SystemIntrospector {
+    /**
+     * @param {Object} config - The bulltrackers.config.js exports
+     * @param {Object} [options] - Optional overrides
+     * @param {Object} [options.schemaRegistry] - SchemaRegistry instance for live column data
+     * @param {Object} [options.rulesRegistry] - RulesRegistry instance for rule metadata
+     */
+    constructor(config, options = {}) {
+        this.config = config;
+        this.schemaRegistry = options.schemaRegistry || null;
+        this.rulesRegistry = options.rulesRegistry || null;
+
+        // Cache for computed values
+        this._cache = {
+            computationConfigs: null,
+            dependencyGraph: null,
+            passLevels: null,
+            ruleIndex: null
+        };
+
+        // System constraints
+        this.constraints = {
+            maxLookback: 90,
+            validTypes: ['global', 'per-entity'],
+            validCategories: ['popular_investor', 'signed_in_user', 'global', 'instrument']
+        };
+    }
+
+    // =========================================================================
+    // TABLE INTROSPECTION
+    // =========================================================================
+
+    /**
+     * Get list of all configured table names.
+     * @returns {string[]}
+     */
+    getAvailableTables() {
+        return Object.keys(this.config.tables || {});
+    }
+
+    /**
+     * Get metadata for a specific table.
+     * @param {string} tableName 
+     * @returns {Object|null} { dateField, entityField, clusterFields, description }
+     */
+    getTableMetadata(tableName) {
+        const table = this.config.tables?.[tableName];
+        if (!table) return null;
+
+        return {
+            tableName: table.tableName || tableName,
+            dateField: table.dateField || null,
+            entityField: table.entityField || null,
+            clusterFields: table.clusterFields || [],
+            description: table.description || null,
+            isPartitioned: !!table.dateField
+        };
+    }
+
+    /**
+     * Check if a table exists in configuration.
+     * @param {string} tableName 
+     * @returns {boolean}
+     */
+    tableExists(tableName) {
+        return tableName in (this.config.tables || {});
+    }
+
+    /**
+     * Get table columns. Uses SchemaRegistry if available, otherwise returns null.
+     * @param {string} tableName 
+     * @returns {Promise<string[]|null>}
+     */
+    async getTableColumns(tableName) {
+        if (!this.schemaRegistry) return null;
+
+        try {
+            const schema = await this.schemaRegistry.getSchema(tableName);
+            return schema?.columns?.map(c => c.name) || null;
+        } catch (e) {
+            return null;
+        }
+    }
+
+    /**
+     * Validate that a column exists in a table.
+     * @param {string} tableName 
+     * @param {string} columnName 
+     * @returns {Promise<boolean>}
+     */
+    async columnExists(tableName, columnName) {
+        const columns = await this.getTableColumns(tableName);
+        if (!columns) return true; // Can't validate without schema, assume valid
+        return columns.includes(columnName);
+    }
+
+    // =========================================================================
+    // RULE INTROSPECTION
+    // =========================================================================
+
+    /**
+     * Get list of all rule module names.
+     * @returns {string[]}
+     */
+    getRuleModules() {
+        const rules = this.config.rules || {};
+        return Object.keys(rules);
+    }
+
+    /**
+     * Get list of functions exported by a rule module.
+     * @param {string} moduleName 
+     * @returns {string[]}
+     */
+    getRuleFunctions(moduleName) {
+        const module = this.config.rules?.[moduleName];
+        if (!module) return [];
+
+        return Object.keys(module).filter(key => typeof module[key] === 'function');
+    }
+
+    /**
+     * Check if a rule function exists.
+     * @param {string} moduleName 
+     * @param {string} functionName 
+     * @returns {boolean}
+     */
+    ruleFunctionExists(moduleName, functionName) {
+        const module = this.config.rules?.[moduleName];
+        if (!module) return false;
+        return typeof module[functionName] === 'function';
+    }
+
+    /**
+     * Get all rule functions as a flat index.
+     * @returns {Map<string, { module: string, path: string }>}
+     */
+    getRuleIndex() {
+        if (this._cache.ruleIndex) return this._cache.ruleIndex;
+
+        const index = new Map();
+        for (const moduleName of this.getRuleModules()) {
+            for (const funcName of this.getRuleFunctions(moduleName)) {
+                index.set(funcName, {
+                    module: moduleName,
+                    path: `rules.${moduleName}.${funcName}`
+                });
+            }
+        }
+
+        this._cache.ruleIndex = index;
+        return index;
+    }
+
+    /**
+     * Find the closest matching rule function (for "did you mean" suggestions).
+     * @param {string} moduleName 
+     * @param {string} attemptedName 
+     * @returns {string|null}
+     */
+    findClosestRuleFunction(moduleName, attemptedName) {
+        const functions = this.getRuleFunctions(moduleName);
+        if (functions.length === 0) return null;
+
+        // Simple Levenshtein-based matching
+        let closest = null;
+        let minDistance = Infinity;
+
+        for (const fn of functions) {
+            const distance = this._levenshtein(attemptedName.toLowerCase(), fn.toLowerCase());
+            if (distance < minDistance && distance <= 3) {
+                minDistance = distance;
+                closest = fn;
+            }
+        }
+
+        return closest;
+    }
+
+    /**
+     * Find the closest matching table name.
+     * @param {string} attemptedName 
+     * @returns {string|null}
+     */
+    findClosestTable(attemptedName) {
+        const tables = this.getAvailableTables();
+        let closest = null;
+        let minDistance = Infinity;
+
+        for (const table of tables) {
+            const distance = this._levenshtein(attemptedName.toLowerCase(), table.toLowerCase());
+            if (distance < minDistance && distance <= 5) {
+                minDistance = distance;
+                closest = table;
+            }
+        }
+
+        return closest;
+    }
+
+    // =========================================================================
+    // CONSTRAINT INTROSPECTION
+    // =========================================================================
+
+    /**
+     * Get the maximum allowed lookback value.
+     * @returns {number}
+     */
+    getMaxLookback() {
+        return this.constraints.maxLookback;
+    }
+
+    /**
+     * Get valid computation types.
+     * @returns {string[]}
+     */
+    getValidTypes() {
+        return [...this.constraints.validTypes];
+    }
+
+    /**
+     * Get valid computation categories.
+     * @returns {string[]}
+     */
+    getValidCategories() {
+        // Extract from existing computations + defaults
+        const categories = new Set(this.constraints.validCategories);
+
+        for (const comp of this.config.computations || []) {
+            try {
+                const cfg = comp.getConfig();
+                if (cfg.category) categories.add(cfg.category);
+            } catch (e) { /* skip */ }
+        }
+
+        return Array.from(categories);
+    }
+
+    // =========================================================================
+    // COMPUTATION INTROSPECTION
+    // =========================================================================
+
+    /**
+     * Get list of all computation names.
+     * @returns {string[]}
+     */
+    getComputationNames() {
+        const names = [];
+        for (const comp of this.config.computations || []) {
+            try {
+                names.push(comp.getConfig().name);
+            } catch (e) { /* skip */ }
+        }
+        return names;
+    }
+
+    /**
+     * Get config for a specific computation.
+     * @param {string} name 
+     * @returns {Object|null}
+     */
+    getComputationConfig(name) {
+        for (const comp of this.config.computations || []) {
+            try {
+                const cfg = comp.getConfig();
+                if (cfg.name === name) return cfg;
+            } catch (e) { /* skip */ }
+        }
+        return null;
+    }
+
+    /**
+     * Get all computation configs, cached.
+     * @returns {Map<string, Object>}
+     */
+    getAllComputationConfigs() {
+        if (this._cache.computationConfigs) return this._cache.computationConfigs;
+
+        const map = new Map();
+        for (const comp of this.config.computations || []) {
+            try {
+                const cfg = comp.getConfig();
+                map.set(cfg.name, cfg);
+            } catch (e) { /* skip */ }
+        }
+
+        this._cache.computationConfigs = map;
+        return map;
+    }
+
+    /**
+     * Build dependency graph from computation configs.
+     * @returns {Map<string, string[]>} computation name -> dependencies
+     */
+    getDependencyGraph() {
+        if (this._cache.dependencyGraph) return this._cache.dependencyGraph;
+
+        const graph = new Map();
+        const configs = this.getAllComputationConfigs();
+
+        for (const [name, cfg] of configs) {
+            const deps = [];
+
+            // Explicit dependencies
+            if (cfg.dependencies) {
+                deps.push(...cfg.dependencies);
+            }
+
+            // Conditional dependencies
+            if (cfg.conditionalDependencies) {
+                for (const cd of cfg.conditionalDependencies) {
+                    if (cd.computation) deps.push(cd.computation);
+                }
+            }
+
+            graph.set(name, deps);
+        }
+
+        this._cache.dependencyGraph = graph;
+        return graph;
+    }
+
+    /**
+     * Calculate pass levels (topological depth) for all computations.
+     * @returns {Map<string, number>}
+     */
+    getPassLevels() {
+        if (this._cache.passLevels) return this._cache.passLevels;
+
+        const graph = this.getDependencyGraph();
+        const levels = new Map();
+        const visited = new Set();
+
+        const calculateLevel = (name) => {
+            if (levels.has(name)) return levels.get(name);
+            if (visited.has(name)) return 0; // Cycle detected
+            visited.add(name);
+
+            const deps = graph.get(name) || [];
+            if (deps.length === 0) {
+                levels.set(name, 1);
+                return 1;
+            }
+
+            let maxDepLevel = 0;
+            for (const dep of deps) {
+                maxDepLevel = Math.max(maxDepLevel, calculateLevel(dep));
+            }
+
+            const level = maxDepLevel + 1;
+            levels.set(name, level);
+            return level;
+        };
+
+        for (const name of graph.keys()) {
+            calculateLevel(name);
+        }
+
+        this._cache.passLevels = levels;
+        return levels;
+    }
+
+    /**
+     * Get the pass level for a specific computation.
+     * @param {string} name 
+     * @returns {number}
+     */
+    getPassLevel(name) {
+        return this.getPassLevels().get(name) || 1;
+    }
+
+    /**
+     * Get total number of passes needed to run all computations.
+     * @returns {number}
+     */
+    getTotalPasses() {
+        const levels = this.getPassLevels();
+        return Math.max(...levels.values(), 1);
+    }
+
+    // =========================================================================
+    // VALIDATION HELPERS
+    // =========================================================================
+
+    /**
+     * Validate a computation config object.
+     * @param {Object} config 
+     * @returns {{ valid: boolean, errors: string[], warnings: string[] }}
+     */
+    validateComputationConfig(config) {
+        const errors = [];
+        const warnings = [];
+
+        // Required fields
+        if (!config.name) errors.push('Missing required field: name');
+        if (!config.requires) errors.push('Missing required field: requires');
+
+        // Type validation
+        if (config.type && !this.constraints.validTypes.includes(config.type)) {
+            errors.push(`Invalid type: "${config.type}". Must be one of: ${this.constraints.validTypes.join(', ')}`);
+        }
+
+        // Table validation
+        if (config.requires) {
+            for (const tableName of Object.keys(config.requires)) {
+                if (!this.tableExists(tableName)) {
+                    const closest = this.findClosestTable(tableName);
+                    const suggestion = closest ? ` Did you mean "${closest}"?` : '';
+                    errors.push(`Unknown table: "${tableName}".${suggestion}`);
+                }
+
+                // Lookback validation
+                const tableReq = config.requires[tableName];
+                if (tableReq.lookback !== undefined) {
+                    if (tableReq.lookback > this.constraints.maxLookback) {
+                        errors.push(`Lookback ${tableReq.lookback} for "${tableName}" exceeds maximum of ${this.constraints.maxLookback}`);
+                    } else if (tableReq.lookback > this.constraints.maxLookback * 0.7) {
+                        warnings.push(`Lookback ${tableReq.lookback} for "${tableName}" is ${Math.round(tableReq.lookback / this.constraints.maxLookback * 100)}% of maximum`);
+                    }
+
+                    // Check lookback on non-partitioned tables
+                    const meta = this.getTableMetadata(tableName);
+                    if (meta && !meta.isPartitioned && tableReq.lookback > 0) {
+                        warnings.push(`Lookback on non-partitioned table "${tableName}" will fetch all rows`);
+                    }
+                }
+            }
+        }
+
+        // Dependency validation
+        if (config.dependencies) {
+            for (const dep of config.dependencies) {
+                if (!this.getComputationNames().includes(dep)) {
+                    errors.push(`Unknown dependency: "${dep}"`);
+                }
+            }
+        }
+
+        return { valid: errors.length === 0, errors, warnings };
+    }
+
+    // =========================================================================
+    // HELPERS
+    // =========================================================================
+
+    /**
+     * Levenshtein distance for fuzzy matching.
+     */
+    _levenshtein(a, b) {
+        const matrix = [];
+        for (let i = 0; i <= b.length; i++) {
+            matrix[i] = [i];
+        }
+        for (let j = 0; j <= a.length; j++) {
+            matrix[0][j] = j;
+        }
+        for (let i = 1; i <= b.length; i++) {
+            for (let j = 1; j <= a.length; j++) {
+                if (b.charAt(i - 1) === a.charAt(j - 1)) {
+                    matrix[i][j] = matrix[i - 1][j - 1];
+                } else {
+                    matrix[i][j] = Math.min(
+                        matrix[i - 1][j - 1] + 1,
+                        matrix[i][j - 1] + 1,
+                        matrix[i - 1][j] + 1
+                    );
+                }
+            }
+        }
+        return matrix[b.length][a.length];
+    }
+
+    /**
+     * Clear all caches.
+     */
+    clearCache() {
+        this._cache = {
+            computationConfigs: null,
+            dependencyGraph: null,
+            passLevels: null,
+            ruleIndex: null
+        };
+    }
+
+    /**
+     * Get summary statistics about the system.
+     * @returns {Object}
+     */
+    getStats() {
+        return {
+            tables: this.getAvailableTables().length,
+            ruleModules: this.getRuleModules().length,
+            ruleFunctions: this.getRuleIndex().size,
+            computations: this.getComputationNames().length,
+            totalPasses: this.getTotalPasses(),
+            maxLookback: this.constraints.maxLookback
+        };
+    }
+}
+
+module.exports = { SystemIntrospector };

package/functions/computation-system-v2/devtools/shared/index.js

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Shared DevTools Module Index
+ * 
+ * Central exports for the developer tooling shared layer.
+ */
+
+const { SystemIntrospector } = require('./SystemIntrospector');
+const { MockDataFactory } = require('./MockDataFactory');
+const { SchemaTemplates, helpers: schemaHelpers } = require('./SchemaTemplates');
+
+module.exports = {
+    SystemIntrospector,
+    MockDataFactory,
+    SchemaTemplates,
+    schemaHelpers
+};