bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/core/Rules.js

@@ -0,0 +1,231 @@
+/**
+ * @fileoverview Business Rules Registry
+ * 
+ * Manages business rules that are injected into computations:
+ * 1. Loads all rule modules from config
+ * 2. Hashes each rule for version tracking
+ * 3. Detects which rules each computation uses
+ * 4. Injects rules into computation context
+ * 
+ * Design Philosophy:
+ * - Computations are "simple recipes" - they orchestrate, not calculate
+ * - Business logic (math, extraction, validation) lives in rules
+ * - Single point of reference for all shared functions
+ * - Rule changes automatically trigger re-runs of dependent computations
+ * 
+ * Example Usage:
+ * ```javascript
+ * // In computation process():
+ * async process(context) {
+ *     const { rules, data, entityId } = context;
+ *     
+ *     // Use injected rules - no require() needed
+ *     const portfolio = rules.portfolio.extractPositions(data['portfolio_snapshots']);
+ *     const sharpeRatio = rules.metrics.calculateSharpe(returns, riskFreeRate);
+ *     const riskScore = rules.risk.calculateRiskScore(portfolio, volatility);
+ *     
+ *     this.setResult(entityId, { sharpeRatio, riskScore });
+ * }
+ * ```
+ */
+
+const crypto = require('crypto');
+
+/**
+ * @typedef {Object} RuleModule
+ * @property {string} name - Module name (e.g., 'portfolio', 'metrics')
+ * @property {Object} exports - The exported functions/objects
+ * @property {string} hash - Hash of the module for version tracking
+ * @property {Object} functionHashes - Hash of each exported function
+ */
+
+class RulesRegistry {
+    /**
+     * @param {Object} config - Configuration containing rule modules
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger;
+        
+        // Storage
+        this.modules = new Map();       // name -> RuleModule
+        this.context = {};              // The injected context object
+        this.functionIndex = new Map(); // functionName -> { module, hash }
+        
+        // Load rules from config
+        this._loadRules();
+    }
+    
+    /**
+     * Get the rules context to inject into computations.
+     * @returns {Object} Rules context with all modules
+     */
+    getContext() {
+        return this.context;
+    }
+    
+    /**
+     * Get all rule hashes for a computation based on which rules it uses.
+     * @param {string} codeString - Computation source code
+     * @returns {Object} { usedRules: { moduleName: [functionNames] }, hashes: { moduleName: hash } }
+     */
+    detectUsage(codeString) {
+        const usedRules = {};
+        const hashes = {};
+        
+        for (const [moduleName, module] of this.modules) {
+            const usedFunctions = [];
+            
+            // Check each exported function
+            for (const [funcName, funcHash] of Object.entries(module.functionHashes)) {
+                // Look for patterns like: rules.moduleName.funcName or just funcName
+                const patterns = [
+                    new RegExp(`rules\\.${moduleName}\\.${funcName}\\b`),  // rules.portfolio.extractPositions
+                    new RegExp(`\\.${funcName}\\(`),                        // .extractPositions(
+                ];
+                
+                for (const pattern of patterns) {
+                    if (pattern.test(codeString)) {
+                        if (!usedFunctions.includes(funcName)) {
+                            usedFunctions.push(funcName);
+                        }
+                        break;
+                    }
+                }
+            }
+            
+            if (usedFunctions.length > 0) {
+                usedRules[moduleName] = usedFunctions;
+                
+                // Combine hashes of used functions
+                const usedHashes = usedFunctions
+                    .map(f => module.functionHashes[f])
+                    .sort()
+                    .join('|');
+                hashes[moduleName] = this._hash(usedHashes);
+            }
+        }
+        
+        return { usedRules, hashes };
+    }
+    
+    /**
+     * Get the combined hash of all rules (for safe mode).
+     * @returns {string} Combined hash
+     */
+    getAllHash() {
+        const allHashes = Array.from(this.modules.values())
+            .map(m => m.hash)
+            .sort()
+            .join('|');
+        return this._hash(allHashes);
+    }
+    
+    /**
+     * Get module hash by name.
+     * @param {string} moduleName - Module name
+     * @returns {string|null} Module hash or null
+     */
+    getModuleHash(moduleName) {
+        return this.modules.get(moduleName)?.hash || null;
+    }
+    
+    /**
+     * Get stats about loaded rules.
+     * @returns {Object} Stats
+     */
+    getStats() {
+        const stats = {
+            moduleCount: this.modules.size,
+            functionCount: this.functionIndex.size,
+            modules: {}
+        };
+        
+        for (const [name, module] of this.modules) {
+            stats.modules[name] = {
+                functionCount: Object.keys(module.functionHashes).length,
+                hash: module.hash
+            };
+        }
+        
+        return stats;
+    }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+    
+    _loadRules() {
+        const rulesConfig = this.config.rules || {};
+        
+        for (const [moduleName, moduleExports] of Object.entries(rulesConfig)) {
+            this._registerModule(moduleName, moduleExports);
+        }
+        
+        this._log('INFO', `Loaded ${this.modules.size} rule modules with ${this.functionIndex.size} functions`);
+    }
+    
+    _registerModule(name, exports) {
+        const functionHashes = {};
+        const cleanExports = {};
+        
+        // Process each export
+        for (const [key, value] of Object.entries(exports)) {
+            if (typeof value === 'function') {
+                // Hash the function source
+                functionHashes[key] = this._hashFunction(value);
+                cleanExports[key] = value;
+                
+                // Index for quick lookup
+                this.functionIndex.set(key, { module: name, hash: functionHashes[key] });
+            } else if (typeof value === 'object' && value !== null) {
+                // For objects (like constants or nested modules), hash the JSON
+                functionHashes[key] = this._hash(JSON.stringify(value));
+                cleanExports[key] = value;
+                this.functionIndex.set(key, { module: name, hash: functionHashes[key] });
+            }
+        }
+        
+        // Create module hash from all function hashes
+        const moduleHash = this._hash(
+            Object.values(functionHashes).sort().join('|')
+        );
+        
+        // Store module
+        this.modules.set(name, {
+            name,
+            exports: cleanExports,
+            hash: moduleHash,
+            functionHashes
+        });
+        
+        // Add to context
+        this.context[name] = cleanExports;
+    }
+    
+    _hashFunction(fn) {
+        const source = fn.toString();
+        // Strip comments and normalize whitespace for loose equality
+        const cleaned = source
+            .replace(/\/\/.*$/gm, '')
+            .replace(/\/\*[\s\S]*?\*\//g, '')
+            .replace(/\s+/g, ' ')
+            .trim();
+        return this._hash(cleaned);
+    }
+    
+    _hash(str) {
+        return crypto.createHash('sha256').update(str).digest('hex').substring(0, 16);
+    }
+    
+    _log(level, message) {
+        if (this.logger && typeof this.logger.log === 'function') {
+            this.logger.log(level, `[Rules] ${message}`);
+        } else {
+            console.log(`[Rules] ${message}`);
+        }
+    }
+}
+
+module.exports = { RulesRegistry };

package/functions/computation-system-v2/framework/core/RunAnalyzer.js

@@ -0,0 +1,163 @@
+/**
+ * @fileoverview Run Analyzer
+ * * Pure logic component that determines which computations need to run.
+ * Decouples decision-making from execution and storage.
+ */
+
+class RunAnalyzer {
+    constructor(manifest, dataFetcher, logger = null) {
+        this.manifest = manifest;
+        this.dataFetcher = dataFetcher;
+        this.logger = logger || console;
+        this.manifestMap = new Map(manifest.map(e => [e.name, e]));
+    }
+
+    /**
+     * Analyze the system state to determine runnable computations.
+     * @param {string} dateStr - Target date
+     * @param {Map} dailyStatus - Current day's status map
+     * @param {Map} prevDayStatus - Previous day's status map
+     * @returns {Promise<Object>} The Analysis Report
+     */
+    async analyze(dateStr, dailyStatus, prevDayStatus) {
+        const report = {
+            runnable: [],
+            skipped: [],
+            blocked: [],
+            impossible: [],
+            reRuns: []
+        };
+
+        const isToday = dateStr === new Date().toISOString().slice(0, 10);
+
+        for (const entry of this.manifest) {
+            const decision = await this._evaluateEntry(
+                entry, 
+                dateStr, 
+                isToday, 
+                dailyStatus, 
+                prevDayStatus
+            );
+
+            report[decision.type].push(decision.payload);
+        }
+        
+        // Merge reRuns into runnable for backward compatibility if needed by consumers
+        // But keeping them distinct in the report is better for logging
+        return report;
+    }
+
+    async _evaluateEntry(entry, dateStr, isToday, dailyStatus, prevDayStatus) {
+        const { name, requires, dependencies, isHistorical, isTest, schedule } = entry;
+        const stored = dailyStatus.get(name);
+
+        // 1. Schedule Check
+        if (!this._shouldRunOnDate(dateStr, schedule)) {
+            return { type: 'skipped', payload: { name, reason: `Not scheduled (${schedule})` } };
+        }
+
+        // 2. Data Availability Check
+        // Note: This is the only async IO part (calls DataFetcher)
+        const availability = await this.dataFetcher.checkAvailability(requires, dateStr);
+        if (!availability.canRun) {
+            if (!isToday) {
+                return { type: 'impossible', payload: { name, reason: `Missing data: ${availability.missing.join(', ')}` } };
+            } else {
+                return { type: 'blocked', payload: { name, reason: `Waiting for data: ${availability.missing.join(', ')}` } };
+            }
+        }
+
+        // 3. Force Test Re-run
+        if (isTest && isToday) {
+            return { 
+                type: 'reRuns', 
+                payload: { 
+                    name, pass: entry.pass, reason: 'Test computation (always runs today)',
+                    oldHash: stored?.hash, newHash: entry.hash
+                }
+            };
+        }
+
+        // 4. Code Change Detection
+        let needsRun = false;
+        let runReason = '';
+
+        if (!stored) {
+            needsRun = true;
+            runReason = 'New computation';
+        } else if (stored.hash !== entry.hash) {
+            needsRun = true;
+            runReason = 'Code changed';
+        }
+
+        // 5. Dependency Checks
+        const missingDeps = [];
+        let hasDataDrift = false;
+
+        for (const dep of dependencies) {
+            const depEntry = this.manifestMap.get(dep);
+            const depStatus = dailyStatus.get(dep);
+
+            if (!depStatus) {
+                missingDeps.push(dep);
+            } else if (depEntry && depStatus.hash !== depEntry.hash) {
+                // Dependency is present but old (needs update first)
+                missingDeps.push(dep);
+            } else if (stored?.dependencyResultHashes) {
+                // Check for data drift
+                const lastSeenResultHash = stored.dependencyResultHashes[dep];
+                if (lastSeenResultHash && depStatus.resultHash !== lastSeenResultHash) {
+                    hasDataDrift = true;
+                    if (!needsRun) {
+                        needsRun = true;
+                        runReason = `Dependency data changed: ${dep}`;
+                    }
+                }
+            }
+        }
+
+        if (missingDeps.length > 0) {
+            return { type: 'blocked', payload: { name, reason: `Waiting for: ${missingDeps.join(', ')}` } };
+        }
+
+        // 6. Historical Check
+        if (isHistorical && !needsRun) {
+            const prevStatus = prevDayStatus.get(name);
+            if (!prevStatus) {
+                needsRun = true;
+                runReason = 'Historical: no previous day result';
+            } else if (prevStatus.hash !== entry.hash) {
+                needsRun = true;
+                runReason = 'Historical: code version discontinuity';
+            }
+        }
+
+        // 7. Final Decision
+        if (needsRun) {
+            const type = stored ? 'reRuns' : 'runnable';
+            return {
+                type,
+                payload: {
+                    name, pass: entry.pass, reason: runReason,
+                    oldHash: stored?.hash, newHash: entry.hash
+                }
+            };
+        }
+
+        return { type: 'skipped', payload: { name, reason: 'Already up to date' } };
+    }
+
+    _shouldRunOnDate(dateStr, schedule) {
+        if (!schedule || schedule === 'daily') return true;
+        
+        const date = new Date(dateStr + 'T00:00:00Z');
+        const dayOfWeek = date.getUTCDay(); // 0=Sun
+        const dayOfMonth = date.getUTCDate();
+
+        if (schedule === 'weekly') return dayOfWeek === 0;
+        if (schedule === 'monthly') return dayOfMonth === 1;
+        return true;
+    }
+}
+
+module.exports = { RunAnalyzer };

package/functions/computation-system-v2/framework/cost/CostTracker.js

@@ -0,0 +1,154 @@
+/**
+ * @fileoverview Cost Attribution Tracker for BigQuery
+ * * Tracks query costs per computation by monitoring bytes processed.
+ * * Manages its own storage table in BigQuery.
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+
+class CostTracker {
+    /**
+     * @param {Object} config - System configuration
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger || console;
+        
+        this.bigquery = new BigQuery({
+            projectId: config.bigquery.projectId,
+            location: config.bigquery.location || 'US'
+        });
+        
+        this.datasetId = config.bigquery.dataset;
+        this.tableName = 'computation_costs';
+        this._tableChecked = false;
+    }
+
+    /**
+     * Record the cost of a computation execution.
+     * @param {string} computationName 
+     * @param {string} dateStr - The execution business date
+     * @param {number} bytesProcessed - Total bytes processed by BigQuery
+     */
+    async trackCost(computationName, dateStr, bytesProcessed) {
+        // If no bytes processed (cached or no-op), don't record
+        if (!bytesProcessed || bytesProcessed <= 0) return;
+
+        try {
+            await this._ensureTable();
+
+            // Pricing: ~$5.00 per TB
+            const COST_PER_TB = 5.0;
+            const BYTES_PER_TB = 1099511627776; // 1024^4
+            const estimatedCost = (bytesProcessed / BYTES_PER_TB) * COST_PER_TB;
+
+            const row = {
+                date: dateStr,
+                computation_name: computationName,
+                bytes_processed: bytesProcessed,
+                estimated_cost_usd: parseFloat(estimatedCost.toFixed(6)), // Keep reasonable precision
+                timestamp: new Date().toISOString()
+            };
+
+            await this.bigquery
+                .dataset(this.datasetId)
+                .table(this.tableName)
+                .insert([row]);
+
+            this._log('DEBUG', `Recorded cost for ${computationName}: $${row.estimated_cost_usd} (${bytesProcessed} bytes)`);
+
+        } catch (e) {
+            // Swallow errors to prevent blocking the main execution pipeline
+            this._log('ERROR', `Failed to track cost for ${computationName}: ${e.message}`);
+        }
+    }
+    
+    /**
+     * Generate a cost report for a date range.
+     * @param {string} startDate - YYYY-MM-DD
+     * @param {string} endDate - YYYY-MM-DD
+     */
+    async getCostReport(startDate, endDate) {
+        try {
+            const query = `
+                SELECT 
+                    computation_name, 
+                    SUM(bytes_processed) as total_bytes, 
+                    SUM(estimated_cost_usd) as total_cost,
+                    COUNT(*) as execution_count,
+                    AVG(estimated_cost_usd) as avg_cost_per_run
+                FROM \`${this.config.bigquery.projectId}.${this.datasetId}.${this.tableName}\`
+                WHERE date BETWEEN @startDate AND @endDate
+                GROUP BY computation_name
+                ORDER BY total_cost DESC
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { startDate, endDate },
+                location: this.config.bigquery.location
+            });
+
+            const totalCost = rows.reduce((sum, r) => sum + (r.total_cost || 0), 0);
+            
+            return {
+                period: { start: startDate, end: endDate },
+                totalCost,
+                breakdown: rows.map(r => ({
+                    computation: r.computation_name,
+                    cost: r.total_cost,
+                    runs: r.execution_count,
+                    avgCost: r.avg_cost_per_run,
+                    bytes: r.total_bytes
+                }))
+            };
+
+        } catch (e) {
+            this._log('ERROR', `Failed to generate cost report: ${e.message}`);
+            return { error: e.message };
+        }
+    }
+
+    async _ensureTable() {
+        if (this._tableChecked) return;
+
+        const dataset = this.bigquery.dataset(this.datasetId);
+        const table = dataset.table(this.tableName);
+
+        const [exists] = await table.exists();
+        
+        if (!exists) {
+            this._log('INFO', `Creating cost tracking table: ${this.tableName}`);
+            await dataset.createTable(this.tableName, {
+                schema: [
+                    { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+                    { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
+                    { name: 'bytes_processed', type: 'INTEGER', mode: 'REQUIRED' },
+                    { name: 'estimated_cost_usd', type: 'FLOAT', mode: 'REQUIRED' },
+                    { name: 'timestamp', type: 'TIMESTAMP', mode: 'REQUIRED' }
+                ],
+                timePartitioning: {
+                    type: 'DAY',
+                    field: 'date'
+                },
+                clustering: {
+                    fields: ['computation_name']
+                }
+            });
+        }
+        
+        this._tableChecked = true;
+    }
+
+    _log(level, message) {
+        const prefix = '[CostTracker]';
+        if (this.logger && typeof this.logger.log === 'function') {
+            this.logger.log(level, `${prefix} ${message}`);
+        } else {
+            console.log(`${level}: ${prefix} ${message}`);
+        }
+    }
+}
+
+module.exports = { CostTracker };