bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/logger/logger.js

@@ -1,297 +0,0 @@
-/**
- * @fileoverview Structured Logging System for Computation Engine
- * UPDATED: Added support for Google Cloud Trace Context injection.
- */
-
-const crypto = require('crypto');
-
-const LOG_LEVELS = {
-    TRACE: 0, DEBUG: 1, INFO: 2, WARN: 3, ERROR: 4, FATAL: 5
-};
-
-const PROCESS_TYPES = {
-    MANIFEST: 'manifest',
-    ORCHESTRATOR: 'orchestrator',
-    EXECUTOR: 'executor',
-    STORAGE: 'storage',
-    WORKER: 'worker',
-    ANALYSIS: 'analysis',
-    DISPATCH: 'dispatch'
-};
-
-function generateProcessId(type, identifier, date = '') {
-    const input = `${type}|${identifier}|${date}`;
-    return crypto.createHash('sha256').update(input).digest('hex').substring(0, 16);
-}
-
-function formatLogEntry(entry) {
-    return JSON.stringify({
-        timestamp: entry.timestamp,
-        level: entry.level, // Google uses 'severity', mapped below
-        severity: entry.severity, 
-        processType: entry.processType,
-        processId: entry.processId,
-        computationName: entry.computationName,
-        date: entry.date,
-        message: entry.message,
-        context: entry.context,
-        metadata: entry.metadata,
-        stats: entry.stats,
-        storage: entry.storage,
-        details: entry.details,
-        // Spread the global Trace fields here
-        ...entry.globalMetadata
-    });
-}
-
-class StructuredLogger {
-    constructor(config = {}) {
-        this.config = {
-            minLevel: config.minLevel || LOG_LEVELS.INFO,
-            enableConsole: config.enableConsole !== false,
-            enableStructured: config.enableStructured !== false,
-            includeStackTrace: config.includeStackTrace !== false,
-            ...config
-        };
-        this.activeProcesses = new Map();
-        
-        // [NEW] Store global metadata (like Trace IDs) for this logger instance
-        this.globalMetadata = config.globalMetadata || {}; 
-    }
-
-    startProcess(processType, computationName = null, date = null) {
-        const processId = (computationName || date) 
-            ? generateProcessId(processType, computationName || 'general', date)
-            : crypto.randomBytes(8).toString('hex');
-
-        const processLogger = new ProcessLogger(this, processType, processId, computationName, date);
-        this.activeProcesses.set(processId, {
-            type: processType,
-            computationName,
-            date,
-            startTime: Date.now(),
-            logger: processLogger
-        });
-        return processLogger;
-    }
-
-    endProcess(processId) {
-        const process = this.activeProcesses.get(processId);
-        if (process) {
-            const duration = Date.now() - process.startTime;
-            this.activeProcesses.delete(processId);
-            return duration;
-        }
-        return null;
-    }
-
-    /**
-     * The Main Date Analysis Logger (UPDATED)
-     * Handles 'impossible' category for permanent failures.
-     */
-    logDateAnalysis(dateStr, analysisReport) {
-        const { 
-            runnable = [], 
-            blocked = [], 
-            impossible = [], // New Category
-            reRuns = [], 
-            failedDependency = [], 
-            skipped = []           
-        } = analysisReport;
-        
-        // 1. Structured Output
-        if (this.config.enableStructured) {
-            console.log(JSON.stringify({
-                timestamp: new Date().toISOString(),
-                level: 'INFO',
-                severity: 'INFO',
-                processType: PROCESS_TYPES.ANALYSIS,
-                date: dateStr,
-                message: `Date Analysis for ${dateStr}`,
-                stats: {
-                    runnable: runnable.length,
-                    blocked: blocked.length,
-                    impossible: impossible.length,
-                    reRuns: reRuns.length,
-                    failedDependency: failedDependency.length,
-                    skipped: skipped.length
-                },
-                details: analysisReport,
-                ...this.globalMetadata
-            }));
-        }
-
-        // 2. Human Readable Output
-        if (this.config.enableConsole) {
-            const symbols = { 
-                info: 'ℹ️', warn: '⚠️', check: '✅', block: '⛔', 
-                cycle: '🔄', skip: '⏭️', dead: '💀' 
-            };
-            
-            console.log(`\n🔍 === DATE ANALYSIS REPORT: ${dateStr} ===`);
-            
-            if (reRuns.length) {
-                console.log(`\n${symbols.cycle} [HASH MISMATCH / RE-RUNS]`);
-                reRuns.forEach(item => {
-                    console.log(`   • ${item.name}: Hash changed. (Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...)`);
-                });
-            }
-
-            if (failedDependency.length) {
-                console.log(`\n${symbols.block} [FAILED DEPENDENCIES] (Upstream failure)`);
-                failedDependency.forEach(item => {
-                    console.log(`   • ${item.name}: Missing ${item.missing ? item.missing.join(', ') : 'dependencies'}`);
-                });
-            }
-
-            // NEW: Impossible items
-            if (impossible.length) {
-                console.log(`\n${symbols.dead} [IMPOSSIBLE] (Permanent Failure - Will not retry)`);
-                impossible.forEach(item => {
-                    console.log(`   • ${item.name}: ${item.reason}`);
-                });
-            }
-
-            if (runnable.length) {
-                console.log(`\n${symbols.check} [READY TO RUN]`);
-                runnable.forEach(item => console.log(`   • ${item.name}`));
-            }
-
-            if (blocked.length) {
-                console.log(`\n${symbols.warn} [BLOCKED] (Waiting for data - Retriable)`);
-                blocked.forEach(item => {
-                    console.log(`   • ${item.name}: ${item.reason}`);
-                });
-            }
-
-            if (skipped.length) {
-                if (skipped.length > 10) {
-                     console.log(`\n${symbols.skip} [SKIPPED] (${skipped.length} calculations up-to-date)`);
-                } else {
-                    console.log(`\n${symbols.skip} [SKIPPED]`);
-                    skipped.forEach(item => console.log(`   • ${item.name}`));
-                }
-            }
-
-            console.log(`\n=============================================\n`);
-        }
-    }
-
-    logStorage(processId, calcName, date, path, sizeBytes, isSharded) {
-        this.log(LOG_LEVELS.INFO, `Results stored for ${calcName}`, {
-            storage: { 
-                path, 
-                sizeBytes, 
-                isSharded, 
-                sizeMB: (sizeBytes / 1024 / 1024).toFixed(2) 
-            }
-        }, PROCESS_TYPES.STORAGE, processId, calcName, date);
-    }
-
-    log(level, message, context = {}, processType = null, processId = null, computationName = null, date = null) {
-        const numericLevel = typeof level === 'string' ? LOG_LEVELS[level] : level;
-        if (numericLevel < this.config.minLevel) return;
-
-        let finalContext = context;
-        let finalMetadata = {};
-        let finalStats = undefined;
-        let finalStorage = undefined;
-
-        if (context && (context.processId || context.storage || context.stats)) {
-            if (context.processId) processId = context.processId;
-            if (context.processType) processType = context.processType;
-            if (context.computationName) computationName = context.computationName;
-            if (context.date) date = context.date;
-            if (context.stats) finalStats = context.stats;
-            if (context.storage) finalStorage = context.storage;
-            finalContext = { ...context };
-            delete finalContext.processId; delete finalContext.processType;
-            delete finalContext.computationName; delete finalContext.date;
-            delete finalContext.stats; delete finalContext.storage;
-        }
-
-        // Map Level to Google Severity
-        let severity = 'INFO';
-        if (numericLevel === LOG_LEVELS.WARN) severity = 'WARNING';
-        else if (numericLevel === LOG_LEVELS.ERROR) severity = 'ERROR';
-        else if (numericLevel === LOG_LEVELS.FATAL) severity = 'CRITICAL';
-        else if (numericLevel === LOG_LEVELS.DEBUG) severity = 'DEBUG';
-
-        const entry = {
-            timestamp: new Date().toISOString(),
-            level: Object.keys(LOG_LEVELS).find(k => LOG_LEVELS[k] === numericLevel) || 'INFO',
-            severity: severity,
-            processType,
-            processId,
-            computationName,
-            date,
-            message,
-            context: typeof finalContext === 'string' ? { error: finalContext } : finalContext,
-            metadata: finalMetadata,
-            stats: finalStats,
-            storage: finalStorage,
-            globalMetadata: this.globalMetadata // [NEW] Pass trace context
-        };
-
-        if (numericLevel >= LOG_LEVELS.ERROR && this.config.includeStackTrace && finalContext.stack) {
-            entry.metadata.stackTrace = finalContext.stack;
-        }
-
-        if (this.config.enableConsole) {
-            this._consoleLog(entry);
-        }
-
-        if (this.config.enableStructured) {
-            // [CRITICAL] This is what Google Cloud Logging picks up
-            console.log(formatLogEntry(entry));
-        }
-    }
-
-    _consoleLog(entry) {
-        const symbols = { TRACE: '🔍', DEBUG: '🐛', INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', FATAL: '💀' };
-        const colors = {
-            TRACE: '\x1b[90m', DEBUG: '\x1b[36m', INFO: '\x1b[32m',
-            WARN: '\x1b[33m', ERROR: '\x1b[31m', FATAL: '\x1b[35m'
-        };
-        const reset = '\x1b[0m';
-        const color = colors[entry.level] || '';
-        const symbol = symbols[entry.level] || 'ℹ️';
-
-        let output = `${color}${symbol} [${entry.level}]${reset}`;
-        if (entry.processType) output += ` [${entry.processType}]`;
-        if (entry.computationName) output += ` [${entry.computationName}]`;
-        
-        output += ` ${entry.message}`;
-
-        console.log(output);
-        if (entry.context && Object.keys(entry.context).length > 0) {
-            console.log(`  Context:`, entry.context);
-        }
-    }
-}
-
-class ProcessLogger {
-    constructor(parent, processType, processId, computationName, date) {
-        this.parent = parent;
-        this.processType = processType;
-        this.processId = processId;
-        this.computationName = computationName;
-        this.date = date;
-        this.startTime = Date.now();
-        this.metrics = { operations: 0, errors: 0, warnings: 0 };
-    }
-    log(level, message, context = {}) {
-        this.metrics.operations++;
-        this.parent.log(level, message, context, this.processType, this.processId, this.computationName, this.date);
-    }
-    complete(success = true, finalMessage = null) {
-        const duration = Date.now() - this.startTime;
-        this.log(success ? LOG_LEVELS.INFO : LOG_LEVELS.ERROR, 
-            finalMessage || (success ? 'Process completed' : 'Process failed'), 
-            { stats: { durationMs: duration, ...this.metrics, success } });
-        this.parent.endProcess(this.processId);
-        return duration;
-    }
-}
-
-module.exports = { StructuredLogger, ProcessLogger, LOG_LEVELS, PROCESS_TYPES, generateProcessId };

package/functions/computation-system/persistence/ContractValidator.js

@@ -1,81 +0,0 @@
-/**
- * @fileoverview Enforces the contracts discovered by the offline tool.
- * Designed to be permissive with volatility ("Anomalies") but strict with logic ("Violations").
- */
-class ContractValidator {
-
-    /**
-     * @param {Object} result - The production output (single item or batch).
-     * @param {Object} contract - The loaded contract JSON.
-     * @returns {Object} { valid: boolean, reason: string }
-     */
-    static validate(result, contract) {
-        if (!result || !contract) return { valid: true };
-
-        // Handle Batches (StandardExecutor produces map of User -> Result)
-        const items = Object.values(result);
-        if (items.length === 0) return { valid: true };
-
-        // We check a sample to save CPU, or check all if critical
-        // For "Cohort" logic, we usually check all because one bad apple spoils the average.
-        for (const item of items) {
-            const check = this._validateItem(item, contract);
-            if (!check.valid) return check;
-        }
-
-        return { valid: true };
-    }
-
-    static _validateItem(item, contract) {
-        // 1. Structure Check
-        if (contract.requiredKeys) {
-            for (const key of contract.requiredKeys) {
-                if (item[key] === undefined) {
-                    return { valid: false, reason: `Schema Violation: Missing key '${key}'` };
-                }
-            }
-        }
-
-        // 2. Numeric Physics Check (Hard Bounds)
-        if (contract.numericBounds) {
-            for (const [key, bounds] of Object.entries(contract.numericBounds)) {
-                const val = item[key];
-                if (typeof val !== 'number') continue;
-
-                if (val < bounds.min) {
-                    return { valid: false, reason: `Physics Violation: ${key} (${val}) is below hard limit ${bounds.min}` };
-                }
-                if (val > bounds.max) {
-                    return { valid: false, reason: `Physics Violation: ${key} (${val}) is above hard limit ${bounds.max}` };
-                }
-            }
-        }
-
-        // 3. Statistical Sanity Check (Soft Bounds)
-        // We generally DO NOT BLOCK on this for financial data, unless it's egregious.
-        // We block if it's "Mathematically Impossible" based on the distribution.
-        if (contract.distributions) {
-            for (const [key, dist] of Object.entries(contract.distributions)) {
-                const val = item[key];
-                if (typeof val !== 'number') continue;
-
-                const diff = Math.abs(val - dist.mean);
-                const sigmas = diff / dist.stdDev;
-
-                // 15 Sigma is our "Ridiculousness Threshold".
-                // Even crypto doesn't move 15 standard deviations in one calculation step 
-                // unless the data is corrupt (e.g. integer overflow, or bad scraping).
-                if (sigmas > 15 && diff > 1.0) { // Ensure diff is material
-                    return { 
-                        valid: false, 
-                        reason: `Statistical Impossibility: ${key} is ${sigmas.toFixed(1)} sigmas from mean. Value: ${val}, Mean: ${dist.mean}` 
-                    };
-                }
-            }
-        }
-
-        return { valid: true };
-    }
-}
-
-module.exports = ContractValidator;

package/functions/computation-system/persistence/FirestoreUtils.js

@@ -1,56 +0,0 @@
-/**
- * @fileoverview Low-level Firestore interactions.
- */
-const { withRetry } = require('../utils/utils.js'); 
-
-async function commitBatchInChunks(config, deps, writes, operationName) {
-    const { db, logger, calculationUtils } = deps;
-    const retryFn = calculationUtils ? calculationUtils.withRetry : (fn) => fn();
-
-    if (!writes || !writes.length) {
-        logger.log('WARN', `[${operationName}] No writes to commit.`);
-        return;
-    }
-
-    const MAX_BATCH_OPS   = 300;
-    const MAX_BATCH_BYTES = 9 * 1024 * 1024; 
-
-    let currentBatch    = db.batch();
-    let currentOpsCount = 0;
-    let currentBytesEst = 0;
-    let batchIndex      = 1;
-
-    const commitAndReset = async () => {
-        if (currentOpsCount > 0) {
-            try {
-                await retryFn(
-                    () => currentBatch.commit(),
-                    `${operationName} (Chunk ${batchIndex})`
-                );
-                logger.log('INFO', `[${operationName}] Committed chunk ${batchIndex} (${currentOpsCount} ops, ~${(currentBytesEst / 1024 / 1024).toFixed(2)} MB).`);
-                batchIndex++;
-            } catch (err) {
-                logger.log('ERROR', `[${operationName}] Failed to commit chunk ${batchIndex}. Size: ${(currentBytesEst / 1024 / 1024).toFixed(2)} MB.`, { error: err.message });
-                throw err;
-            }
-        }
-        currentBatch    = db.batch();
-        currentOpsCount = 0;
-        currentBytesEst = 0;
-    };
-
-    for (const write of writes) {
-        let docSize = 100;
-        try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
-        if (docSize > 900 * 1024) { logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB).`); }
-        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) { await commitAndReset(); }
-        const options = write.options || { merge: true };
-        currentBatch.set(write.ref, write.data, options);
-        currentOpsCount++;
-        currentBytesEst += docSize;
-    }
-
-    await commitAndReset();
-}
-
-module.exports = { commitBatchInChunks };