bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/tools/FinalSweepReporter.js

@@ -1,322 +0,0 @@
-/**
- * FILENAME: computation-system/tools/FinalSweepReporter.js
- * PURPOSE: Comprehensive forensic tool to diagnose why computations failed.
- * EXECUTION: Triggered by Dispatcher (via Workflow) after execution passes complete.
- */
-
-const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
-const { fetchComputationStatus } = require('../persistence/StatusRepository');
-const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
-const { analyzeDateExecution } = require('../WorkflowOrchestrator');
-
-const REPORT_COLLECTION = 'Final_sweep_check_report';
-const LEDGER_BASE = 'computation_audit_ledger';
-const STALE_THRESHOLD_MS = 1000 * 60 * 20; // 20 minutes
-
-class FinalSweepReporter {
-    constructor(config, dependencies) {
-        this.config = config;
-        this.db = dependencies.db;
-        this.logger = dependencies.logger;
-    }
-
-    /**
-     * Main Entry Point
-     * @param {string} dateStr - Target date (YYYY-MM-DD)
-     * @param {string} passId - The pass to analyze ("1", "2"...)
-     * @param {Array} manifest - Full computation manifest
-     */
-    async runSweep(dateStr, passId, manifest) {
-        this.logger.log('INFO', `[FinalSweep] 🕵️ Starting forensic analysis for ${dateStr} (Pass ${passId})`);
-
-        // 1. Filter Manifest for Target Pass
-        const passCalcs = manifest.filter(c => String(c.pass) === passId);
-        if (passCalcs.length === 0) {
-            this.logger.log('WARN', `[FinalSweep] No calculations found for Pass ${passId}.`);
-            return { issuesCount: 0 };
-        }
-
-        const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
-
-        // 2. Fetch Global State
-        const [dailyStatus, availability] = await Promise.all([
-            fetchComputationStatus(dateStr, this.config, { db: this.db }),
-            checkRootDataAvailability(dateStr, this.config, { db: this.db, logger: this.logger }, DEFINITIVE_EARLIEST_DATES)
-        ]);
-
-        // 3. Fetch Yesterday (if needed for historical checks)
-        let prevDailyStatus = null;
-        if (passCalcs.some(c => c.isHistorical)) {
-            const prevDate = new Date(dateStr + 'T00:00:00Z');
-            prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-            if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
-                prevDailyStatus = await fetchComputationStatus(prevDate.toISOString().slice(0, 10), this.config, { db: this.db });
-            }
-        }
-
-        const rootDataStatus = availability ? availability.status : {};
-
-        // 4. Run Logic Analysis
-        // We use the Orchestrator's logic to see who SHOULD have run but didn't
-        const analysis = analyzeDateExecution(
-            dateStr, 
-            passCalcs, 
-            rootDataStatus, 
-            dailyStatus, 
-            manifestMap, 
-            prevDailyStatus
-        );
-
-        // Collect all potential issues
-        const problematicItems = [
-            ...analysis.impossible.map(x => ({ ...x, category: 'IMPOSSIBLE' })),
-            ...analysis.blocked.map(x => ({ ...x, category: 'BLOCKED' })),
-            ...analysis.failedDependency.map(x => ({ ...x, category: 'DEPENDENCY_FAILURE' })),
-            // If it's still 'RUNNABLE' after the final pass, it means it was skipped/missed/failed silently
-            ...analysis.runnable.map(x => ({ ...x, category: 'STUCK_RUNNABLE' })), 
-            ...analysis.reRuns.map(x => ({ ...x, category: 'STUCK_RERUN' }))
-        ];
-
-        // 5. Deep Investigation & Reporting
-        let issuesCount = 0;
-        const batch = this.db.batch();
-        let opCount = 0;
-
-        // A. Analyze Standard Problems
-        for (const item of problematicItems) {
-            const forensics = await this.investigateComputation(
-                item.name, dateStr, passId, item.category, manifestMap, 
-                dailyStatus, prevDailyStatus, rootDataStatus
-            );
-            
-            if (forensics) {
-                this._queueReportWrite(batch, dateStr, item.name, forensics);
-                opCount++;
-                issuesCount++;
-            }
-        }
-
-        // B. Analyze "Ghost" Completions (Marked Complete but Empty)
-        for (const calc of passCalcs) {
-            const name = normalizeName(calc.name);
-            const status = dailyStatus[name];
-            
-            if (status && status.hash === calc.hash) {
-                const hasData = await this.verifyDataExists(name, calc.category, dateStr);
-                if (!hasData) {
-                    const forensics = {
-                        computation: name,
-                        date: dateStr,
-                        category: 'GHOST_COMPLETION',
-                        rootCause: 'DATA_WRITE_FAILURE',
-                        reason: 'Status index says COMPLETE, but storage document is missing.',
-                        severity: 'HIGH',
-                        recommendations: ['Force Re-run', 'Check Write Permissions']
-                    };
-                    this._queueReportWrite(batch, dateStr, name, forensics);
-                    opCount++;
-                    issuesCount++;
-                }
-            }
-        }
-
-        if (opCount > 0) await batch.commit();
-        
-        this.logger.log('SUCCESS', `[FinalSweep] Pass ${passId}: Generated ${issuesCount} forensic reports.`);
-        return { issuesCount };
-    }
-
-    _queueReportWrite(batch, dateStr, calcName, forensics) {
-        // Path: Final_sweep_check_report/YYYY-MM-DD/Computation_Name/document
-        const reportRef = this.db.collection(REPORT_COLLECTION)
-            .doc(dateStr)
-            .collection(normalizeName(calcName))
-            .doc('document');
-
-        batch.set(reportRef, {
-            ...forensics,
-            generatedAt: new Date().toISOString()
-        });
-    }
-
-    /**
-     * Deep dive investigation logic
-     */
-    async investigateComputation(calcName, dateStr, passId, category, manifestMap, dailyStatus, prevDailyStatus, rootDataStatus) {
-        const manifest = manifestMap.get(calcName);
-        if (!manifest) return null;
-
-        const forensics = {
-            computation: calcName,
-            pass: passId,
-            date: dateStr,
-            category: category,
-            rootCause: null,
-            reason: null,
-            ledgerState: null,
-            recommendations: [],
-            severity: 'MEDIUM'
-        };
-
-        // 1. CHECK AUDIT LEDGER
-        const ledgerPath = `${LEDGER_BASE}/${dateStr}/passes/${passId}/tasks/${normalizeName(calcName)}`;
-        const ledgerSnap = await this.db.doc(ledgerPath).get();
-
-        if (ledgerSnap.exists) {
-            const data = ledgerSnap.data();
-            // Filter out undefined values to prevent Firestore errors
-            const ledgerState = {
-                status: data.status,
-                workerId: data.workerId
-            };
-            // Only include error if it's defined
-            if (data.error !== undefined && data.error !== null) {
-                ledgerState.error = data.error;
-            }
-            forensics.ledgerState = ledgerState;
-
-            if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-                // CRITICAL FIX: Handle Firestore Timestamp objects correctly
-                const getMillis = (field) => {
-                    if (!field) return 0;
-                    if (field.toDate && typeof field.toDate === 'function') {
-                        return field.toDate().getTime();
-                    }
-                    const date = new Date(field);
-                    return isNaN(date.getTime()) ? 0 : date.getTime();
-                };
-                const lastHb = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
-                if (Date.now() - lastHb > STALE_THRESHOLD_MS) {
-                    forensics.rootCause = 'ZOMBIE_PROCESS';
-                    forensics.reason = `Worker ${data.workerId} stopped heartbeating. Likely crashed/timeout.`;
-                    forensics.severity = 'CRITICAL';
-                } else {
-                    forensics.rootCause = 'STILL_RUNNING'; // Rare if sweep runs after everything
-                }
-            } else if (data.status === 'FAILED') {
-                if (data.resourceTier === 'high-mem') {
-                    forensics.rootCause = 'CRASH_HIGH_MEM';
-                    forensics.reason = 'Failed even on High-Memory tier. Code optimization required.';
-                    forensics.severity = 'CRITICAL';
-                } else {
-                    forensics.rootCause = 'FAILED_STANDARD';
-                    forensics.reason = data.error?.message || 'Unknown Error';
-                }
-                
-                if (data.error?.stage === 'OOM' || (data.error?.message || '').includes('memory')) {
-                    forensics.rootCause = 'OUT_OF_MEMORY';
-                }
-            }
-        } else {
-            forensics.ledgerState = 'NEVER_DISPATCHED';
-            if (category === 'STUCK_RUNNABLE') {
-                forensics.rootCause = 'DISPATCHER_MISS';
-                forensics.reason = 'Logic says runnable, but Dispatcher never queued it.';
-                forensics.severity = 'HIGH';
-            }
-        }
-
-        // 2. ROOT DATA ANALYSIS
-        if (category === 'IMPOSSIBLE') {
-            const missing = [];
-            const userType = manifest.userType || 'all';
-            const deps = manifest.rootDataDependencies || [];
-
-            if (deps.includes('portfolio')) {
-                if (userType === 'speculator' && !rootDataStatus.speculatorPortfolio) missing.push('speculatorPortfolio');
-                else if (userType === 'normal' && !rootDataStatus.normalPortfolio) missing.push('normalPortfolio');
-                else if (userType === 'all' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-            }
-            if (deps.includes('history')) {
-                if (userType === 'speculator' && !rootDataStatus.speculatorHistory) missing.push('speculatorHistory');
-                else if (userType === 'normal' && !rootDataStatus.normalHistory) missing.push('normalHistory');
-                else if (userType === 'all' && !rootDataStatus.hasHistory) missing.push('history');
-            }
-            if (deps.includes('price') && !rootDataStatus.hasPrices) missing.push('price');
-            if (deps.includes('insights') && !rootDataStatus.hasInsights) missing.push('insights');
-
-            forensics.rootCause = 'MISSING_ROOT_DATA';
-            forensics.reason = `Missing: ${missing.join(', ')}`;
-            forensics.severity = 'LOW'; // Expected behavior
-        }
-
-        // 3. DEPENDENCY ANALYSIS
-        if (category === 'DEPENDENCY_FAILURE' || category === 'BLOCKED') {
-            const chain = await this.traceDependencyChain(calcName, manifestMap, dailyStatus, prevDailyStatus, manifest.isHistorical);
-            if (chain.length > 0) {
-                const root = chain[chain.length - 1];
-                forensics.rootCause = 'UPSTREAM_FAILURE';
-                forensics.reason = `Blocked by ${root.name} (${root.reason})`;
-                forensics.chain = chain;
-            }
-        }
-
-        return forensics;
-    }
-
-    async traceDependencyChain(calcName, manifestMap, dailyStatus, prevDailyStatus, isHistorical) {
-        const chain = [];
-        const visited = new Set();
-
-        const trace = (name, isPrev) => {
-            if (visited.has(name)) return;
-            visited.add(name);
-            
-            const m = manifestMap.get(name);
-            if (!m) return; // Unknown calc
-            
-            const status = isPrev ? (prevDailyStatus?.[name]) : (dailyStatus?.[name]);
-            
-            if (!status) {
-                chain.push({ name, reason: 'MISSING_STATUS', isPrev });
-                return;
-            }
-            if (String(status.hash).startsWith('IMPOSSIBLE')) {
-                chain.push({ name, reason: 'IMPOSSIBLE', isPrev });
-                return;
-            }
-            if (m.hash !== status.hash) {
-                chain.push({ name, reason: 'VERSION_MISMATCH', isPrev });
-                return;
-            }
-
-            if (m.dependencies) {
-                for (const dep of m.dependencies) trace(normalizeName(dep), false);
-            }
-        };
-
-        const m = manifestMap.get(calcName);
-        if (m) {
-            if (m.dependencies) m.dependencies.forEach(d => trace(normalizeName(d), false));
-            if (isHistorical) trace(calcName, true);
-        }
-        return chain;
-    }
-
-    async verifyDataExists(calcName, category, dateStr) {
-        try {
-            const docRef = this.db.collection(this.config.resultsCollection)
-                .doc(dateStr)
-                .collection(this.config.resultsSubcollection)
-                .doc(category)
-                .collection(this.config.computationsSubcollection)
-                .doc(calcName);
-            
-            const snap = await docRef.get();
-            if (!snap.exists) return false;
-            
-            const data = snap.data();
-            if (data._completed === true) return true;
-            return false;
-        } catch (e) {
-            return false;
-        }
-    }
-}
-
-async function runFinalSweepCheck(config, dependencies, dateStr, pass, manifest) {
-    const reporter = new FinalSweepReporter(config, dependencies);
-    return await reporter.runSweep(dateStr, pass, manifest);
-}
-
-module.exports = { runFinalSweepCheck };

package/functions/computation-system/topology/HashManager.js

@@ -1,55 +0,0 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/topology/HashManager.js
- * }
- */
-/**
- * @fileoverview Manages code versioning and legacy mappings.
- * UPDATED: Removed global infrastructure scanning. Now relies on Manual Epochs.
- */
-const crypto = require('crypto');
-
-// Legacy Keys Mapping (Ensures backward compatibility)
-const LEGACY_MAPPING = {
-    DataExtractor: 'extract',
-    HistoryExtractor: 'history',
-    MathPrimitives: 'compute',
-    Aggregators: 'aggregate',
-    Validators: 'validate',
-    SignalPrimitives: 'signals',
-    SCHEMAS: 'schemas',
-    DistributionAnalytics: 'distribution',
-    TimeSeries: 'TimeSeries',
-    priceExtractor: 'priceExtractor',
-    InsightsExtractor: 'insights',
-    UserClassifier: 'classifier',
-    Psychometrics: 'psychometrics',
-    CognitiveBiases: 'bias',
-    SkillAttribution: 'skill',
-    ExecutionAnalytics: 'execution',
-    AdaptiveAnalytics: 'adaptive'
-};
-
-/**
- * Generates a SHA-256 hash of a code string.
- * Strips comments and whitespace for loose equality.
- */
-function generateCodeHash(codeString) {
-    if (!codeString) return 'unknown';
-    let clean = codeString.replace(/\/\/.*$/gm, '');
-    clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
-    clean = clean.replace(/\s+/g, '');
-    return crypto.createHash('sha256').update(clean).digest('hex');
-}
-
-/**
- * DEPRECATED: Previously walked the file system.
- * Now returns a static string because we rely on SYSTEM_EPOCH for global versioning.
- * Kept for backward compatibility with older consumers.
- */
-function getInfrastructureHash() {
-    return 'MANUAL_EPOCH_MODE';
-}
-
-module.exports = { LEGACY_MAPPING, generateCodeHash, getInfrastructureHash };

package/functions/computation-system/topology/ManifestLoader.js

@@ -1,47 +0,0 @@
-/**
- * @fileoverview Singleton Loader for the Manifest.
- * Prevents expensive manifest rebuilding on every function invocation if not needed.
- */
-const { build } = require('../context/ManifestBuilder');
-const { StructuredLogger, PROCESS_TYPES, generateProcessId } = require('../logger/logger');
-
-// [FIX] Cache using a Map to handle different productLine combinations
-const manifestCache = new Map();
-
-function getManifest(productLines = [], calculationsDir, dependencies = {}) {
-    // Generate a unique key for this specific request configuration
-    const cacheKey = JSON.stringify(productLines ? productLines.slice().sort() : ['ALL']);
-
-    if (manifestCache.has(cacheKey)) {
-        return manifestCache.get(cacheKey);
-    }
-
-    const logger = dependencies.logger || new StructuredLogger();
-    const pid = generateProcessId(PROCESS_TYPES.MANIFEST, 'build', new Date().toISOString().slice(0,10));
-    
-    logger.log('INFO', 'Starting Manifest Build...', { processId: pid, scope: cacheKey });
-    
-    const startTime = Date.now();
-    try {
-        const manifest = build(productLines, calculationsDir);
-        
-        // Log Topology Stats
-        const passCounts = {};
-        manifest.forEach(c => { passCounts[c.pass] = (passCounts[c.pass] || 0) + 1; });
-        
-        logger.log('INFO', 'Manifest Build Success', { 
-            processId: pid, 
-            durationMs: Date.now() - startTime,
-            totalCalculations: manifest.length,
-            topology: passCounts
-        });
-
-        manifestCache.set(cacheKey, manifest);
-        return manifest;
-    } catch (e) {
-        logger.log('FATAL', 'Manifest Build Failed', { processId: pid, error: e.message });
-        throw e;
-    }
-}
-
-module.exports = { getManifest };