bulltrackers-module 1.0.346 → 1.0.348

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -11,8 +11,8 @@ const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.j
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const { runFinalSweepCheck } = require('../tools/FinalSweepReporter'); // [NEW]
 const crypto = require('crypto');
-const monConfig = require('../config/monitoring_config');
 
 const OOM_THRESHOLD_MB = 1500; // Unused
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
@@ -148,7 +148,7 @@ async function getStableDateSession(config, dependencies, passToRun, dateLimitSt
 // MAIN ENTRY POINT
 // =============================================================================
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
-    const action = reqBody.action || 'DISPATCH'; // DISPATCH, VERIFY, SWEEP
+    const action = reqBody.action || 'DISPATCH'; 
     
     if (action === 'VERIFY') {
         return handlePassVerification(config, dependencies, computationManifest, reqBody);
@@ -156,10 +156,39 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     else if (action === 'SWEEP') {
         return handleSweepDispatch(config, dependencies, computationManifest, reqBody);
     }
+    // [NEW] Handler for Final Forensics Reporting
+    else if (action === 'REPORT') {
+        return handleFinalSweepReporting(config, dependencies, computationManifest, reqBody);
+    }
     
     return handleStandardDispatch(config, dependencies, computationManifest, reqBody);
 }
 
+// =============================================================================
+// NEW: Final Sweep Reporting Handler
+// =============================================================================
+async function handleFinalSweepReporting(config, dependencies, computationManifest, reqBody) {
+    const { logger } = dependencies;
+    const passToRun = String(reqBody.pass || "1");
+    // Target date is required for detailed forensics
+    const date = reqBody.date || new Date().toISOString().slice(0, 10); 
+    
+    logger.log('INFO', `[Dispatcher] 📝 Triggering Final Sweep Forensics for Pass ${passToRun} on ${date}...`);
+
+    try {
+        const result = await runFinalSweepCheck(config, dependencies, date, passToRun, computationManifest);
+        return { 
+            status: 'COMPLETED',
+            date: date,
+            pass: passToRun,
+            issuesFound: result.issuesCount 
+        };
+    } catch (e) {
+        logger.log('ERROR', `[Dispatcher] Forensics failed: ${e.message}`);
+        return { status: 'ERROR', error: e.message };
+    }
+}
+
 // =============================================================================
 // LOGIC: Verify Pass Completion
 // =============================================================================

package/functions/computation-system/tools/FinalSweepReporter.js

@@ -0,0 +1,304 @@
+/**
+ * FILENAME: computation-system/tools/FinalSweepReporter.js
+ * PURPOSE: Comprehensive forensic tool to diagnose why computations failed.
+ * EXECUTION: Triggered by Dispatcher (via Workflow) after execution passes complete.
+ */
+
+const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
+const { fetchComputationStatus } = require('../persistence/StatusRepository');
+const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const { analyzeDateExecution } = require('../WorkflowOrchestrator');
+
+const REPORT_COLLECTION = 'Final_sweep_check_report';
+const LEDGER_BASE = 'computation_audit_ledger';
+const STALE_THRESHOLD_MS = 1000 * 60 * 20; // 20 minutes
+
+class FinalSweepReporter {
+    constructor(config, dependencies) {
+        this.config = config;
+        this.db = dependencies.db;
+        this.logger = dependencies.logger;
+    }
+
+    /**
+     * Main Entry Point
+     * @param {string} dateStr - Target date (YYYY-MM-DD)
+     * @param {string} passId - The pass to analyze ("1", "2"...)
+     * @param {Array} manifest - Full computation manifest
+     */
+    async runSweep(dateStr, passId, manifest) {
+        this.logger.log('INFO', `[FinalSweep] 🕵️ Starting forensic analysis for ${dateStr} (Pass ${passId})`);
+
+        // 1. Filter Manifest for Target Pass
+        const passCalcs = manifest.filter(c => String(c.pass) === passId);
+        if (passCalcs.length === 0) {
+            this.logger.log('WARN', `[FinalSweep] No calculations found for Pass ${passId}.`);
+            return { issuesCount: 0 };
+        }
+
+        const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
+
+        // 2. Fetch Global State
+        const [dailyStatus, availability] = await Promise.all([
+            fetchComputationStatus(dateStr, this.config, { db: this.db }),
+            checkRootDataAvailability(dateStr, this.config, { db: this.db, logger: this.logger }, DEFINITIVE_EARLIEST_DATES)
+        ]);
+
+        // 3. Fetch Yesterday (if needed for historical checks)
+        let prevDailyStatus = null;
+        if (passCalcs.some(c => c.isHistorical)) {
+            const prevDate = new Date(dateStr + 'T00:00:00Z');
+            prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
+                prevDailyStatus = await fetchComputationStatus(prevDate.toISOString().slice(0, 10), this.config, { db: this.db });
+            }
+        }
+
+        const rootDataStatus = availability ? availability.status : {};
+
+        // 4. Run Logic Analysis
+        // We use the Orchestrator's logic to see who SHOULD have run but didn't
+        const analysis = analyzeDateExecution(
+            dateStr, 
+            passCalcs, 
+            rootDataStatus, 
+            dailyStatus, 
+            manifestMap, 
+            prevDailyStatus
+        );
+
+        // Collect all potential issues
+        const problematicItems = [
+            ...analysis.impossible.map(x => ({ ...x, category: 'IMPOSSIBLE' })),
+            ...analysis.blocked.map(x => ({ ...x, category: 'BLOCKED' })),
+            ...analysis.failedDependency.map(x => ({ ...x, category: 'DEPENDENCY_FAILURE' })),
+            // If it's still 'RUNNABLE' after the final pass, it means it was skipped/missed/failed silently
+            ...analysis.runnable.map(x => ({ ...x, category: 'STUCK_RUNNABLE' })), 
+            ...analysis.reRuns.map(x => ({ ...x, category: 'STUCK_RERUN' }))
+        ];
+
+        // 5. Deep Investigation & Reporting
+        let issuesCount = 0;
+        const batch = this.db.batch();
+        let opCount = 0;
+
+        // A. Analyze Standard Problems
+        for (const item of problematicItems) {
+            const forensics = await this.investigateComputation(
+                item.name, dateStr, passId, item.category, manifestMap, 
+                dailyStatus, prevDailyStatus, rootDataStatus
+            );
+            
+            if (forensics) {
+                this._queueReportWrite(batch, dateStr, item.name, forensics);
+                opCount++;
+                issuesCount++;
+            }
+        }
+
+        // B. Analyze "Ghost" Completions (Marked Complete but Empty)
+        for (const calc of passCalcs) {
+            const name = normalizeName(calc.name);
+            const status = dailyStatus[name];
+            
+            if (status && status.hash === calc.hash) {
+                const hasData = await this.verifyDataExists(name, calc.category, dateStr);
+                if (!hasData) {
+                    const forensics = {
+                        computation: name,
+                        date: dateStr,
+                        category: 'GHOST_COMPLETION',
+                        rootCause: 'DATA_WRITE_FAILURE',
+                        reason: 'Status index says COMPLETE, but storage document is missing.',
+                        severity: 'HIGH',
+                        recommendations: ['Force Re-run', 'Check Write Permissions']
+                    };
+                    this._queueReportWrite(batch, dateStr, name, forensics);
+                    opCount++;
+                    issuesCount++;
+                }
+            }
+        }
+
+        if (opCount > 0) await batch.commit();
+        
+        this.logger.log('SUCCESS', `[FinalSweep] Pass ${passId}: Generated ${issuesCount} forensic reports.`);
+        return { issuesCount };
+    }
+
+    _queueReportWrite(batch, dateStr, calcName, forensics) {
+        // Path: Final_sweep_check_report/YYYY-MM-DD/Computation_Name/document
+        const reportRef = this.db.collection(REPORT_COLLECTION)
+            .doc(dateStr)
+            .collection(normalizeName(calcName))
+            .doc('document');
+
+        batch.set(reportRef, {
+            ...forensics,
+            generatedAt: new Date().toISOString()
+        });
+    }
+
+    /**
+     * Deep dive investigation logic
+     */
+    async investigateComputation(calcName, dateStr, passId, category, manifestMap, dailyStatus, prevDailyStatus, rootDataStatus) {
+        const manifest = manifestMap.get(calcName);
+        if (!manifest) return null;
+
+        const forensics = {
+            computation: calcName,
+            pass: passId,
+            date: dateStr,
+            category: category,
+            rootCause: null,
+            reason: null,
+            ledgerState: null,
+            recommendations: [],
+            severity: 'MEDIUM'
+        };
+
+        // 1. CHECK AUDIT LEDGER
+        const ledgerPath = `${LEDGER_BASE}/${dateStr}/passes/${passId}/tasks/${normalizeName(calcName)}`;
+        const ledgerSnap = await this.db.doc(ledgerPath).get();
+
+        if (ledgerSnap.exists) {
+            const data = ledgerSnap.data();
+            forensics.ledgerState = { status: data.status, workerId: data.workerId, error: data.error };
+
+            if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
+                const lastHb = data.telemetry?.lastHeartbeat ? new Date(data.telemetry.lastHeartbeat).getTime() : 0;
+                if (Date.now() - lastHb > STALE_THRESHOLD_MS) {
+                    forensics.rootCause = 'ZOMBIE_PROCESS';
+                    forensics.reason = `Worker ${data.workerId} stopped heartbeating. Likely crashed/timeout.`;
+                    forensics.severity = 'CRITICAL';
+                } else {
+                    forensics.rootCause = 'STILL_RUNNING'; // Rare if sweep runs after everything
+                }
+            } else if (data.status === 'FAILED') {
+                if (data.resourceTier === 'high-mem') {
+                    forensics.rootCause = 'CRASH_HIGH_MEM';
+                    forensics.reason = 'Failed even on High-Memory tier. Code optimization required.';
+                    forensics.severity = 'CRITICAL';
+                } else {
+                    forensics.rootCause = 'FAILED_STANDARD';
+                    forensics.reason = data.error?.message || 'Unknown Error';
+                }
+                
+                if (data.error?.stage === 'OOM' || (data.error?.message || '').includes('memory')) {
+                    forensics.rootCause = 'OUT_OF_MEMORY';
+                }
+            }
+        } else {
+            forensics.ledgerState = 'NEVER_DISPATCHED';
+            if (category === 'STUCK_RUNNABLE') {
+                forensics.rootCause = 'DISPATCHER_MISS';
+                forensics.reason = 'Logic says runnable, but Dispatcher never queued it.';
+                forensics.severity = 'HIGH';
+            }
+        }
+
+        // 2. ROOT DATA ANALYSIS
+        if (category === 'IMPOSSIBLE') {
+            const missing = [];
+            const userType = manifest.userType || 'all';
+            const deps = manifest.rootDataDependencies || [];
+
+            if (deps.includes('portfolio')) {
+                if (userType === 'speculator' && !rootDataStatus.speculatorPortfolio) missing.push('speculatorPortfolio');
+                else if (userType === 'normal' && !rootDataStatus.normalPortfolio) missing.push('normalPortfolio');
+                else if (userType === 'all' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+            }
+            if (deps.includes('history')) {
+                if (userType === 'speculator' && !rootDataStatus.speculatorHistory) missing.push('speculatorHistory');
+                else if (userType === 'normal' && !rootDataStatus.normalHistory) missing.push('normalHistory');
+                else if (userType === 'all' && !rootDataStatus.hasHistory) missing.push('history');
+            }
+            if (deps.includes('price') && !rootDataStatus.hasPrices) missing.push('price');
+            if (deps.includes('insights') && !rootDataStatus.hasInsights) missing.push('insights');
+
+            forensics.rootCause = 'MISSING_ROOT_DATA';
+            forensics.reason = `Missing: ${missing.join(', ')}`;
+            forensics.severity = 'LOW'; // Expected behavior
+        }
+
+        // 3. DEPENDENCY ANALYSIS
+        if (category === 'DEPENDENCY_FAILURE' || category === 'BLOCKED') {
+            const chain = await this.traceDependencyChain(calcName, manifestMap, dailyStatus, prevDailyStatus, manifest.isHistorical);
+            if (chain.length > 0) {
+                const root = chain[chain.length - 1];
+                forensics.rootCause = 'UPSTREAM_FAILURE';
+                forensics.reason = `Blocked by ${root.name} (${root.reason})`;
+                forensics.chain = chain;
+            }
+        }
+
+        return forensics;
+    }
+
+    async traceDependencyChain(calcName, manifestMap, dailyStatus, prevDailyStatus, isHistorical) {
+        const chain = [];
+        const visited = new Set();
+
+        const trace = (name, isPrev) => {
+            if (visited.has(name)) return;
+            visited.add(name);
+            
+            const m = manifestMap.get(name);
+            if (!m) return; // Unknown calc
+            
+            const status = isPrev ? (prevDailyStatus?.[name]) : (dailyStatus?.[name]);
+            
+            if (!status) {
+                chain.push({ name, reason: 'MISSING_STATUS', isPrev });
+                return;
+            }
+            if (String(status.hash).startsWith('IMPOSSIBLE')) {
+                chain.push({ name, reason: 'IMPOSSIBLE', isPrev });
+                return;
+            }
+            if (m.hash !== status.hash) {
+                chain.push({ name, reason: 'VERSION_MISMATCH', isPrev });
+                return;
+            }
+
+            if (m.dependencies) {
+                for (const dep of m.dependencies) trace(normalizeName(dep), false);
+            }
+        };
+
+        const m = manifestMap.get(calcName);
+        if (m) {
+            if (m.dependencies) m.dependencies.forEach(d => trace(normalizeName(d), false));
+            if (isHistorical) trace(calcName, true);
+        }
+        return chain;
+    }
+
+    async verifyDataExists(calcName, category, dateStr) {
+        try {
+            const docRef = this.db.collection(this.config.resultsCollection)
+                .doc(dateStr)
+                .collection(this.config.resultsSubcollection)
+                .doc(category)
+                .collection(this.config.computationsSubcollection)
+                .doc(calcName);
+            
+            const snap = await docRef.get();
+            if (!snap.exists) return false;
+            
+            const data = snap.data();
+            if (data._completed === true) return true;
+            return false;
+        } catch (e) {
+            return false;
+        }
+    }
+}
+
+async function runFinalSweepCheck(config, dependencies, dateStr, pass, manifest) {
+    const reporter = new FinalSweepReporter(config, dependencies);
+    return await reporter.runSweep(dateStr, pass, manifest);
+}
+
+module.exports = { runFinalSweepCheck };

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,7 +1,4 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# FAST-FORWARD MODE: Dispatch -> (Internal Loop) -> Wait ETA -> Jump to Next Valid Cursor
-# UPGRADED: Supports 'nextCursor' to skip empty date ranges instantly.
-
 main:
   params: [input]
   steps:
@@ -12,6 +9,7 @@ main:
           - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
           - date_to_run: '${default(map.get(input, "date"), current_date)}'
 
+    # --- PHASE 1: EXECUTION (Standard + High Mem Retry) ---
     - run_sequential_passes:
         for:
           value: pass_id
@@ -34,28 +32,29 @@ main:
                               pass: '${pass_id}'
                               cursorIndex: '${n_cursor}'
                               date: '${date_to_run}'
+                              action: 'DISPATCH'
                             auth: { type: OIDC }
                           result: dispatch_res
 
                       - evaluate_dispatch:
                           switch:
-                            # 1. End of Session (Dispatcher reached absolute end of date list)
+                            # 1. End of Session
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # 2. Satiation Check (Dispatcher scanned to end, found nothing)
+                            # 2. Satiation Check
                             - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
                                     call: sys.log
                                     args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Starting Verification Sweep."}'
+                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated. Starting Verification."}'
                                 - mark_complete:
                                     assign:
                                       - pass_complete: true
 
-                            # 3. Work Dispatched: Wait ETA -> Jump to 'nextCursor'
+                            # 3. Work Dispatched
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
                                 - wait_for_completion:
@@ -64,26 +63,24 @@ main:
                                       seconds: '${int(dispatch_res.body.etaSeconds)}'
                                 - update_cursor:
                                     assign:
-                                      # CRITICAL CHANGE: Use dispatcher's specific jump target
                                       - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_work:
                                     next: sequential_date_loop
 
-                            # 4. No Work (Fast-Forward limit reached or Single Check): Jump to 'nextCursor'
+                            # 4. No Work (Fast-Forward)
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
                                 - wait_short:
                                     call: sys.sleep
                                     args:
-                                      seconds: 2 # Tiny debounce for API limits
+                                      seconds: 2
                                 - update_cursor_retry:
                                     assign:
-                                        # CRITICAL CHANGE: Resume exactly where the Fast-Forward loop stopped
                                         - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_retry:
                                     next: sequential_date_loop
             
-            # --- VERIFICATION & SWEEP (No changes needed here) ---
+            # --- VERIFICATION & SWEEP ---
             - verify_pass_completion:
                 call: http.post
                 args:
@@ -103,8 +100,7 @@ main:
                     - log_sweep:
                         call: sys.log
                         args:
-                          text: '${"🧹 SWEEP: Disposing " + sweep_task.taskCount + " high-mem tasks for " + sweep_task.date + ". Wait: " + sweep_task.eta + "s"}'
-                    
+                          text: '${"🧹 SWEEP: Disposing " + sweep_task.taskCount + " high-mem tasks for " + sweep_task.date}'
                     - dispatch_force_sweep:
                         call: http.post
                         args:
@@ -114,11 +110,34 @@ main:
                             pass: '${pass_id}'
                             date: '${sweep_task.date}'
                           auth: { type: OIDC }
-                    
                     - wait_sweep_completion:
                         call: sys.sleep
                         args:
                           seconds: '${int(sweep_task.eta)}'
 
+    # --- PHASE 2: FINAL FORENSIC REPORTING ---
+    # Triggered after ALL execution attempts for this pass (Standard -> Verify -> HighMem Sweep)
+    # We ask the dispatcher to run the FinalSweepReporter for the target date.
+    - run_final_forensics:
+        for:
+          value: pass_id
+          in: ${passes}
+          steps:
+            - generate_final_report:
+                call: http.post
+                args:
+                  url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}'
+                  body:
+                    action: 'REPORT'
+                    pass: '${pass_id}'
+                    date: '${date_to_run}'
+                  auth: { type: OIDC }
+                result: report_res
+            
+            - log_forensics:
+                call: sys.log
+                args:
+                   text: '${"📝 FINAL REPORT: Pass " + pass_id + " -> " + report_res.body.issuesFound + " detailed forensic documents created."}'
+
     - finish:
-        return: "Pipeline Execution Satiated and Verified"
+        return: "Pipeline Complete with Forensic Analysis"

package/functions/generic-api/helpers/api_helpers.js

@@ -77,16 +77,38 @@ class AvailabilityCache {
 
 /**
  * Helper: Resolve which dates to fetch based on mode and availability.
+ * UPDATED: Added "Inferred Availability" for backfilled computations (user-history-reconstructor).
  */
 async function resolveTargetDates(availabilityCache, computationKeys, mode, limit) {
     const map = await availabilityCache.getMap();
-    
     const relevantDatesSet = new Set();
+    
+    // [NEW] Special Handling for Backfilled Computations
+    // These computations write to past dates but only update status for the current run date.
+    // If we see a recent status, we infer that history exists backward from that point.
+    const BACKFILL_COMPUTATIONS = ['user-history-reconstructor'];
+    const hasBackfillCalc = computationKeys.some(k => BACKFILL_COMPUTATIONS.includes(k));
+
     computationKeys.forEach(key => {
         const dates = map[key] || [];
         dates.forEach(d => relevantDatesSet.add(d));
     });
 
+    // If we are requesting a backfill-capable computation and have at least one valid date
+    if (hasBackfillCalc && relevantDatesSet.size > 0 && mode === 'series') {
+        const sortedExisting = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
+        const anchorDate = sortedExisting[0]; // Latest available date acts as the anchor
+
+        // Generate 'limit' days backwards from the anchor
+        const anchorTime = new Date(anchorDate).getTime();
+        for (let i = 0; i < limit; i++) {
+            const d = new Date(anchorTime - (i * 86400000));
+            const dateStr = d.toISOString().slice(0, 10);
+            relevantDatesSet.add(dateStr);
+        }
+    }
+
+    // Standard sorting and slicing
     const sortedDates = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
 
     if (sortedDates.length === 0) return [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.346",
+  "version": "1.0.348",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [