bulltrackers-module 1.0.269 → 1.0.271

package/functions/computation-system/tools/BuildReporter.js

@@ -1,40 +1,91 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * UPDATED: Shards report details to subcollections to bypass 1MB limit.
- * UPDATED: Explicitly reports SKIPPED items for 100% visibility.
+ * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
+ * UPDATED: Added transactional locking to prevent duplicate reports on concurrent cold starts.
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
 const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
-const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils'); // Reuse chunker
+const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils');
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion                                                       = packageJson.version;
 
+/**
+ * Helper: Determines if a calculation should be excluded from the report
+ * because the date is prior to the earliest possible data existence.
+ */
+function isDateBeforeAvailability(dateStr, calcManifest) {
+    const targetDate = new Date(dateStr + 'T00:00:00Z');
+    const deps = calcManifest.rootDataDependencies || [];
+    
+    // If no data dependencies, it's always valid (e.g., pure math)
+    if (deps.length === 0) return false;
+
+    for (const dep of deps) {
+        // Map dependency name to start date
+        let startDate = null;
+        if (dep === 'portfolio') startDate = DEFINITIVE_EARLIEST_DATES.portfolio;
+        else if (dep === 'history')   startDate = DEFINITIVE_EARLIEST_DATES.history;
+        else if (dep === 'social')    startDate = DEFINITIVE_EARLIEST_DATES.social;
+        else if (dep === 'insights')  startDate = DEFINITIVE_EARLIEST_DATES.insights;
+        else if (dep === 'price')     startDate = DEFINITIVE_EARLIEST_DATES.price;
+
+        // If we have a start date and the target is BEFORE it, exclude this calc.
+        if (startDate && targetDate < startDate) {
+            return true;
+        }
+    }
+    return false;
+}
+
 /**
  * AUTO-RUN ENTRY POINT
+ * UPDATED: Uses transactional locking to prevent race conditions.
  */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
     const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}-${String(now.getMinutes()).padStart(2,'0')}-${String(now.getSeconds()).padStart(2,'0')}`;
-    const latestRef = db.collection('computation_build_records').doc('latest');
+    
+    // Lock document specific to this version
+    const lockRef = db.collection('computation_build_records').doc(`init_lock_v${packageVersion}`);
 
     try {
-        const latestDoc = await latestRef.get();
-        const priorVersion = latestDoc.exists ? latestDoc.data().packageVersion : null;
+        // Transaction: "Hey I am deploying" check
+        const shouldRun = await db.runTransaction(async (t) => {
+            const doc = await t.get(lockRef);
+            
+            if (doc.exists) {
+                // Someone else beat us to it
+                return false; 
+            }
 
-        if (priorVersion === packageVersion) {
-            logger.log('INFO', `[BuildReporter] ✅ Version ${packageVersion} already has a report. Skipping.`);
+            // Claim the lock
+            t.set(lockRef, { 
+                status: 'IN_PROGRESS', 
+                startedAt: new Date(),
+                workerId: process.env.K_REVISION || 'unknown',
+                buildId: buildId
+            });
+            return true;
+        });
+
+        if (!shouldRun) {
+            logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} is already being generated (Locked). Skipping.`);
             return;
         }
 
-        logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${packageVersion}). Auto-running Pre-flight Report...`);
+        logger.log('INFO', `[BuildReporter] 🚀 Lock Acquired. Running Pre-flight Report for v${packageVersion}...`);
+        
         await generateBuildReport(config, dependencies, manifest, 90, buildId);
+        
+        // Optional: Update lock to completed (fire-and-forget update)
+        lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
 
     } catch (e) {
         logger.log('ERROR', `[BuildReporter] Auto-run check failed: ${e.message}`);
@@ -57,17 +108,17 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
-    // Main Report Header (Summary Only)
+    // Main Report Header
     const reportHeader = {
         buildId,
         packageVersion: packageVersion, 
         generatedAt: new Date().toISOString(),
-        summary: {},
+        summary: {}, 
         _sharded: true 
     };
 
-    let totalReRuns = 0;
-    let totalNew    = 0;
+    let totalRun = 0;
+    let totalReRun = 0;
     const detailWrites = []; 
 
     const limit = pLimit(20);
@@ -97,31 +148,78 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
             const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
-            // [NEW] Added 'skipped' to the summary object
-            const dateSummary = { willRun: [], willReRun: [], blocked: [], impossible: [], skipped: [] };
+            // ---------------------------------------------------------
+            // STRICT 5-CATEGORY MAPPING
+            // ---------------------------------------------------------
+            const dateSummary = {
+                run: [],        // New / No Hash / "Runnable"
+                rerun: [],      // Hash Mismatch / Category Migration
+                blocked: [],    // Missing Data (Today) / Dependency Missing
+                impossible: [], // Missing Data (Historical) / Impossible Dependency
+                uptodate: [],   // Hash Match (Previously "Skipped")
+                
+                // [NEW] Metadata for Verification
+                meta: {
+                    totalIncluded: 0,
+                    totalExpected: 0,
+                    match: false
+                }
+            };
 
-            analysis.runnable.forEach   (item => dateSummary.willRun.push    ({ name: item.name, reason: "New / No Previous Record" }));
-            analysis.reRuns.forEach     (item => dateSummary.willReRun.push  ({ name: item.name, reason: item.reason || "Hash Mismatch" }));
-            analysis.impossible.forEach (item => dateSummary.impossible.push ({ name: item.name, reason: item.reason }));
-            [...analysis.blocked, ...analysis.failedDependency].forEach(item => dateSummary.blocked.push({ name: item.name, reason: item.reason || 'Dependency' }));
-            
-            // [NEW] Map skipped items so the math adds up to 92
-            analysis.skipped.forEach    (item => dateSummary.skipped.push    ({ name: item.name, reason: item.reason || "Up To Date" }));
-
-            // Update: We write the report if there is ANY data, not just updates
-            // This ensures full visibility into the day's state
-            if (dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length || dateSummary.skipped.length) {
-                const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
-                detailWrites.push({
-                    ref: detailRef,
-                    data: dateSummary
-                });
-
-                return {
-                    stats: { new: dateSummary.willRun.length, rerun: dateSummary.willReRun.length }
-                };
+            // Calculate Expected Count (Computations in manifest that exist for this date)
+            const expectedCount = manifest.filter(c => !isDateBeforeAvailability(dateStr, c)).length;
+            dateSummary.meta.totalExpected = expectedCount;
+
+            // Helper to push only if date is valid for this specific calc
+            const pushIfValid = (targetArray, item, extraReason = null) => {
+                const calcManifest = manifestMap.get(item.name);
+                if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) {
+                    return; // EXCLUDED: Date is before data exists
+                }
+                targetArray.push({ name: item.name, reason: item.reason || extraReason });
+            };
+
+            // 1. RUN (New)
+            analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
+
+            // 2. RE-RUN (Hash Mismatch)
+            analysis.reRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Hash Mismatch"));
+
+            // 3. BLOCKED (Temporary Issues)
+            // Merging 'blocked' and 'failedDependency' as both are temporary blocks
+            analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
+            analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
+
+            // 4. IMPOSSIBLE (Permanent Issues)
+            analysis.impossible.forEach(item => pushIfValid(dateSummary.impossible, item));
+
+            // 5. UP-TO-DATE (Previously "Skipped")
+            analysis.skipped.forEach(item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
+
+            // Calculate Included Count
+            const includedCount = dateSummary.run.length + 
+                                  dateSummary.rerun.length + 
+                                  dateSummary.blocked.length + 
+                                  dateSummary.impossible.length + 
+                                  dateSummary.uptodate.length;
+
+            dateSummary.meta.totalIncluded = includedCount;
+            dateSummary.meta.match = (includedCount === expectedCount);
+
+            if (!dateSummary.meta.match) {
+                logger.log('WARN', `[BuildReporter] ⚠️ Mismatch on ${dateStr}: Expected ${expectedCount} but got ${includedCount}.`);
             }
-            return null;
+
+            // ALWAYS WRITE THE REPORT (No filtering based on activity)
+            const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
+            detailWrites.push({
+                ref: detailRef,
+                data: dateSummary
+            });
+
+            return {
+                stats: { run: dateSummary.run.length, rerun: dateSummary.rerun.length }
+            };
 
         } catch (err) {
             logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
@@ -133,12 +231,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     results.forEach(res => {
         if (res) {
-            totalNew += res.stats.new;
-            totalReRuns += res.stats.rerun;
+            totalRun   += res.stats.run;
+            totalReRun += res.stats.rerun;
         }
     });
 
-    reportHeader.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
+    reportHeader.summary = { 
+        totalReRuns: totalReRun, 
+        totalNew: totalRun, 
+        scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
+    };
 
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportHeader);
@@ -150,7 +252,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer (See subcollection for details)." });
 
-    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRuns}, New: ${totalNew}.`);
+    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}.`);
     
     return {
         success: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.269",
+  "version": "1.0.271",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [