bulltrackers-module 1.0.317 → 1.0.318

package/functions/computation-system/tools/BuildReporter.js

@@ -1,7 +1,9 @@
 /**
- * FILENAME: computation-system/tools/BuildReporter.js
- * UPDATED: Replaced hardcoded 90-day window with a dynamic window based on system data availability.
+ * FILENAME: bulltrackers-module/functions/computation-system/tools/BuildReporter.js
+ * UPGRADED: Offloads heavy logic to a dedicated Cloud Function via Pub/Sub.
+ * FEATURES: Patch versioning, data-drift detection (window changes), and checkpointed writes.
  */
+
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
@@ -11,12 +13,48 @@ const SYSTEM_EPOCH = require('../system_epoch');
 const pLimit = require('p-limit');
 const path = require('path');
 const crypto = require('crypto');
-const fs = require('fs'); 
+
+// Load package info for versioning
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
 
+const BUILD_RECORDS_COLLECTION = 'computation_build_records';
+const BUILD_METADATA_DOC = 'system_build_metadata';
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 
+/**
+ * Publishes a message to trigger the dedicated Build Reporter Cloud Function.
+ * Replaces the old ensureBuildReport that ran locally on module load.
+ */
+async function requestBuildReport(config, dependencies) {
+    const { pubsubUtils, logger } = dependencies;
+    try {
+        await pubsubUtils.publish(config.buildReporterTopic, { 
+            requestedAt: new Date().toISOString(),
+            packageVersion: packageVersion 
+        });
+        logger.log('INFO', `[BuildReporter] 🛰️ Trigger message sent to ${config.buildReporterTopic}`);
+        return { success: true };
+    } catch (e) {
+        logger.log('ERROR', `[BuildReporter] Failed to publish trigger: ${e.message}`);
+        throw e;
+    }
+}
+
+/**
+ * Cloud Function Entry Point for the Build Reporter.
+ */
+async function handleBuildReportTrigger(message, context, config, dependencies, manifest) {
+    const { logger } = dependencies;
+    logger.log('INFO', `[BuildReporter] 📥 Trigger received. Starting build analysis...`);
+    try {
+        return await generateBuildReport(config, dependencies, manifest);
+    } catch (e) {
+        logger.log('ERROR', `[BuildReporter] Fatal error in execution: ${e.message}`);
+        throw e; 
+    }
+}
+
 /**
  * Replaces expensive file walking with System Epoch + Manifest Hash.
  */
@@ -27,6 +65,20 @@ function getSystemFingerprint(manifest) {
         .digest('hex');
 }
 
+/**
+ * Increments the patch number for the current package version in Firestore.
+ */
+async function getNextBuildId(db, version) {
+    const metaRef = db.collection(BUILD_RECORDS_COLLECTION).doc(BUILD_METADATA_DOC);
+    return await db.runTransaction(async (t) => {
+        const doc = await t.get(metaRef);
+        const data = doc.exists ? doc.data() : {};
+        const currentPatch = (data[version] || 0) + 1;
+        t.set(metaRef, { [version]: currentPatch }, { merge: true });
+        return `v${version}_p${currentPatch}`;
+    });
+}
+
 function isDateBeforeAvailability(dateStr, calcManifest) {
     const targetDate = new Date(dateStr + 'T00:00:00Z');
     const deps = calcManifest.rootDataDependencies || [];
@@ -115,72 +167,44 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
 }
 
 /**
- * UPDATED: Now calculates daysBack dynamically based on absoluteEarliest data date.
+ * The main reporter logic. Handles drift detection, minor versioning, 
+ * and checkpointed batch writes to Firestore.
  */
-async function ensureBuildReport(config, dependencies, manifest) {
+async function generateBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
-    const now = new Date();
-    const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}`;
-    const lockRef = db.collection('computation_build_records').doc(`init_lock_v${packageVersion}`);
-
-    try {
-        const shouldRun = await db.runTransaction(async (t) => {
-            const doc = await t.get(lockRef);
-            if (doc.exists) { return false; }
-            t.set(lockRef, { status: 'IN_PROGRESS', startedAt: new Date(), buildId: buildId });
-            return true;
-        });
-
-        if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} locked. Skipping.`); return; }
-
-        const currentSystemHash = getSystemFingerprint(manifest);
-        const latestBuildDoc = await db.collection('computation_build_records').doc('latest').get();
-        
-        if (latestBuildDoc.exists) {
-            const latestData = latestBuildDoc.data();
-            if (latestData.systemFingerprint === currentSystemHash) {
-                logger.log('INFO', `[BuildReporter] ⚡ System Fingerprint (${currentSystemHash.substring(0,8)}) matches latest build. Skipping Report.`);
-                await db.collection('computation_build_records').doc(buildId).set({
-                    buildId,
-                    packageVersion,
-                    systemFingerprint: currentSystemHash,
-                    status: 'SKIPPED_IDENTICAL',
-                    referenceBuild: latestData.buildId,
-                    generatedAt: new Date().toISOString()
-                });
-                lockRef.update({ status: 'SKIPPED', completedAt: new Date() }).catch(() => {});
-                return;
-            }
-        }
-
-        // [DYNAMIC WINDOW CALCULATION]
-        const { absoluteEarliest } = DEFINITIVE_EARLIEST_DATES;
-        let dynamicDaysBack = 90; // Fallback default
-        if (absoluteEarliest) {
-            const today = new Date();
-            const diffTime = Math.abs(today - absoluteEarliest);
-            dynamicDaysBack = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 2; // Days difference + 2 day buffer
-        }
-
-        logger.log('INFO', `[BuildReporter] 🚀 Change Detected. Running Pre-flight Report for v${packageVersion} (Scope: ${dynamicDaysBack} days)...`);
-        await generateBuildReport(config, dependencies, manifest, dynamicDaysBack, buildId, currentSystemHash);
-        lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
-
-    } catch (e) {
-        logger.log('ERROR', `[BuildReporter] Auto-run check failed: ${e.message}`);
+    
+    const currentFingerprint = getSystemFingerprint(manifest);
+    const latestBuildDoc = await db.collection(BUILD_RECORDS_COLLECTION).doc('latest').get();
+    const latest = latestBuildDoc.exists ? latestBuildDoc.data() : null;
+
+    // [DATA DRIFT DETECTION]
+    // Force a run if the definitive earliest data date has changed (e.g. new history backfilled)
+    const currentEarliestStr = DEFINITIVE_EARLIEST_DATES.absoluteEarliest?.toISOString() || 'NONE';
+    const lastEarliestStr = latest?.windowEarliest || 'NONE';
+    const windowChanged = currentEarliestStr !== lastEarliestStr;
+
+    // If fingerprints match AND the window is the same, we can truly skip.
+    if (latest && latest.systemFingerprint === currentFingerprint && !windowChanged) {
+        logger.log('INFO', `[BuildReporter] ⚡ System fingerprint and window stable. Skipping report.`);
+        return { success: true, status: 'SKIPPED_IDENTICAL' };
     }
-}
 
-async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null, systemFingerprint = null) {
-    const { db, logger } = dependencies;
-    const buildId = customBuildId || `manual_${Date.now()}`;
-    const finalFingerprint = systemFingerprint || getSystemFingerprint(manifest);
+    // Increment patch version (e.g., v1.0.0_p1, v1.0.0_p2)
+    const buildId = await getNextBuildId(db, packageVersion);
+    logger.log('INFO', `[BuildReporter] 🚀 Change Detected. Generating Build ${buildId}. Reason: ${windowChanged ? 'Data Window Drift' : 'Code Change'}`);
 
-    logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days, Fingerprint: ${finalFingerprint.substring(0,8)})...`);
+    const today = new Date();
+    const { absoluteEarliest } = DEFINITIVE_EARLIEST_DATES;
+    
+    // Dynamic Window calculation
+    let dynamicDaysBack = 90; 
+    if (absoluteEarliest) {
+        const diffTime = Math.abs(today - absoluteEarliest);
+        dynamicDaysBack = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 2; 
+    }
 
-    const today     = new Date();
     const startDate = new Date();
-    startDate.setDate(today.getDate() - daysBack);
+    startDate.setDate(today.getDate() - dynamicDaysBack);
     
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
@@ -201,131 +225,131 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportHeader = { 
         buildId, 
         packageVersion, 
-        systemFingerprint: finalFingerprint, 
+        systemFingerprint: currentFingerprint, 
+        windowEarliest: currentEarliestStr,
         generatedAt: new Date().toISOString(), 
+        status: 'IN_PROGRESS',
         summary: {}, 
         _sharded: true 
     };
 
+    // Initialize the build record
+    await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).set(reportHeader);
+
     let totalRun = 0, totalReRun = 0, totalStable = 0;
-    const limit = pLimit(5);
+    const limit = pLimit(10); // Concurrency for fetching statuses
 
-    const processingPromises = datesToCheck.map(dateStr => limit(async () => {
-        try {
-            const fetchPromises = [
-                fetchComputationStatus(dateStr, config, dependencies),
-                checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES)
-            ];
-
-            let prevDateStr = null;
-            if (manifest.some(c => c.isHistorical)) {
-                const prevDate = new Date(dateStr + 'T00:00:00Z');
-                prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-                prevDateStr = prevDate.toISOString().slice(0, 10);
-                if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
-                    fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
+    // Process dates in chunks of 5 for checkpointed writing
+    for (let i = 0; i < datesToCheck.length; i += 5) {
+        const dateBatch = datesToCheck.slice(i, i + 5);
+        
+        const results = await Promise.all(dateBatch.map(dateStr => limit(async () => {
+            try {
+                const fetchPromises = [
+                    fetchComputationStatus(dateStr, config, dependencies),
+                    checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES)
+                ];
+
+                let prevDateStr = null;
+                if (manifest.some(c => c.isHistorical)) {
+                    const prevDate = new Date(dateStr + 'T00:00:00Z');
+                    prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+                    prevDateStr = prevDate.toISOString().slice(0, 10);
+                    if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
+                        fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
+                    }
                 }
-            }
 
-            const results         = await Promise.all(fetchPromises);
-            const dailyStatus     = results[0];
-            const availability    = results[1];
-            const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
-            const rootDataStatus  = availability ? availability.status : { hasPortfolio: false, hasHistory: false };
-
-            const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
-
-            const dateSummary = {
-                run: [], rerun: [], stable: [], blocked: [], impossible: [], uptodate: [],
-                meta: { totalIncluded: 0, totalExpected: 0, match: false }
-            };
-
-            const expectedCount = manifest.filter(c => !isDateBeforeAvailability(dateStr, c)).length;
-            dateSummary.meta.totalExpected = expectedCount;
-
-            const pushIfValid = (targetArray, item, extraReason = null) => {
-                const calcManifest = manifestMap.get(item.name);
-                if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) return;
-                
-                const entry = { 
-                    name: item.name, 
-                    reason: item.reason || extraReason,
-                    pass: calcManifest ? calcManifest.pass : '?'
+                const [dailyStatus, availability, prevRes] = await Promise.all(fetchPromises);
+                const prevDailyStatus = (prevDateStr && prevRes) ? prevRes : (prevDateStr ? {} : null);
+                const rootDataStatus  = availability ? availability.status : { hasPortfolio: false, hasHistory: false };
+
+                const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
+
+                const dateSummary = {
+                    run: [], rerun: [], stable: [], blocked: [], impossible: [], uptodate: [],
+                    meta: { totalIncluded: 0, totalExpected: 0, match: false }
                 };
-                if (targetArray === dateSummary.rerun) {
-                    entry.impact = calculateBlastRadius(item.name, reverseGraph);
-                }
-                targetArray.push(entry);
-            };
-
-            analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
-
-            if (analysis.reRuns.length > 0) {
-                const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
-                
-                trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
-                stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Cosmetic Change"));
-
-                if (stableUpdates.length > 0) {
-                    const updatesPayload = {};
-                    for (const stable of stableUpdates) {
-                        const m = manifestMap.get(stable.name);
-                        const stored = dailyStatus[stable.name];
-                        if (m && stored) {
-                            updatesPayload[stable.name] = {
-                                hash: m.hash, 
-                                simHash: stable.simHash, 
-                                resultHash: stored.resultHash, 
-                                dependencyResultHashes: stored.dependencyResultHashes || {}, 
-                                category: m.category,
-                                composition: m.composition,
-                                lastUpdated: new Date()
-                            };
+
+                const expectedCount = manifest.filter(c => !isDateBeforeAvailability(dateStr, c)).length;
+                dateSummary.meta.totalExpected = expectedCount;
+
+                const pushIfValid = (targetArray, item, extraReason = null) => {
+                    const calcManifest = manifestMap.get(item.name);
+                    if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) return;
+                    const entry = { name: item.name, reason: item.reason || extraReason, pass: calcManifest?.pass || '?' };
+                    if (targetArray === dateSummary.rerun) entry.impact = calculateBlastRadius(item.name, reverseGraph);
+                    targetArray.push(entry);
+                };
+
+                analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
+
+                if (analysis.reRuns.length > 0) {
+                    const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
+                    trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
+                    stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Logic Stable"));
+
+                    if (stableUpdates.length > 0) {
+                        const updatesPayload = {};
+                        for (const stable of stableUpdates) {
+                            const m = manifestMap.get(stable.name);
+                            const stored = dailyStatus[stable.name];
+                            if (m && stored) {
+                                updatesPayload[stable.name] = {
+                                    hash: m.hash, simHash: stable.simHash, resultHash: stored.resultHash, 
+                                    dependencyResultHashes: stored.dependencyResultHashes || {}, 
+                                    category: m.category, composition: m.composition, lastUpdated: new Date()
+                                };
+                            }
+                        }
+                        if (Object.keys(updatesPayload).length > 0) {
+                            await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
                         }
-                    }
-                    if (Object.keys(updatesPayload).length > 0) {
-                        await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
-                        logger.log('INFO', `[BuildReporter] 🩹 Fixed ${Object.keys(updatesPayload).length} stable items for ${dateStr}. They will NOT re-run.`);
                     }
                 }
-            }
 
-            analysis.blocked.forEach          (item => pushIfValid(dateSummary.blocked,    item));
-            analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked,    item, "Dependency Missing"));
-            analysis.impossible.forEach       (item => pushIfValid(dateSummary.impossible, item));
-            analysis.skipped.forEach          (item => pushIfValid(dateSummary.uptodate,   item, "Up To Date"));
+                analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
+                analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
+                analysis.impossible.forEach(item => pushIfValid(dateSummary.impossible, item));
+                analysis.skipped.forEach(item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
 
-            const includedCount = dateSummary.run.length     + dateSummary.rerun.length      + dateSummary.stable.length +
-                                  dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
-            dateSummary.meta.totalIncluded = includedCount;
-            dateSummary.meta.match = (includedCount === expectedCount);
+                const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
+                                      dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
+                dateSummary.meta.totalIncluded = includedCount;
+                dateSummary.meta.match = (includedCount === expectedCount);
 
-            await db.collection('computation_build_records')
-            .doc(buildId)
-            .collection('details')
-            .doc(dateStr)
-            .set(dateSummary);
+                // Write detailed date record
+                await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).collection('details').doc(dateStr).set(dateSummary);
 
-            return { stats: { run: dateSummary.run.length, rerun: dateSummary.rerun.length, stable: dateSummary.stable.length } };
-
-        } catch (err) {
-            logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
-            return null;
-        }
-    }));
-
-    const results = await Promise.all(processingPromises);
+                return { run: dateSummary.run.length, rerun: dateSummary.rerun.length, stable: dateSummary.stable.length };
+            } catch (err) {
+                logger.log('ERROR', `[BuildReporter] Analysis failed for ${dateStr}: ${err.message}`);
+                return { run: 0, rerun: 0, stable: 0 };
+            }
+        })));
 
-    results.forEach(res => { if (res) { totalRun += res.stats.run; totalReRun += res.stats.rerun; totalStable += res.stats.stable; } });
+        // Accumulate stats and write a progress checkpoint
+        results.forEach(res => { totalRun += res.run; totalReRun += res.rerun; totalStable += res.stable; });
+        await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).update({ 
+            checkpoint: `Processed ${i + dateBatch.length}/${datesToCheck.length} dates` 
+        });
+    }
 
-    reportHeader.summary = { totalReRuns: totalReRun, totalNew: totalRun, totalStable: totalStable, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
+    reportHeader.status = 'COMPLETED';
+    reportHeader.summary = { 
+        totalReRuns: totalReRun, 
+        totalNew: totalRun, 
+        totalStable: totalStable, 
+        scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
+    };
 
-    await db.collection('computation_build_records').doc(buildId).set(reportHeader);
-    await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer." });
+    // Finalize build record and update the 'latest' pointer
+    await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).set(reportHeader);
+    await db.collection(BUILD_RECORDS_COLLECTION).doc('latest').set({ ...reportHeader, note: "Latest completed build report." });
     
-    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, Stable (Fixed): ${totalStable}, New: ${totalRun}.`);
+    logger.log('SUCCESS', `[BuildReporter] Build ${buildId} completed. Re-runs: ${totalReRun}, Stable: ${totalStable}, New: ${totalRun}.`);
     
-    return { success: true, reportId: buildId, summary: reportHeader.summary };
+    return { success: true, buildId, summary: reportHeader.summary };
 }
 
-module.exports = { ensureBuildReport, generateBuildReport };
+module.exports = { requestBuildReport, handleBuildReportTrigger, generateBuildReport };

package/index.js

@@ -29,8 +29,12 @@ const { handleUpdate }                                    = require('./functions
 const { build: buildManifest }                            = require('./functions/computation-system/context/ManifestBuilder');
 const { dispatchComputationPass }                         = require('./functions/computation-system/helpers/computation_dispatcher');
 const { handleComputationTask }                           = require('./functions/computation-system/helpers/computation_worker');
-const { ensureBuildReport, generateBuildReport }          = require('./functions/computation-system/tools/BuildReporter');
-// [NEW] Import Monitor
+const { 
+    requestBuildReport, 
+    handleBuildReportTrigger, 
+    generateBuildReport 
+} = require('./functions/computation-system/tools/BuildReporter');// [NEW] Import Monitor
+
 const { checkPassStatus }                                 = require('./functions/computation-system/helpers/monitor');
 
 const dataLoader                                          = require('./functions/computation-system/utils/data_loader');
@@ -92,9 +96,10 @@ const computationSystem = {
   dataLoader,
   computationUtils,
   buildManifest,
-  ensureBuildReport,
+  // [UPDATED] Refactored Reporter Pipes
+  requestBuildReport,
+  handleBuildReportTrigger,
   generateBuildReport,
-  // [NEW] Export Monitor Pipe
   checkPassStatus
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.317",
+  "version": "1.0.318",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [