bulltrackers-module 1.0.291 → 1.0.292

package/functions/computation-system/tools/BuildReporter.js

@@ -6,41 +6,29 @@ const SimRunner = require('../simulation/SimRunner');
 const pLimit = require('p-limit');
 const path = require('path');
 const crypto = require('crypto');
-const fs = require('fs'); // [NEW] Required for file reading
+const fs = require('fs'); 
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
-const { generateCodeHash } = require('../utils/utils'); // Reuse your standard logic
+const { generateCodeHash } = require('../utils/utils'); 
 
-// Persistent Registry for SimHashes
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 
-// --- [NEW] RECURSIVE SYSTEM HASHING ---
-
-// 1. define the Root of the system (one level up from 'tools')
+// --- RECURSIVE SYSTEM HASHING ---
 const SYSTEM_ROOT = path.resolve(__dirname, '..');
-
-// 2. Define what to ignore to prevent noise or infinite loops
 const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests']);
 const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
 
-/**
- * Recursively walks a directory and returns a list of file paths.
- */
 function walkSync(dir, fileList = []) {
     const files = fs.readdirSync(dir);
-    
     files.forEach(file => {
         if (IGNORED_FILES.has(file)) return;
-        
         const filePath = path.join(dir, file);
         const stat = fs.statSync(filePath);
-        
         if (stat.isDirectory()) {
             if (!IGNORED_DIRS.has(file)) {
                 walkSync(filePath, fileList);
             }
         } else {
-            // Only hash code files (add .yaml if you want workflows included)
             if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
                 fileList.push(filePath);
             }
@@ -49,69 +37,41 @@ function walkSync(dir, fileList = []) {
     return fileList;
 }
 
-/**
- * Generates a single hash representing the entire infrastructure code state.
- */
 function getInfrastructureHash() {
     try {
         const allFiles = walkSync(SYSTEM_ROOT);
-        allFiles.sort(); // Crucial for determinism
-
+        allFiles.sort(); 
         const bigHash = crypto.createHash('sha256');
-
         for (const filePath of allFiles) {
             const content = fs.readFileSync(filePath, 'utf8');
             const relativePath = path.relative(SYSTEM_ROOT, filePath);
-
-            // DECISION: How to clean?
             let cleanContent = content;
-
-            // 1. If it's JS, use your system standard for code hashing
             if (filePath.endsWith('.js')) {
-                // This strips comments and whitespace consistently with ManifestBuilder
-                // Note: generateCodeHash returns a hash, we can just use that hash
                 cleanContent = generateCodeHash(content); 
-            } 
-            // 2. If it's JSON/YAML, just strip basic whitespace to ignore indent changes
-            else {
+            } else {
                 cleanContent = content.replace(/\s+/g, '');
             }
-
-            // Feed the PATH and the CLEAN CONTENT into the master hash
             bigHash.update(`${relativePath}:${cleanContent}|`);
         }
-
         return bigHash.digest('hex');
     } catch (e) {
         console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
-        return 'infra_hash_error'; // Fail safe so we run the report rather than crash
+        return 'infra_hash_error'; 
     }
 }
 
-/**
- * UPDATED: System Fingerprint = Manifest Hash + Infrastructure Hash
- */
 function getSystemFingerprint(manifest) {
-    // 1. Business Logic Hash (The Calculations)
     const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
-    
-    // 2. Infrastructure Hash (The System Code)
     const infraHash = getInfrastructureHash();
-
-    // 3. Combine
     return crypto.createHash('sha256')
         .update(sortedManifestHashes + infraHash)
         .digest('hex');
 }
 
-/**
- * Helper: Determines if a calculation should be excluded from the report.
- */
 function isDateBeforeAvailability(dateStr, calcManifest) {
     const targetDate = new Date(dateStr + 'T00:00:00Z');
     const deps = calcManifest.rootDataDependencies || [];
     if (deps.length === 0) return false;
-
     for (const dep of deps) {
         let startDate = null;
         if (dep === 'portfolio')      startDate = DEFINITIVE_EARLIEST_DATES.portfolio;
@@ -119,19 +79,14 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
         else if (dep === 'social')    startDate = DEFINITIVE_EARLIEST_DATES.social;
         else if (dep === 'insights')  startDate = DEFINITIVE_EARLIEST_DATES.insights;
         else if (dep === 'price')     startDate = DEFINITIVE_EARLIEST_DATES.price;
-
         if (startDate && targetDate < startDate) { return true; }
     }
     return false;
 }
 
-/**
- * Helper: Calculates the transitive closure of dependents (Blast Radius).
- */
 function calculateBlastRadius(targetCalcName, reverseGraph) {
     const impactSet = new Set();
     const queue = [targetCalcName];
-    
     while(queue.length > 0) {
         const current = queue.shift();
         const dependents = reverseGraph.get(current) || [];
@@ -142,7 +97,6 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
             }
         });
     }
-
     return {
         directDependents: (reverseGraph.get(targetCalcName) || []).length,
         totalCascadingDependents: impactSet.size,
@@ -150,14 +104,11 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
     };
 }
 
-/**
- * [NEW] Helper: Runs SimHash check with Caching and Registry Persistence.
- */
 async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
     const trueReRuns = [];
     const stableUpdates = [];
     
-    // Limit concurrency for simulations
+    // Concurrency for simulations
     const limit = pLimit(10);
 
     const checks = candidates.map(item => limit(async () => {
@@ -170,16 +121,10 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 return;
             }
 
-            // 1. Check Cache first (Avoid re-simulating the same code for 100 different dates)
             let newSimHash = simHashCache.get(manifest.hash);
-
-            // 2. If Miss, Run Simulation & Persist to Registry
             if (!newSimHash) {
                 newSimHash = await SimRunner.run(manifest, manifestMap);
                 simHashCache.set(manifest.hash, newSimHash);
-                
-                // Write to Registry so Production Workers can find it without running SimRunner
-                // Fire-and-forget write to reduce latency
                 db.collection(SIMHASH_REGISTRY_COLLECTION).doc(manifest.hash).set({
                     simHash: newSimHash,
                     createdAt: new Date(),
@@ -187,17 +132,14 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 }).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
             }
 
-            // 3. Compare
             if (newSimHash === stored.simHash) {
-                // BEHAVIORAL MATCH: Code changed, but output is identical.
                 stableUpdates.push({
                     ...item,
                     reason: "Code Updated (Logic Stable)",
-                    simHash: newSimHash, // New SimHash (same as old)
-                    newHash: manifest.hash // New Code Hash
+                    simHash: newSimHash, 
+                    newHash: manifest.hash 
                 });
             } else {
-                // BEHAVIORAL MISMATCH: Logic changed.
                 trueReRuns.push({
                     ...item,
                     reason: item.reason + ` [SimHash Mismatch]`,
@@ -215,9 +157,6 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
     return { trueReRuns, stableUpdates };
 }
 
-/**
- * AUTO-RUN ENTRY POINT
- */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
@@ -234,40 +173,28 @@ async function ensureBuildReport(config, dependencies, manifest) {
 
         if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} locked. Skipping.`); return; }
 
-        // [NEW] 1. Calculate Current System Fingerprint
         const currentSystemHash = getSystemFingerprint(manifest);
-
-        // [NEW] 2. Fetch Last Build's Fingerprint
         const latestBuildDoc = await db.collection('computation_build_records').doc('latest').get();
         
         if (latestBuildDoc.exists) {
             const latestData = latestBuildDoc.data();
-            
-            // [OPTIMIZATION] If signatures match, we can clone the report or just skip
             if (latestData.systemFingerprint === currentSystemHash) {
                 logger.log('INFO', `[BuildReporter] ⚡ System Fingerprint (${currentSystemHash.substring(0,8)}) matches latest build. Skipping Report.`);
-                
-                // Create a "Skipped" record for the new version so we know it deployed
                 await db.collection('computation_build_records').doc(buildId).set({
                     buildId,
                     packageVersion,
                     systemFingerprint: currentSystemHash,
                     status: 'SKIPPED_IDENTICAL',
-                    referenceBuild: latestData.buildId, // Pointer to the build that actually did the work
+                    referenceBuild: latestData.buildId,
                     generatedAt: new Date().toISOString()
                 });
-                
-                // Release lock and exit
                 lockRef.update({ status: 'SKIPPED', completedAt: new Date() }).catch(() => {});
                 return;
             }
         }
 
         logger.log('INFO', `[BuildReporter] 🚀 Change Detected. Running Pre-flight Report for v${packageVersion}...`);
-        
-        // Pass the fingerprint to generateBuildReport so it can save it
         await generateBuildReport(config, dependencies, manifest, 90, buildId, currentSystemHash);
-        
         lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
 
     } catch (e) {
@@ -275,14 +202,9 @@ async function ensureBuildReport(config, dependencies, manifest) {
     }
 }
 
-/**
- * Generates the report, writes to Firestore, AND FIXES STABLE UPDATES.
- */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null, systemFingerprint = null) {
     const { db, logger } = dependencies;
     const buildId = customBuildId || `manual_${Date.now()}`;
-    
-    // Calculate fingerprint if not provided (for manual runs)
     const finalFingerprint = systemFingerprint || getSystemFingerprint(manifest);
 
     logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days, Fingerprint: ${finalFingerprint.substring(0,8)})...`);
@@ -293,8 +215,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
-    
-    // [OPTIMIZATION] Cache SimHashes across dates so we only calculate once per code version
     const simHashCache = new Map();
 
     const reverseGraph = new Map();
@@ -312,16 +232,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportHeader = { 
         buildId, 
         packageVersion, 
-        systemFingerprint: finalFingerprint, // Saved to Firestore
+        systemFingerprint: finalFingerprint, 
         generatedAt: new Date().toISOString(), 
         summary: {}, 
         _sharded: true 
     };
 
     let totalRun = 0, totalReRun = 0, totalStable = 0;
-    const detailWrites = []; 
 
-    const limit = pLimit(20);
+    // [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
+    const limit = pLimit(5);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
         try {
@@ -374,29 +294,22 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             // 1. RUN
             analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
 
-            // 2. RE-RUN & STABLE Analysis (SimHash Integration)
+            // 2. RE-RUN & STABLE Analysis
             if (analysis.reRuns.length > 0) {
-                // Pass simHashCache and db for registry writes
                 const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
                 
                 trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
                 stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Cosmetic Change"));
 
-                // [CRITICAL FIX] "Fix the Blast Radius"
-                // If updates are STABLE, we update the status NOW. 
-                // This implies: Code Hash changes, but Sim Hash stays same.
-                // The Dispatcher will see the new Code Hash in status matches the Manifest, so it won't dispatch.
                 if (stableUpdates.length > 0) {
                     const updatesPayload = {};
                     for (const stable of stableUpdates) {
                         const m = manifestMap.get(stable.name);
-                        // We preserve the *existing* resultHash because the logic is proven stable.
-                        // We update the 'hash' to the NEW code hash.
                         if (m && dailyStatus[stable.name]) {
                             updatesPayload[stable.name] = {
-                                hash: m.hash, // New Code Hash
-                                simHash: stable.simHash, // Same Sim Hash
-                                resultHash: dailyStatus[stable.name].resultHash, // Same Result Hash
+                                hash: m.hash, 
+                                simHash: stable.simHash, 
+                                resultHash: dailyStatus[stable.name].resultHash, 
                                 dependencyResultHashes: dailyStatus[stable.name].dependencyResultHashes || {},
                                 category: m.category,
                                 composition: m.composition,
@@ -404,7 +317,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                             };
                         }
                     }
-                    // Perform the "Fix"
                     if (Object.keys(updatesPayload).length > 0) {
                         await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
                         logger.log('INFO', `[BuildReporter] 🩹 Fixed ${Object.keys(updatesPayload).length} stable items for ${dateStr}. They will NOT re-run.`);
@@ -424,11 +336,8 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             dateSummary.meta.totalIncluded = includedCount;
             dateSummary.meta.match = (includedCount === expectedCount);
 
-            // REMOVE THIS: Inefficient memory usage causes timeouts
-            // detailWrites.push({ ref: db.collection(...), data: dateSummary });
-
-            // ADD THIS (Write immediately):
-           await db.collection('computation_build_records')
+            // Write Immediately
+            await db.collection('computation_build_records')
             .doc(buildId)
             .collection('details')
             .doc(dateStr)
@@ -450,10 +359,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     await db.collection('computation_build_records').doc(buildId).set(reportHeader);
     
-    // Parallel write details ---> Now redundant.
-    //const writeLimit = pLimit(15);
-    //await Promise.all(detailWrites.map(w => writeLimit(() => w.ref.set(w.data))));
-
     await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer." });
     
     logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, Stable (Fixed): ${totalStable}, New: ${totalRun}.`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.291",
+  "version": "1.0.292",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [