bulltrackers-module 1.0.281 → 1.0.282

package/functions/computation-system/tools/BuildReporter.js

@@ -1,34 +1,32 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
- * UPDATED: Replaced Batch Writes with Parallel Writes to prevent DEADLINE_EXCEEDED timeouts.
- * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
- * UPDATED (IDEA 1): Added Dependency Impact Analysis ("Blast Radius").
+ * UPGRADED: Implements Behavioral Hashing (SimHash) to detect Cosmetic vs Logic changes.
+ * OPTIMIZED: Caches SimHashes and actively updates status for Stable items to prevent re-runs.
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
-const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
+const { fetchComputationStatus, updateComputationStatus }                  = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
+const SimRunner                                                            = require('../simulation/SimRunner');
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion                                                       = packageJson.version;
 
+// Persistent Registry for SimHashes (so Workers don't have to recalc)
+const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
+
 /**
- * Helper: Determines if a calculation should be excluded from the report
- * because the date is prior to the earliest possible data existence.
+ * Helper: Determines if a calculation should be excluded from the report.
  */
 function isDateBeforeAvailability(dateStr, calcManifest) {
     const targetDate = new Date(dateStr + 'T00:00:00Z');
     const deps = calcManifest.rootDataDependencies || [];
-    
-    // If no data dependencies, it's always valid (e.g., pure math)
     if (deps.length === 0) return false;
 
     for (const dep of deps) {
-        // Map dependency name to start date
         let startDate = null;
         if (dep === 'portfolio')      startDate = DEFINITIVE_EARLIEST_DATES.portfolio;
         else if (dep === 'history')   startDate = DEFINITIVE_EARLIEST_DATES.history;
@@ -36,7 +34,6 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
         else if (dep === 'insights')  startDate = DEFINITIVE_EARLIEST_DATES.insights;
         else if (dep === 'price')     startDate = DEFINITIVE_EARLIEST_DATES.price;
 
-        // If we have a start date and the target is BEFORE it, exclude this calc.
         if (startDate && targetDate < startDate) { return true; }
     }
     return false;
@@ -44,17 +41,14 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
 
 /**
  * Helper: Calculates the transitive closure of dependents (Blast Radius).
- * Returns the count of direct and total cascading dependents.
  */
 function calculateBlastRadius(targetCalcName, reverseGraph) {
     const impactSet = new Set();
     const queue = [targetCalcName];
     
-    // BFS Traversal
     while(queue.length > 0) {
         const current = queue.shift();
         const dependents = reverseGraph.get(current) || [];
-        
         dependents.forEach(child => {
             if (!impactSet.has(child)) {
                 impactSet.add(child);
@@ -66,46 +60,96 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
     return {
         directDependents: (reverseGraph.get(targetCalcName) || []).length,
         totalCascadingDependents: impactSet.size,
-        affectedCalculations: Array.from(impactSet).slice(0, 50) // Cap list size for storage safety
+        affectedCalculations: Array.from(impactSet).slice(0, 50)
     };
 }
 
+/**
+ * [NEW] Helper: Runs SimHash check with Caching and Registry Persistence.
+ */
+async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
+    const trueReRuns = [];
+    const stableUpdates = [];
+    
+    // Limit concurrency for simulations
+    const limit = pLimit(10);
+
+    const checks = candidates.map(item => limit(async () => {
+        try {
+            const manifest = manifestMap.get(item.name);
+            const stored = dailyStatus[item.name];
+
+            if (!stored || !stored.simHash || !manifest) {
+                trueReRuns.push(item);
+                return;
+            }
+
+            // 1. Check Cache first (Avoid re-simulating the same code for 100 different dates)
+            let newSimHash = simHashCache.get(manifest.hash);
+
+            // 2. If Miss, Run Simulation & Persist to Registry
+            if (!newSimHash) {
+                newSimHash = await SimRunner.run(manifest, manifestMap);
+                simHashCache.set(manifest.hash, newSimHash);
+                
+                // Write to Registry so Production Workers can find it without running SimRunner
+                // Fire-and-forget write to reduce latency
+                db.collection(SIMHASH_REGISTRY_COLLECTION).doc(manifest.hash).set({
+                    simHash: newSimHash,
+                    createdAt: new Date(),
+                    calcName: manifest.name
+                }).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
+            }
+
+            // 3. Compare
+            if (newSimHash === stored.simHash) {
+                // BEHAVIORAL MATCH: Code changed, but output is identical.
+                stableUpdates.push({
+                    ...item,
+                    reason: "Code Updated (Logic Stable)",
+                    simHash: newSimHash, // New SimHash (same as old)
+                    newHash: manifest.hash // New Code Hash
+                });
+            } else {
+                // BEHAVIORAL MISMATCH: Logic changed.
+                trueReRuns.push({
+                    ...item,
+                    reason: item.reason + ` [SimHash Mismatch]`,
+                    oldSimHash: stored.simHash,
+                    newSimHash: newSimHash
+                });
+            }
+        } catch (e) {
+            logger.log('WARN', `[BuildReporter] SimHash check failed for ${item.name}: ${e.message}`);
+            trueReRuns.push(item);
+        }
+    }));
+
+    await Promise.all(checks);
+    return { trueReRuns, stableUpdates };
+}
+
 /**
  * AUTO-RUN ENTRY POINT
- * Uses transactional locking to prevent race conditions.
  */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
-    const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}-${String(now.getMinutes()).padStart(2,'0')}-${String(now.getSeconds()).padStart(2,'0')}`;
-    
-    // Lock document specific to this version
+    const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}`;
     const lockRef = db.collection('computation_build_records').doc(`init_lock_v${packageVersion}`);
 
     try {
-        // Transaction: "Hey I am deploying" check
         const shouldRun = await db.runTransaction(async (t) => {
-            const doc   = await t.get(lockRef);
-            
-            if (doc.exists) { return false; } // Someone else beat us to it
-
-            // Claim the lock
-            t.set(lockRef, { 
-                status: 'IN_PROGRESS', 
-                startedAt: new Date(),
-                workerId: process.env.K_REVISION || 'unknown',
-                buildId: buildId
-            });
+            const doc = await t.get(lockRef);
+            if (doc.exists) { return false; }
+            t.set(lockRef, { status: 'IN_PROGRESS', startedAt: new Date(), buildId: buildId });
             return true;
         });
 
-        if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} is already being generated (Locked). Skipping.`); return; }
+        if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} locked. Skipping.`); return; }
 
-        logger.log('INFO', `[BuildReporter] 🚀 Lock Acquired. Running Pre-flight Report for v${packageVersion}...`);
-        
+        logger.log('INFO', `[BuildReporter] 🚀 Running Pre-flight Report for v${packageVersion}...`);
         await generateBuildReport(config, dependencies, manifest, 90, buildId);
-        
-        // Optional: Update lock to completed (fire-and-forget update)
         lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
 
     } catch (e) {
@@ -114,7 +158,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 }
 
 /**
- * Generates the report and saves to Firestore.
+ * Generates the report, writes to Firestore, AND FIXES STABLE UPDATES.
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -129,7 +173,9 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
-    // [IDEA 1] Build Reverse Dependency Graph (Parent -> Children)
+    // [OPTIMIZATION] Cache SimHashes across dates so we only calculate once per code version
+    const simHashCache = new Map();
+
     const reverseGraph = new Map();
     manifest.forEach(c => {
         const parentName = normalizeName(c.name);
@@ -142,17 +188,9 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
         }
     });
 
-    // Main Report Header
-    const reportHeader = {
-        buildId,
-        packageVersion: packageVersion, 
-        generatedAt: new Date().toISOString(),
-        summary: {}, 
-        _sharded: true 
-    };
+    const reportHeader = { buildId, packageVersion, generatedAt: new Date().toISOString(), summary: {}, _sharded: true };
 
-    let totalRun = 0;
-    let totalReRun = 0;
+    let totalRun = 0, totalReRun = 0, totalStable = 0;
     const detailWrites = []; 
 
     const limit = pLimit(20);
@@ -178,93 +216,89 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             const dailyStatus     = results[0];
             const availability    = results[1];
             const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
-            const rootDataStatus  = availability ? availability.status : { hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false };
+            const rootDataStatus  = availability ? availability.status : { hasPortfolio: false, hasHistory: false };
 
             const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
-            // ---------------------------------------------------------
-            // STRICT 5-CATEGORY MAPPING
-            // ---------------------------------------------------------
             const dateSummary = {
-                run:        [], // New / No Hash / "Runnable"
-                rerun:      [], // Hash Mismatch / Category Migration
-                blocked:    [], // Missing Data (Today) / Dependency Missing
-                impossible: [], // Missing Data (Historical) / Impossible Dependency
-                uptodate:   [], // Hash Match (Previously "Skipped")
-                
-                // Metadata for Verification
-                meta: {
-                    totalIncluded: 0,
-                    totalExpected: 0,
-                    match: false
-                }
+                run: [], rerun: [], stable: [], blocked: [], impossible: [], uptodate: [],
+                meta: { totalIncluded: 0, totalExpected: 0, match: false }
             };
 
-            // Calculate Expected Count (Computations in manifest that exist for this date)
             const expectedCount = manifest.filter(c => !isDateBeforeAvailability(dateStr, c)).length;
             dateSummary.meta.totalExpected = expectedCount;
 
-            // Helper to push only if date is valid for this specific calc
             const pushIfValid = (targetArray, item, extraReason = null) => {
                 const calcManifest = manifestMap.get(item.name);
-                if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) {
-                    return; // EXCLUDED: Date is before data exists
-                }
+                if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) return;
                 
                 const entry = { 
-                    name:   item.name, 
+                    name: item.name, 
                     reason: item.reason || extraReason,
-                    pass:   calcManifest ? calcManifest.pass : '?'
+                    pass: calcManifest ? calcManifest.pass : '?'
                 };
-
-                // [IDEA 1] If this is a Re-Run, calculate Blast Radius
                 if (targetArray === dateSummary.rerun) {
                     entry.impact = calculateBlastRadius(item.name, reverseGraph);
                 }
-
                 targetArray.push(entry);
             };
 
-            // 1. RUN (New)
+            // 1. RUN
             analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
 
-            // 2. RE-RUN (Hash Mismatch)
-            analysis.reRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Hash Mismatch"));
+            // 2. RE-RUN & STABLE Analysis (SimHash Integration)
+            if (analysis.reRuns.length > 0) {
+                // Pass simHashCache and db for registry writes
+                const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
+                
+                trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
+                stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Cosmetic Change"));
+
+                // [CRITICAL FIX] "Fix the Blast Radius"
+                // If updates are STABLE, we update the status NOW. 
+                // This implies: Code Hash changes, but Sim Hash stays same.
+                // The Dispatcher will see the new Code Hash in status matches the Manifest, so it won't dispatch.
+                if (stableUpdates.length > 0) {
+                    const updatesPayload = {};
+                    for (const stable of stableUpdates) {
+                        const m = manifestMap.get(stable.name);
+                        // We preserve the *existing* resultHash because the logic is proven stable.
+                        // We update the 'hash' to the NEW code hash.
+                        if (m && dailyStatus[stable.name]) {
+                            updatesPayload[stable.name] = {
+                                hash: m.hash, // New Code Hash
+                                simHash: stable.simHash, // Same Sim Hash
+                                resultHash: dailyStatus[stable.name].resultHash, // Same Result Hash
+                                dependencyResultHashes: dailyStatus[stable.name].dependencyResultHashes || {},
+                                category: m.category,
+                                composition: m.composition,
+                                lastUpdated: new Date()
+                            };
+                        }
+                    }
+                    // Perform the "Fix"
+                    if (Object.keys(updatesPayload).length > 0) {
+                        await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
+                        logger.log('INFO', `[BuildReporter] 🩹 Fixed ${Object.keys(updatesPayload).length} stable items for ${dateStr}. They will NOT re-run.`);
+                    }
+                }
+            }
 
-            // 3. BLOCKED (Temporary Issues)
+            // 3. BLOCKED / IMPOSSIBLE / UPTODATE
             analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
             analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
-
-            // 4. IMPOSSIBLE (Permanent Issues)
             analysis.impossible.forEach(item => pushIfValid(dateSummary.impossible, item));
-
-            // 5. UP-TO-DATE (Previously "Skipped")
             analysis.skipped.forEach(item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
 
-            // Calculate Included Count
-            const includedCount = dateSummary.run.length + 
-                                  dateSummary.rerun.length + 
-                                  dateSummary.blocked.length + 
-                                  dateSummary.impossible.length + 
-                                  dateSummary.uptodate.length;
-
+            // Meta stats
+            const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
+                                  dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
             dateSummary.meta.totalIncluded = includedCount;
-            dateSummary.meta.match         = (includedCount === expectedCount);
-
-            if (!dateSummary.meta.match) {
-                logger.log('WARN', `[BuildReporter] ⚠️ Mismatch on ${dateStr}: Expected ${expectedCount} but got ${includedCount}.`);
-            }
+            dateSummary.meta.match = (includedCount === expectedCount);
 
-            // QUEUE THE WRITE (Don't write yet)
-            const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
-            detailWrites.push({
-                ref: detailRef,
-                data: dateSummary
-            });
+            detailWrites.push({ ref: db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr), data: dateSummary });
 
-            return {
-                stats: { run: dateSummary.run.length, rerun: dateSummary.rerun.length }
-            };
+            return { stats: { run: dateSummary.run.length, rerun: dateSummary.rerun.length, stable: dateSummary.stable.length } };
 
         } catch (err) {
             logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
@@ -274,69 +308,21 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     const results = await Promise.all(processingPromises);
 
-    results.forEach(res => {
-        if (res) {
-            totalRun   += res.stats.run;
-            totalReRun += res.stats.rerun;
-        }
-    });
-
-    reportHeader.summary = { 
-        totalReRuns: totalReRun, 
-        totalNew: totalRun, 
-        scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
-    };
+    results.forEach(res => { if (res) { totalRun += res.stats.run; totalReRun += res.stats.rerun; totalStable += res.stats.stable; } });
 
-    // 1. Write the main report header (Specific Version)
-    const reportRef = db.collection('computation_build_records').doc(buildId);
-    await reportRef.set(reportHeader);
+    reportHeader.summary = { totalReRuns: totalReRun, totalNew: totalRun, totalStable: totalStable, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
 
-    // 2. Write Details (Protected & Parallelized)
-    // FIX: Using parallel individual writes instead of Batch to avoid DEADLINE_EXCEEDED
-    let detailsSuccess = true;
-    if (detailWrites.length > 0) {
-        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records (Parallel Strategy)...`);
-        
-        try {
-            // Concurrency limit of 15 to be safe
-            const writeLimit = pLimit(15);
-            const writePromises = detailWrites.map(w => writeLimit(() => 
-                w.ref.set(w.data).catch(e => {
-                    logger.log('WARN', `[BuildReporter] Failed to write detail for ${w.ref.path}: ${e.message}`);
-                    throw e;
-                })
-            ));
-            
-            await Promise.all(writePromises);
-            logger.log('INFO', `[BuildReporter] Successfully wrote all detail records.`);
-
-        } catch (detailErr) {
-            detailsSuccess = false;
-            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write some details. Report Header is preserved.`, detailErr);
-        }
-    }
+    await db.collection('computation_build_records').doc(buildId).set(reportHeader);
+    
+    // Parallel write details
+    const writeLimit = pLimit(15);
+    await Promise.all(detailWrites.map(w => writeLimit(() => w.ref.set(w.data))));
 
-    // 3. Update 'latest' pointer
-    // This runs regardless of detail write success/failure
-    const latestMetadata = { 
-        ...reportHeader, 
-        note: detailsSuccess 
-            ? "Latest build report pointer (See subcollection for details)." 
-            : "Latest build report pointer (WARNING: Partial detail records due to write error)."
-    };
+    await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer." });
     
-    try {
-        await db.collection('computation_build_records').doc('latest').set(latestMetadata);
-        logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}. Pointer Updated.`);
-    } catch (pointerErr) {
-        logger.log('FATAL', `[BuildReporter] Failed to update 'latest' pointer!`, pointerErr);
-    }
+    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, Stable (Fixed): ${totalStable}, New: ${totalRun}.`);
     
-    return {
-        success: true,
-        reportId: buildId,
-        summary: reportHeader.summary
-    };
+    return { success: true, reportId: buildId, summary: reportHeader.summary };
 }
 
 module.exports = { ensureBuildReport, generateBuildReport };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.281",
+  "version": "1.0.282",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [