bulltrackers-module 1.0.237 → 1.0.239

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,7 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Detects Category changes to trigger migration/cleanup.
- * FIX: Decouples migration detection from hash verification to handle simultaneous code & category changes.
+ * UPDATED: Exports analyzeDateExecution for Build Reporting tools.
  */
 const { normalizeName }                                    = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -124,7 +123,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             report.runnable.push(calc);
         } else if (storedHash !== currentHash) {
             // Hash Mismatch (Code Changed).
-            // Pass migration info here too, in case category ALSO changed.
             report.reRuns.push({ 
                 name: cName, 
                 oldHash: storedHash, 
@@ -268,4 +266,4 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     return { date: dateStr, updates: dateUpdates };
 }
 
-module.exports = { runDateComputation, groupByPass };
+module.exports = { runDateComputation, groupByPass, analyzeDateExecution };

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,18 +1,18 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Dispatches computation tasks to Pub/Sub for scalable execution.
- * REFACTORED: Now uses WorkflowOrchestrator for helper functions.
+ * PURPOSE: Dispatches granular computation tasks (1 task = 1 calculation/date).
+ * UPDATED: Implements "Atomic Task Dispatch" for maximum reliability.
  */
 
-const { getExpectedDateStrings } = require('../utils/utils.js');
-const { groupByPass }            = require('../WorkflowOrchestrator.js');
-const { PubSubUtils }            = require('../../core/utils/pubsub_utils');
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+const { groupByPass }                           = require('../WorkflowOrchestrator.js');
+const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
 
 const TOPIC_NAME = 'computation-tasks';
 
 /**
  * Dispatches computation tasks for a specific pass.
- * Instead of running them, it queues them in Pub/Sub.
+ * Generates one Pub/Sub message per calculation per date.
  */
 async function dispatchComputationPass(config, dependencies, computationManifest) {
     const { logger }  = dependencies;
@@ -21,17 +21,18 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     
     if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
 
-    // 1. Validate Pass Existence
+    // 1. Get Calculations for this Pass
     const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    const calcNames = calcsInThisPass.map(c => c.name).join(', ');
-    logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun}.`);
-    logger.log('INFO', `[Dispatcher] Included Calculations: [${calcNames}]`);
+    const calcNames = calcsInThisPass.map(c => c.name);
+    logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun} (Granular Mode).`);
+    logger.log('INFO', `[Dispatcher] Target Calculations: [${calcNames.join(', ')}]`);
 
     // 2. Determine Date Range
+    // (You can make this dynamic or config-driven)
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
         history:   new Date('2025-11-05T00:00:00Z'),
@@ -44,35 +45,34 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    logger.log('INFO', `[Dispatcher] Dispatches checks for ${allExpectedDates.length} dates (${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length - 1]}). Workers will validate dependencies.`);
-
-    // 3. Dispatch Messages
-    let dispatchedCount = 0;
-    const BATCH_SIZE    = 50;
-
-    // We can publish in parallel batches
-    const chunks = [];
-    for (let i = 0; i < allExpectedDates.length; i += BATCH_SIZE) { chunks.push(allExpectedDates.slice(i, i + BATCH_SIZE)); }
-
-    for (const chunk of chunks) {
-        const messages = chunk.map(dateStr => ({
-            json: {
-                action: 'RUN_COMPUTATION_DATE',
+    // 3. Generate Granular Tasks (Cartesian Product: Dates x Calculations)
+    const allTasks = [];
+    
+    for (const dateStr of allExpectedDates) {
+        for (const calc of calcsInThisPass) {
+            allTasks.push({
+                action: 'RUN_COMPUTATION_DATE', // Maintained for compatibility
                 date: dateStr,
                 pass: passToRun,
+                computation: normalizeName(calc.name), // CRITICAL: Target specific calc
                 timestamp: Date.now()
-            }
-        }));
-
-        try {
-            await pubsubUtils.publishMessageBatch(TOPIC_NAME, messages);
-            dispatchedCount += messages.length;
-            logger.log('INFO', `[Dispatcher] Dispatched batch of ${messages.length} tasks.`);
-        } catch (err) { logger.log('ERROR', `[Dispatcher] Failed to dispatch batch: ${err.message}`); }
+            });
+        }
     }
 
-    logger.log('INFO', `[Dispatcher] Finished. Dispatched ${dispatchedCount} checks for Pass ${passToRun}.`);
-    return { dispatched: dispatchedCount };
+    logger.log('INFO', `[Dispatcher] Generated ${allTasks.length} atomic tasks (${allExpectedDates.length} days × ${calcsInThisPass.length} calcs).`);
+
+    // 4. Batch Dispatch
+    // We send tasks in batches to Pub/Sub to be efficient, 
+    // but the WORKERS will process them individually.
+    await pubsubUtils.batchPublishTasks(dependencies, {
+        topicName: TOPIC_NAME,
+        tasks: allTasks,
+        taskType: `computation-pass-${passToRun}`,
+        maxPubsubBatchSize: 100 // Safe batch size
+    });
+
+    return { dispatched: allTasks.length };
 }
 
 module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_worker.js

@@ -1,34 +1,27 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * UPDATED: Instantiates the internal StructuredLogger to replace the generic injected logger.
+ * UPDATED: Supports Granular Execution (Single Calculation filtering).
  */
 
 const { runDateComputation, groupByPass } = require('../WorkflowOrchestrator.js');
 const { getManifest }                     = require('../topology/ManifestLoader');
 const { StructuredLogger }                = require('../logger/logger'); 
+const { normalizeName }                   = require('../utils/utils');
 
 // 1. IMPORT CALCULATIONS
-// We import the specific package containing the strategies (gem, pyro, core, etc.)
 let calculationPackage;
 try {
-    // Primary: Try to load from the installed npm package
     calculationPackage = require('aiden-shared-calculations-unified');
 } catch (e) {
     console.error("FATAL: Could not load 'aiden-shared-calculations-unified'. Ensure it is installed or linked.");
     throw e;
 }
 
-
-// The package exports { calculations: { ... }, utils: { ... } }
 const calculations = calculationPackage.calculations;
 
 /**
  * Handles a single Pub/Sub message for a computation task.
- * Supports both Gen 1 (Message) and Gen 2 (CloudEvent) formats.
- * @param {object} message - The Pub/Sub message payload.
- * @param {object} config - System configuration.
- * @param {object} dependencies - Injected dependencies (db, generic logger, etc.).
  */
 async function handleComputationTask(message, config, dependencies) {
     
@@ -45,7 +38,7 @@ async function handleComputationTask(message, config, dependencies) {
         logger: systemLogger
     };
 
-    const { logger } = runDependencies; // Use this for local logging
+    const { logger } = runDependencies;
 
     // 4. LAZY LOAD MANIFEST
     let computationManifest;
@@ -86,19 +79,37 @@ async function handleComputationTask(message, config, dependencies) {
             return; 
         }
         
-        const { date, pass } = data;
+        const { date, pass, computation } = data; // Extract 'computation'
+        
         if (!date || !pass) { 
             logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`); 
             return; 
         }
         
-        logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
-        
+        // Load Full Pass
         const passes = groupByPass(computationManifest);
-        const calcsInThisPass = passes[pass] || [];
+        let calcsInThisPass = passes[pass] || [];
         
+        // --- GRANULAR FILTERING ---
+        if (computation) {
+            const targetName = normalizeName(computation);
+            const targetCalc = calcsInThisPass.find(c => normalizeName(c.name) === targetName);
+            
+            if (!targetCalc) {
+                logger.log('WARN', `[Worker] Targeted computation '${computation}' not found in Pass ${pass}. Skipping.`);
+                return;
+            }
+            
+            // We run ONLY this calculation
+            calcsInThisPass = [targetCalc];
+            logger.log('INFO', `[Worker] Granular Mode: Running ONLY ${targetCalc.name} for ${date}`);
+        } else {
+            logger.log('INFO', `[Worker] Bulk Mode: Running ${calcsInThisPass.length} calculations for ${date}`);
+        }
+        // ---------------------------
+
         if (!calcsInThisPass.length) { 
-            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`); 
+            logger.log('WARN', `[Worker] No calculations found to run.`); 
             return; 
         }
         
@@ -112,14 +123,15 @@ async function handleComputationTask(message, config, dependencies) {
         );
         
         if (result && result.updates && Object.keys(result.updates).length > 0) { 
-            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates).length}`);
+            logger.log('INFO', `[Worker] Success ${date}. Updated: ${Object.keys(result.updates).join(', ')}`);
         } else { 
-            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - No updates.`); 
+            // In Granular Mode, this is common (e.g. if hash matched)
+            logger.log('INFO', `[Worker] Completed ${date} - No DB Writes (Up to date or skipped).`); 
         }
         
     } catch (err) { 
         logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); 
-        throw err;
+        throw err; // Throwing ensures Pub/Sub retries this specific computation
     }
 }
 

package/functions/computation-system/tools/BuildReporter.js

@@ -0,0 +1,161 @@
+/**
+ * @fileoverview Build Reporter & Auto-Runner.
+ * Generates a "Pre-Flight" report of what the computation system WILL do.
+ * Simulates execution logic (Hash Mismatches) assuming perfect data availability.
+ */
+
+const { analyzeDateExecution } = require('../WorkflowOrchestrator');
+const { fetchComputationStatus } = require('../persistence/StatusRepository');
+const { normalizeName, getExpectedDateStrings } = require('../utils/utils');
+const { FieldValue } = require('@google-cloud/firestore');
+
+// Attempt to load package.json to get version. Path depends on where this is invoked.
+let packageVersion = '1.0.237';
+
+
+/**
+ * AUTO-RUN ENTRY POINT
+ * Checks if a report for the current version exists. If not, runs it.
+ * Designed to be called "Fire-and-Forget" at system init.
+ */
+async function ensureBuildReport(config, dependencies, manifest) {
+    const { db, logger } = dependencies;
+    const buildId = `v${packageVersion}`; 
+    const reportRef = db.collection('computation_build_records').doc(buildId);
+
+    try {
+        const doc = await reportRef.get();
+        if (doc.exists) {
+            logger.log('INFO', `[BuildReporter] ✅ Pre-flight check for ${buildId} already exists. Skipping.`);
+            return;
+        }
+        
+        logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${buildId}). Auto-running Pre-flight Report...`);
+        // We do NOT await this in the main thread if called during init, 
+        // but here we are in an async function so we proceed.
+        await generateBuildReport(config, dependencies, manifest, 7, buildId);
+        
+    } catch (e) {
+        logger.log('ERROR', `[BuildReporter] Auto-run check failed: ${e.message}`);
+    }
+}
+
+/**
+ * Generates the report and saves to Firestore.
+ * @param {object} config 
+ * @param {object} dependencies 
+ * @param {Array} manifest 
+ * @param {number} daysBack - Days to simulate (default 7)
+ * @param {string} customBuildId - Optional ID override
+ */
+async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
+    const { db, logger } = dependencies;
+    const buildId = customBuildId || `manual_${Date.now()}`;
+    
+    logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days)...`);
+
+    // 1. Determine Date Range
+    const today = new Date();
+    const startDate = new Date();
+    startDate.setDate(today.getDate() - daysBack);
+    
+    // We check UP TO yesterday usually, as today might be partial. 
+    // But let's check today too to see immediate effects.
+    const datesToCheck = getExpectedDateStrings(startDate, today);
+    const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
+    
+    const reportData = {
+        buildId,
+        version: packageVersion,
+        generatedAt: new Date().toISOString(),
+        summary: {},
+        dates: {}
+    };
+
+    let totalReRuns = 0;
+    let totalNew = 0;
+
+    // 2. Iterate Dates & Simulate
+    for (const dateStr of datesToCheck) {
+        // A. Fetch REAL status from DB (What ran previously?)
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+        // B. MOCK Root Data Availability
+        // We force this to TRUE to prove logic changes.
+        // We want to know: "IF data exists, would this run?"
+        const mockRootDataStatus = {
+            hasPortfolio: true,
+            hasInsights: true,
+            hasSocial: true,
+            hasHistory: true,
+            hasPrices: true
+        };
+
+        // C. Run Logic Analysis
+        // Pass ENTIRE manifest to see global state
+        const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
+
+        // D. Format Findings
+        const dateSummary = {
+            willRun: [],
+            willReRun: [],
+            blocked: []
+            // We omit 'skipped' to save space unless explicitly IMPOSSIBLE
+        };
+
+        // -- Runnable (New) --
+        analysis.runnable.forEach(item => {
+            dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
+        });
+
+        // -- Re-Runs (Hash Mismatch / Migration) --
+        analysis.reRuns.forEach(item => {
+            let reason = "Hash Mismatch";
+            let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+            
+            if (item.previousCategory) {
+                reason = "Migration";
+                details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
+            }
+
+            dateSummary.willReRun.push({ name: item.name, reason, details });
+        });
+
+        // -- Blocked / Impossible --
+        analysis.impossible.forEach(item => {
+             dateSummary.blocked.push({ name: item.name, reason: item.reason }); // e.g. "Permanently Impossible"
+        });
+        analysis.failedDependency.forEach(item => {
+             dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
+        });
+
+        // Only add date to report if something interesting is happening
+        if (dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length) {
+            reportData.dates[dateStr] = dateSummary;
+            totalNew += dateSummary.willRun.length;
+            totalReRuns += dateSummary.willReRun.length;
+        }
+    }
+
+    reportData.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
+
+    // 3. Store Report
+    const reportRef = db.collection('computation_build_records').doc(buildId);
+    await reportRef.set(reportData);
+
+    // 4. Update 'latest' pointer
+    await db.collection('computation_build_records').doc('latest').set({
+        ...reportData,
+        note: "Latest build report pointer."
+    });
+
+    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRuns}, New: ${totalNew}.`);
+    
+    return {
+        success: true,
+        reportId: buildId,
+        summary: reportData.summary
+    };
+}
+
+module.exports = { ensureBuildReport, generateBuildReport };

package/index.js

@@ -30,6 +30,9 @@ const { build: buildManifest }                            = require('./functions
 const { runDateComputation: runComputationPass }          = require('./functions/computation-system/WorkflowOrchestrator');
 const { dispatchComputationPass }                         = require('./functions/computation-system/helpers/computation_dispatcher');
 const { handleComputationTask }                           = require('./functions/computation-system/helpers/computation_worker');
+// [NEW] Import Report Tools
+const { ensureBuildReport, generateBuildReport }          = require('./functions/computation-system/tools/BuildReporter');
+
 const dataLoader                                          = require('./functions/computation-system/utils/data_loader');
 const computationUtils                                    = require('./functions/computation-system/utils/utils');
 
@@ -87,6 +90,9 @@ const computationSystem = {
   dataLoader,
   computationUtils,
   buildManifest,
+  // [NEW] Export Tools
+  ensureBuildReport,
+  generateBuildReport,
 };
 
 const api = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.237",
+  "version": "1.0.239",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [