bulltrackers-module 1.0.211 → 1.0.213

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,158 +1,169 @@
-/**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
- * FIXED: 'runDateComputation' now executes ALL calculation types (Standard, Meta, AND Price).
- */
-
-const {
-    groupByPass,
-    checkRootDataAvailability,
-    fetchExistingResults,
-    fetchComputationStatus,
-    updateComputationStatus,
-    runStandardComputationPass,
-    runMetaComputationPass,
-    checkRootDependencies,
-    runBatchPriceComputation
-} = require('./orchestration_helpers.js');
-
-const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
-
-const PARALLEL_BATCH_SIZE = 7;
-
-/**
- * LEGACY / MANUAL RUNNER
- * (Kept for backward compatibility if you run the old HTTP endpoint directly)
- */
-async function runComputationPass(config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
-
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
-
-    // Hardcoded earliest dates
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
-
-    const passes          = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || [];
-
-    if (!calcsInThisPass.length)
-        return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-
-    const passEarliestDate = earliestDates.absoluteEarliest;
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
-
-    // Legacy Batch Optimization for Price (Only used in legacy loop)
-    const priceBatchCalcs = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
-    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
-
-    if (priceBatchCalcs.length > 0) {
-        try {
-            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); // Simplified for legacy
-        } catch (e) { logger.log('ERROR', 'Legacy Batch Price failed', e); }
-    }
-
-    if (standardAndOtherMetaCalcs.length === 0) return;
-
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
-        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(dateStr => runDateComputation(dateStr, passToRun, standardAndOtherMetaCalcs, config, dependencies, computationManifest)));
-    }
-}
-
-/**
- * UPDATED: Isolated function to run computations for a single date.
- * Used by the Pub/Sub Worker.
- */
-async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-
-    // 1. Fetch Status for THIS specific date only
-    const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-
-    // Helper: Check status
-    const shouldRun = (calc) => {
-        const cName = normalizeName(calc.name);
-        if (dailyStatus[cName] === true) return false;
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
-            if (missing.length > 0) return false;
-        }
-        return true;
-    };
-
-    // --- FIX: Run ALL calc types (Standard, Meta, Price) ---
-    const calcsToAttempt = calcsInThisPass.filter(shouldRun);
-
-    if (!calcsToAttempt.length) return null;
-
-    // 2. Check Root Data Availability
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-
-    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-    if (!rootData) {
-        logger.log('INFO', `[DateRunner] Root data missing for ${dateStr}. Skipping.`);
-        return null;
-    }
-
-    // 3. Filter again based on Root Data availability
-    const runnableCalcs = calcsToAttempt.filter(c => checkRootDependencies(c, rootData.status).canRun);
-
-    if (!runnableCalcs.length) return null;
-
-    // Split into Standard (Streaming) and Meta (Once-Per-Day/Price)
-    const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
-    // Note: Meta includes Price calcs in this flow
-    const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
-
-    logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
-
-    const dateUpdates = {};
-
-    try {
-        const calcsRunning = [...standardToRun, ...metaToRun];
-
-        // Fetch dependencies (results from this day or yesterday)
-        const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr     = prevDate.toISOString().slice(0, 10);
-        const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
-
-        if (standardToRun.length) {
-            const updates = await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false);
-            Object.assign(dateUpdates, updates);
-        }
-        if (metaToRun.length) {
-            // runMetaComputationPass uses the Controller, which handles Price Sharding logic internally for single dates.
-            const updates = await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false);
-            Object.assign(dateUpdates, updates);
-        }
-    } catch (err) {
-        logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-        [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
-        throw err; // Re-throw to trigger Pub/Sub retry
-    }
-
-    if (Object.keys(dateUpdates).length > 0) {
-        await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
-    }
-
-    return { date: dateStr, updates: dateUpdates };
-}
-
+/**
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
+ * FIXED: 'storedStatus.substring' crash and 'missing dependency' log clarity.
+ */
+
+const {
+    groupByPass,
+    checkRootDataAvailability,
+    fetchExistingResults,
+    fetchComputationStatus,
+    updateComputationStatus,
+    runStandardComputationPass,
+    runMetaComputationPass,
+    checkRootDependencies,
+    runBatchPriceComputation
+} = require('./orchestration_helpers.js');
+
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+
+const PARALLEL_BATCH_SIZE = 7;
+
+async function runComputationPass(config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
+
+    const earliestDates = {
+        portfolio: new Date('2025-09-25T00:00:00Z'),
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
+    };
+    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
+
+    const passes          = groupByPass(computationManifest);
+    const calcsInThisPass = passes[passToRun] || [];
+
+    if (!calcsInThisPass.length)
+        return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+
+    const passEarliestDate = earliestDates.absoluteEarliest;
+    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
+
+    const priceBatchCalcs = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
+    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
+
+    if (priceBatchCalcs.length > 0) {
+        try {
+            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); 
+        } catch (e) { logger.log('ERROR', 'Legacy Batch Price failed', e); }
+    }
+
+    if (standardAndOtherMetaCalcs.length === 0) return;
+
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
+        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+        await Promise.all(batch.map(dateStr => runDateComputation(dateStr, passToRun, standardAndOtherMetaCalcs, config, dependencies, computationManifest)));
+    }
+}
+
+async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+
+    const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+    // Filter AND Log reason for skipping
+    const calcsToAttempt = [];
+    
+    for (const calc of calcsInThisPass) {
+        const cName = normalizeName(calc.name);
+        const storedStatus = dailyStatus[cName];
+        const currentHash = calc.hash;
+
+        // 1. Dependency Check
+        if (calc.dependencies && calc.dependencies.length > 0) {
+            const missing = calc.dependencies.filter(depName => !dailyStatus[normalizeName(depName)]);
+            if (missing.length > 0) {
+                // Too noisy to log every skip, but useful for debugging if needed. 
+                // Only logging if it's NOT a bulk skip.
+                // logger.log('TRACE', `[Skip] ${cName} missing deps: ${missing.join(', ')}`);
+                continue; 
+            }
+        }
+
+        // 2. Logic A: No previous run
+        if (!storedStatus) {
+            logger.log('INFO', `[Versioning] ${cName}: New run needed (No prior status).`);
+            calcsToAttempt.push(calc);
+            continue;
+        }
+
+        // 3. Logic B: Hash Mismatch
+        // FIX: Ensure storedStatus is a string before calling substring
+        if (typeof storedStatus === 'string' && currentHash && storedStatus !== currentHash) {
+            logger.log('INFO', `[Versioning] ${cName}: Code Changed. (Old: ${storedStatus.substring(0,6)}... New: ${currentHash.substring(0,6)}...)`);
+            calcsToAttempt.push(calc);
+            continue;
+        }
+
+        // 4. Logic C: Upgrade Legacy Boolean -> Hash
+        if (storedStatus === true && currentHash) {
+            logger.log('INFO', `[Versioning] ${cName}: Upgrading legacy status to Hash.`);
+            calcsToAttempt.push(calc);
+            continue;
+        }
+    }
+
+    if (!calcsToAttempt.length) return null;
+
+    const earliestDates = {
+        portfolio: new Date('2025-09-25T00:00:00Z'),
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
+    };
+
+    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+    if (!rootData) {
+        logger.log('INFO', `[DateRunner] Root data missing for ${dateStr}. Skipping.`);
+        return null;
+    }
+
+    const runnableCalcs = calcsToAttempt.filter(c => checkRootDependencies(c, rootData.status).canRun);
+
+    if (!runnableCalcs.length) return null;
+
+    const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
+    const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
+
+    logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
+
+    const dateUpdates = {};
+
+    try {
+        const calcsRunning = [...standardToRun, ...metaToRun];
+
+        const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
+        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr     = prevDate.toISOString().slice(0, 10);
+        const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
+
+        if (standardToRun.length) {
+            const updates = await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false);
+            Object.assign(dateUpdates, updates);
+        }
+        if (metaToRun.length) {
+            const updates = await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false);
+            Object.assign(dateUpdates, updates);
+        }
+    } catch (err) {
+        logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
+        [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        throw err; 
+    }
+
+    if (Object.keys(dateUpdates).length > 0) {
+        await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
+    }
+
+    return { date: dateStr, updates: dateUpdates };
+}
+
 module.exports = { runComputationPass, runDateComputation };

package/functions/computation-system/helpers/computation_worker.js

@@ -1,86 +1,86 @@
-/**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_worker.js
- * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * FIXED: Added robust payload parsing to handle Cloud Functions Gen 2 (CloudEvents).
- */
-
-const { runDateComputation } = require('./computation_pass_runner.js');
-const { groupByPass } = require('./orchestration_helpers.js');
-
-/**
- * Handles a single Pub/Sub message for a computation task.
- * Supports both Gen 1 (Message) and Gen 2 (CloudEvent) formats.
- */
-async function handleComputationTask(message, config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-
-    let data;
-    try {
-        // 1. Handle Cloud Functions Gen 2 (CloudEvent)
-        // Structure: event.data.message.data (base64)
-        if (message.data && message.data.message && message.data.message.data) {
-            const buffer = Buffer.from(message.data.message.data, 'base64');
-            data = JSON.parse(buffer.toString());
-        }
-        // 2. Handle Cloud Functions Gen 1 / Legacy PubSub
-        // Structure: message.data (base64) or message.json
-        else if (message.data && typeof message.data === 'string') {
-            const buffer = Buffer.from(message.data, 'base64');
-            data = JSON.parse(buffer.toString());
-        }
-        // 3. Handle Direct JSON (Test harness or simulator)
-        else if (message.json) {
-            data = message.json;
-        }
-        // 4. Fallback: Assume message is the payload
-        else {
-            data = message;
-        }
-    } catch (parseError) {
-        logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message });
-        return;
-    }
-
-    try {
-        // Validate Action
-        if (!data || data.action !== 'RUN_COMPUTATION_DATE') {
-            // Only log if data exists but action is wrong, prevents log spam on empty messages
-            if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`);
-            return;
-        }
-
-        const { date, pass } = data;
-
-        if (!date || !pass) {
-            logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`);
-            return;
-        }
-
-        logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
-
-        // Resolve calculations for this pass
-        const passes = groupByPass(computationManifest);
-        const calcsInThisPass = passes[pass] || [];
-
-        if (!calcsInThisPass.length) {
-            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`);
-            return;
-        }
-
-        // Execute the computation for this specific date
-        // The runner internally checks dependencies (Pass 1, 2, 3 status) and skips if not ready.
-        const result = await runDateComputation(date, pass, calcsInThisPass, config, dependencies, computationManifest);
-
-        if (result) {
-            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
-        } else {
-            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - Skipped (Dependencies missing or already done).`);
-        }
-
-    } catch (err) {
-        logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack });
-        throw err; // Re-throw to trigger Pub/Sub retry
-    }
-}
-
+/**
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_worker.js
+ * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
+ * FIXED: Added robust payload parsing to handle Cloud Functions Gen 2 (CloudEvents).
+ */
+
+const { runDateComputation } = require('./computation_pass_runner.js');
+const { groupByPass } = require('./orchestration_helpers.js');
+
+/**
+ * Handles a single Pub/Sub message for a computation task.
+ * Supports both Gen 1 (Message) and Gen 2 (CloudEvent) formats.
+ */
+async function handleComputationTask(message, config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+
+    let data;
+    try {
+        // 1. Handle Cloud Functions Gen 2 (CloudEvent)
+        // Structure: event.data.message.data (base64)
+        if (message.data && message.data.message && message.data.message.data) {
+            const buffer = Buffer.from(message.data.message.data, 'base64');
+            data = JSON.parse(buffer.toString());
+        }
+        // 2. Handle Cloud Functions Gen 1 / Legacy PubSub
+        // Structure: message.data (base64) or message.json
+        else if (message.data && typeof message.data === 'string') {
+            const buffer = Buffer.from(message.data, 'base64');
+            data = JSON.parse(buffer.toString());
+        }
+        // 3. Handle Direct JSON (Test harness or simulator)
+        else if (message.json) {
+            data = message.json;
+        }
+        // 4. Fallback: Assume message is the payload
+        else {
+            data = message;
+        }
+    } catch (parseError) {
+        logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message });
+        return;
+    }
+
+    try {
+        // Validate Action
+        if (!data || data.action !== 'RUN_COMPUTATION_DATE') {
+            // Only log if data exists but action is wrong, prevents log spam on empty messages
+            if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`);
+            return;
+        }
+
+        const { date, pass } = data;
+
+        if (!date || !pass) {
+            logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`);
+            return;
+        }
+
+        logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
+
+        // Resolve calculations for this pass
+        const passes = groupByPass(computationManifest);
+        const calcsInThisPass = passes[pass] || [];
+
+        if (!calcsInThisPass.length) {
+            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`);
+            return;
+        }
+
+        // Execute the computation for this specific date
+        // The runner internally checks dependencies (Pass 1, 2, 3 status) and skips if not ready.
+        const result = await runDateComputation(date, pass, calcsInThisPass, config, dependencies, computationManifest);
+
+        if (result) {
+            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
+        } else {
+            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - Skipped (Dependencies missing or already done).`);
+        }
+
+    } catch (err) {
+        logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack });
+        throw err; // Re-throw to trigger Pub/Sub retry
+    }
+}
+
 module.exports = { handleComputationTask };