bulltrackers-module 1.0.693 → 1.0.695

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -11,6 +11,8 @@ const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const { runFinalSweepCheck } = require('../tools/FinalSweepReporter');
+const { resolveDependencyChain } = require('./on_demand_helpers'); 
+const { checkRootDependencies } = require('../data/AvailabilityChecker');
 // 1. IMPORT SNAPSHOT SERVICE
 const { generateDailySnapshots } = require('../services/SnapshotService');
 const crypto = require('crypto');
@@ -318,6 +320,16 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
     const manifestItem = computationManifest.find(c => normalizeName(c.name) === normalizeName(computationName));
     if (!manifestItem) throw new Error(`Computation '${computationName}' not found.`);
 
+    // --- STEP 1: RESOLVE FULL ANCESTRY ---
+    // We get the full chain of dependencies (ancestors) for the target.
+    // This includes the target itself and all upstream computations.
+    const chainPasses = resolveDependencyChain(computationName, computationManifest);
+    const allAncestors = chainPasses.flatMap(p => p.computations); // Flat list of all required names
+    
+    // Create a map for quick lookup of ancestor manifests
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+
+    // --- STEP 2: DETERMINE CANDIDATE DATES ---
     let candidateDates = [];
     if (dateInput) {
         candidateDates = [dateInput];
@@ -327,50 +339,85 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
         candidateDates = getExpectedDateStrings(earliest.absoluteEarliest, new Date());
     }
     
-    logger.log('INFO', `[ForceRun] Checking ${candidateDates.length} candidate dates for runnability...`);
+    logger.log('INFO', `[ForceRun] Checking ${candidateDates.length} candidate dates for runnability (Deep Check)...`);
 
     const runnableDates = [];
     const skippedDates = [];
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    
-    const targetComp = { ...manifestItem, schedule: null }; 
     const targetComputationNormalized = normalizeName(computationName);
+    // Remove schedule constraints for the force run assessment
+    const targetComp = { ...manifestItem, schedule: null }; 
 
     for (const date of candidateDates) {
+        // --- STEP 3: DEEP ROOT CHECK ---
+        // Before even asking if the *target* is runnable, we ask:
+        // "Are the raw ingredients available for the ENTIRE chain?"
+        
+        // Fetch Root Data Status for this date once
+        const availability = await checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+        const rootStatus = availability ? availability.status : null;
+
+        if (!rootStatus) {
+            skippedDates.push({ date, reason: 'Availability Index Missing' });
+            continue;
+        }
+
+        let deepImpossibleReason = null;
+        
+        // Check EVERY ancestor's root requirements
+        for (const ancName of allAncestors) {
+            const ancManifest = manifestMap.get(normalizeName(ancName));
+            if (!ancManifest) continue;
+            
+            // Re-use the standard checker for each ancestor
+            const ancCheck = checkRootDependencies(ancManifest, rootStatus);
+            
+            if (!ancCheck.canRun) {
+                // If an ancestor cannot exist, the target cannot exist.
+                deepImpossibleReason = `Ancestor '${ancName}' is missing roots: ${ancCheck.missing.join(', ')}`;
+                break;
+            }
+        }
+
+        if (deepImpossibleReason) {
+            // Skip this date entirely - it is strictly impossible.
+            skippedDates.push({ date, reason: deepImpossibleReason });
+            continue;
+        }
+
+        // --- STEP 4: STANDARD RUNNABILITY ---
+        // If deep roots are fine, we proceed to the standard check.
+        // This handles logic like "Waiting for yesterday" or "Already Complete"
         const result = await assessDateRunnability(date, [targetComp], config, dependencies, manifestMap);
-        if (!result) {
-            skippedDates.push({ date, reason: 'Root data unavailable' });
-            continue; 
+        
+        if (!result) { 
+             skippedDates.push({ date, reason: 'Assessment Failed' }); 
+             continue; 
         }
 
         const { report } = result;
         const isRunnable = report.runnable.some(t => normalizeName(t.name) === targetComputationNormalized);
         const needsReRun = report.reRuns.some(t => normalizeName(t.name) === targetComputationNormalized);
         const hasFailedDep = report.failedDependency.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const isImpossible = report.impossible.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const isBlocked = report.blocked.some(t => normalizeName(t.name) === targetComputationNormalized);
         const isSkipped = report.skipped.some(t => normalizeName(t.name) === targetComputationNormalized);
         
-        // For force runs: treat skipped computations (already stored with valid hash) as runnable
-        // They will overwrite with the same result, which is fine for testing
-        // Only mark as impossible if root data or dependencies don't exist at all
-        if (isImpossible) {
-            skippedDates.push({ date, reason: report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Impossible' });
-        } else if (isRunnable || needsReRun || hasFailedDep || isSkipped) {
-            // Runnable, needs re-run, has failed deps (but not impossible), or skipped (already stored)
-            // All of these are runnable for force runs - will overwrite existing results if needed
+        // NOTE: hasFailedDep is ALLOWED here because we are "Forcing" it.
+        // We know the roots exist (checked above), so the missing dependency is likely just 
+        // "Not Computed Yet", which is exactly what the user wants to fix manually.
+        if (isRunnable || needsReRun || hasFailedDep || isSkipped) {
             runnableDates.push(date);
-        } else if (isBlocked) {
-            // Blocked usually means waiting for data - for force runs, if root data exists, still runnable
-            // Only skip if truly impossible (handled above)
+        } else if (report.blocked.length > 0) {
+            // Blocked usually means "Waiting for yesterday"
+            // For force runs, we often want to override this, but if it's strictly blocked 
+            // by logic, we might still count it. Usually, we treat it as runnable.
             runnableDates.push(date);
         } else {
-            // Unknown state - for force runs, if root data exists (which it does, since result is not null), treat as runnable
-            logger.log('WARN', `[ForceRun] Computation ${computationName} in unknown state for ${date}, treating as runnable`);
-            runnableDates.push(date);
+             const imp = report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized);
+             skippedDates.push({ date, reason: imp ? imp.reason : 'Unknown State' });
         }
     }
     
+    // ... (Remainder of the function remains the same: dispatching tasks) ...
+    
     logger.log('INFO', `[ForceRun] ✅ Found ${runnableDates.length} runnable dates out of ${candidateDates.length} candidates`);
 
     if (runnableDates.length === 0) {
@@ -384,13 +431,12 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
         };
     }
     
-    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${runnableDates.length} runnable dates. Pass: ${manifestItem.pass}`);
-
+    // Dispatch Logic
     const topic = (reqBody.resources === 'high-mem') 
         ? (config.computationTopicHighMem || 'computation-tasks-highmem')
         : (config.computationTopicStandard || 'computation-tasks');
 
-    const dispatchId = crypto.randomUUID();
+    const dispatchId = require('crypto').randomUUID();
     const tasks = runnableDates.map(date => 
         createTaskPayload(manifestItem, date, manifestItem.pass || "1", dispatchId, reqBody.resources, 'MANUAL_FORCE_API')
     );

package/functions/computation-system/persistence/ResultCommitter.js

@@ -420,7 +420,8 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
             }
             
             const rootUpdate = updates.find(u => u.ref.path === docRef.path && u.type !== 'DELETE');
-            if (rootUpdate) { rootUpdate.options = { merge: rootMergeOption }; }
+            // FIX: Always use merge: false to ensure old fields (like _compressed/payload) are wiped
+            if (rootUpdate) { rootUpdate.options = { merge: false }; }
 
             const writes = updates.filter(u => u.type !== 'DELETE').length;
             const deletes = updates.filter(u => u.type === 'DELETE').length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.693",
+  "version": "1.0.695",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/alert-system/helpers/alert_type_registry.js

@@ -1,172 +0,0 @@
-/**
- * file: alert-system/helpers/alert_type_registry.js
- */
-const ALERT_TYPES = {
-  increasedRisk: {
-    id: 'increasedRisk',
-    name: 'Increased Risk',
-    description: 'Alert when a Popular Investor\'s risk score increases',
-    computationName: 'RiskScoreIncrease',
-    category: 'alerts',
-    messageTemplate: '{piUsername}\'s risk score increased by {change} points (from {previous} to {current})',
-    severity: 'high',
-    enabled: true
-  },
-  volatilityChanges: {
-    id: 'volatilityChanges',
-    name: 'Significant Volatility',
-    description: 'Alert when a Popular Investor\'s portfolio volatility exceeds 50%',
-    computationName: 'SignificantVolatility',
-    category: 'alerts',
-    messageTemplate: '{piUsername}\'s portfolio volatility is {volatility}% (threshold: {threshold}%)',
-    severity: 'medium',
-    enabled: true
-  },
-  newSector: {
-    id: 'newSector',
-    name: 'New Sector Entry',
-    description: 'Alert when a Popular Investor enters a new sector',
-    computationName: 'NewSectorExposure',
-    category: 'alerts',
-    messageTemplate: '{piUsername} entered new sector(s): {sectorName}',
-    severity: 'low',
-    enabled: true
-  },
-  increasedPositionSize: {
-    id: 'increasedPositionSize',
-    name: 'Increased Position Size',
-    description: 'Alert when a Popular Investor significantly increases a position size (>5%)',
-    computationName: 'PositionInvestedIncrease',
-    category: 'alerts',
-    messageTemplate: '{piUsername} increased position size for {symbol} by {diff}% (from {prev}% to {curr}%)',
-    severity: 'medium',
-    enabled: true
-  },
-  newSocialPost: {
-    id: 'newSocialPost',
-    name: 'New Social Post',
-    description: 'Alert when a Popular Investor makes a new social post',
-    computationName: 'NewSocialPost',
-    category: 'alerts',
-    messageTemplate: '{piUsername} posted a new update: {title}',
-    severity: 'low',
-    enabled: true
-  },
-  // [NEW] Behavioral Anomaly Registration
-  behavioralAnomaly: {
-    id: 'behavioralAnomaly',
-    name: 'Behavioral Anomaly',
-    description: 'Alert when a Popular Investor deviates significantly from their baseline behavior',
-    computationName: 'BehavioralAnomaly',
-    category: 'alerts',
-    // Uses metadata keys from Behaviour.js: primaryDriver, driverSignificance, anomalyScore
-    messageTemplate: 'Behavioral Alert for {piUsername}: {primaryDriver} Deviation ({driverSignificance}) detected.',
-    severity: 'high',
-    enabled: true
-  },
-  testSystemProbe: {
-    id: 'testSystemProbe',
-    name: 'Test System Probe',
-    description: 'Always-on debug alert',
-    computationName: 'TestSystemProbe',
-    category: 'alerts',
-    messageTemplate: 'Probe Triggered for {status} at {timestamp}', 
-    severity: 'info',
-    enabled: true
-  },
-};
-
-/**
- * Get alert type by ID
- */
-function getAlertType(alertTypeId) {
-  return ALERT_TYPES[alertTypeId] || null;
-}
-
-/**
- * Get alert type by computation name
- */
-function getAlertTypeByComputation(computationName) {
-  for (const [id, alertType] of Object.entries(ALERT_TYPES)) {
-    if (alertType.computationName === computationName) {
-      return alertType;
-    }
-  }
-  return null;
-}
-
-/**
- * Get all enabled alert types
- */
-function getAllAlertTypes() {
-  return Object.values(ALERT_TYPES).filter(type => type.enabled);
-}
-
-/**
- * Check if a computation is an alert computation
- */
-function isAlertComputation(computationName) {
-  return getAlertTypeByComputation(computationName) !== null;
-}
-
-/**
- * Generate alert message from template and metadata
- */
-function generateAlertMessage(alertType, piUsername, metadata = {}) {
-  let message = alertType.messageTemplate;
-  
-  // Replace placeholders
-  message = message.replace('{piUsername}', piUsername || 'Unknown');
-  message = message.replace('{count}', metadata.count || metadata.positions?.length || metadata.moveCount || 0);
-  message = message.replace('{change}', metadata.change || metadata.changePercent || metadata.diff || 'N/A');
-  message = message.replace('{previous}', metadata.previous || metadata.previousValue || metadata.prev || metadata.previousRisk || 'N/A');
-  message = message.replace('{current}', metadata.current || metadata.currentValue || metadata.curr || metadata.currentRisk || 'N/A');
-  message = message.replace('{sectorName}', metadata.sectorName || (metadata.newExposures && metadata.newExposures.length > 0 ? metadata.newExposures.join(', ') : 'Unknown'));
-  message = message.replace('{ticker}', metadata.ticker || metadata.symbol || 'Unknown');
-  
-  // Format numeric values
-  message = message.replace('{volatility}', metadata.volatility ? `${(metadata.volatility * 100).toFixed(1)}` : 'N/A');
-  message = message.replace('{threshold}', metadata.threshold ? `${(metadata.threshold * 100).toFixed(0)}` : 'N/A');
-  message = message.replace('{diff}', metadata.diff ? `${metadata.diff.toFixed(1)}` : 'N/A');
-  message = message.replace('{prev}', metadata.prev ? `${metadata.prev.toFixed(1)}` : 'N/A');
-  message = message.replace('{curr}', metadata.curr ? `${metadata.curr.toFixed(1)}` : 'N/A');
-  message = message.replace('{title}', metadata.title || 'New Update');
-  
-  // [FIX] Probe Placeholders (Missing in original)
-  message = message.replace('{status}', metadata.status || 'Unknown Status');
-  message = message.replace('{timestamp}', metadata.timestamp || new Date().toISOString());
-
-  // [NEW] Behavioral Anomaly Placeholders
-  message = message.replace('{primaryDriver}', metadata.primaryDriver || 'Unknown Factor');
-  message = message.replace('{driverSignificance}', metadata.driverSignificance || 'N/A');
-  message = message.replace('{anomalyScore}', metadata.anomalyScore || 'N/A');
-  
-  // Handle positions list if available
-  if (metadata.positions && Array.isArray(metadata.positions) && metadata.positions.length > 0) {
-    const positionsList = metadata.positions
-      .slice(0, 3)
-      .map(p => p.ticker || p.instrumentId)
-      .join(', ');
-    message = message.replace('{positions}', positionsList);
-  }
-  
-  // Handle moves array for PositionInvestedIncrease
-  if (metadata.moves && Array.isArray(metadata.moves) && metadata.moves.length > 0) {
-    const firstMove = metadata.moves[0];
-    message = message.replace('{symbol}', firstMove.symbol || 'Unknown');
-    message = message.replace('{diff}', firstMove.diff ? `${firstMove.diff.toFixed(1)}` : 'N/A');
-    message = message.replace('{prev}', firstMove.prev ? `${firstMove.prev.toFixed(1)}` : 'N/A');
-    message = message.replace('{curr}', firstMove.curr ? `${firstMove.curr.toFixed(1)}` : 'N/A');
-  }
-  
-  return message;
-}
-
-module.exports = {
-  ALERT_TYPES,
-  getAlertType,
-  getAlertTypeByComputation,
-  getAllAlertTypes,
-  isAlertComputation,
-  generateAlertMessage
-};