bulltrackers-module 1.0.644 → 1.0.647

package/functions/computation-system/context/ManifestBuilder.js

@@ -231,7 +231,8 @@ function buildManifest(productLinesToRun = [], calculations) {
         const manifestEntry = {
             name:          normalizedName,
             class:         Class,
-            category:      folderName === 'core' && metadata.category ? metadata.category : folderName,     
+            // [CHANGED] Strictly use the folderName as the category.
+            category:      folderName,     
             sourcePackage: folderName,   
             type:                 metadata.type,
             isPage:               metadata.isPage === true,

package/functions/computation-system/data/CachedDataLoader.js

@@ -84,6 +84,9 @@ class CachedDataLoader {
 
     async loadInsights(dateStr) { 
         if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); 
+        const collectionName = this.config.insightsCollectionName || 'daily_instrument_insights';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'insights' from: ${path}`);
         const promise = loadDailyInsights(this.config, this.deps, dateStr); 
         this.cache.insights.set(dateStr, promise); 
         return promise; 
@@ -91,6 +94,9 @@ class CachedDataLoader {
 
     async loadSocial(dateStr) { 
         if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); 
+        const collectionName = this.config.socialInsightsCollection || 'daily_social_insights';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'social' from: ${path}`);
         const promise = loadDailySocialPostInsights(this.config, this.deps, dateStr); 
         this.cache.social.set(dateStr, promise); 
         return promise; 
@@ -98,6 +104,9 @@ class CachedDataLoader {
 
     async loadVerifications() {
         if (this.cache.verifications) return this.cache.verifications;
+        const collectionName = this.config.verificationsCollection || 'verification_profiles';
+        const path = `${collectionName}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'verifications' from: ${path}`);
         const verifications = await loadVerificationProfiles(this.config, this.deps);
         this.cache.verifications = verifications;
         return verifications;
@@ -105,6 +114,9 @@ class CachedDataLoader {
 
     async loadRankings(dateStr) {
         if (this.cache.rankings.has(dateStr)) return this.cache.rankings.get(dateStr);
+        const collectionName = this.config.popularInvestorRankingsCollection || 'popular_investor_rankings';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'rankings' from: ${path}`);
         const promise = loadPopularInvestorRankings(this.config, this.deps, dateStr);
         this.cache.rankings.set(dateStr, promise);
         return promise;
@@ -112,6 +124,9 @@ class CachedDataLoader {
 
     async loadRatings(dateStr) {
         if (this.cache.ratings.has(dateStr)) return this.cache.ratings.get(dateStr);
+        const collectionName = this.config.piRatingsCollection || 'PIRatingsData';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'ratings' from: ${path}`);
         const promise = loadPIRatings(this.config, this.deps, dateStr);
         this.cache.ratings.set(dateStr, promise);
         return promise;
@@ -119,6 +134,9 @@ class CachedDataLoader {
 
     async loadPageViews(dateStr) {
         if (this.cache.pageViews.has(dateStr)) return this.cache.pageViews.get(dateStr);
+        const collectionName = this.config.piPageViewsCollection || 'PIPageViewsData';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'pageViews' from: ${path}`);
         const promise = loadPIPageViews(this.config, this.deps, dateStr);
         this.cache.pageViews.set(dateStr, promise);
         return promise;
@@ -126,6 +144,9 @@ class CachedDataLoader {
 
     async loadWatchlistMembership(dateStr) {
         if (this.cache.watchlistMembership.has(dateStr)) return this.cache.watchlistMembership.get(dateStr);
+        const collectionName = this.config.watchlistMembershipCollection || 'WatchlistMembershipData';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'watchlistMembership' from: ${path}`);
         const promise = loadWatchlistMembershipData(this.config, this.deps, dateStr);
         this.cache.watchlistMembership.set(dateStr, promise);
         return promise;
@@ -133,6 +154,9 @@ class CachedDataLoader {
 
     async loadAlertHistory(dateStr) {
         if (this.cache.alertHistory.has(dateStr)) return this.cache.alertHistory.get(dateStr);
+        const collectionName = this.config.piAlertHistoryCollection || 'PIAlertHistoryData';
+        const path = `${collectionName}/${dateStr}`;
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'alertHistory' from: ${path}`);
         const promise = loadPIAlertHistory(this.config, this.deps, dateStr);
         this.cache.alertHistory.set(dateStr, promise);
         return promise;
@@ -243,6 +267,10 @@ class CachedDataLoader {
         let foundCount = 0;
 
         if (batchRefs.length > 0) {
+            // Log summary of all paths being loaded
+            const paths = batchRefs.map(ref => ref.path).join(', ');
+            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Batch loading ${batchRefs.length} documents for '${loaderMethod}': ${paths}`);
+            
             try {
                 const snapshots = await this.deps.db.getAll(...batchRefs);
                 
@@ -279,12 +307,16 @@ class CachedDataLoader {
             }
         }
 
-        return {
+        const summary = {
             dates: Object.keys(results).sort(),
             data: results,
             found: foundCount,
             requested: lookbackDays
         };
+        
+        this.deps.logger?.log('INFO', `[CachedDataLoader] ✅ Loaded ${foundCount}/${lookbackDays} dates for '${loaderMethod}' (found: ${summary.dates.join(', ') || 'none'})`);
+        
+        return summary;
     }
 
     /**
@@ -300,6 +332,7 @@ class CachedDataLoader {
             d.setUTCDate(d.getUTCDate() - i);
             const dString = d.toISOString().slice(0, 10);
             
+            // Log path for legacy loader (will be logged by individual load methods)
             promises.push(
                 this[loaderMethod](dString)
                     .then(data => ({ date: dString, data }))

package/functions/computation-system/data/DependencyFetcher.js

@@ -9,6 +9,34 @@
 const { normalizeName } = require('../utils/utils');
 const zlib = require('zlib');
 
+/**
+ * Checks if data is effectively empty (no usable content).
+ * @param {any} data - The data to check
+ * @returns {boolean} True if data is empty/null/undefined or contains no meaningful content
+ */
+function isDataEmpty(data) {
+    if (!data || data === null || data === undefined) return true;
+    
+    // Check if it's an object with only metadata fields
+    if (typeof data === 'object' && !Array.isArray(data)) {
+        const keys = Object.keys(data);
+        // If only metadata/internal fields, consider it empty
+        const metadataFields = ['_completed', '_compressed', '_sharded', '_shardCount', '_isPageMode', '_pageCount', '_lastUpdated', '_expireAt'];
+        const hasOnlyMetadata = keys.length > 0 && keys.every(k => metadataFields.includes(k) || k.startsWith('_'));
+        
+        if (hasOnlyMetadata) return true;
+        
+        // If object has no keys (after filtering metadata), it's empty
+        const dataKeys = keys.filter(k => !k.startsWith('_'));
+        if (dataKeys.length === 0) return true;
+    }
+    
+    // Check if it's an empty array
+    if (Array.isArray(data) && data.length === 0) return true;
+    
+    return false;
+}
+
 /**
  * BRIDGE FUNCTION: Matches WorkflowOrchestrator signature.
  * Adapts (dateStr, calcs, manifest, ...) -> (dateObj, calcs, ..., manifestLookup).
@@ -28,7 +56,9 @@ async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps,
     const dateObj = new Date(dateStr + (dateStr.includes('T') ? '' : 'T00:00:00Z'));
 
     // 3. Delegate to fetchDependencies
-    return fetchDependencies(dateObj, calcs, config, deps, manifestLookup);
+    // CRITICAL: For historical context (yesterday's data), allow missing dependencies
+    // Historical lookbacks are optional - gaps in historical data are permissible
+    return fetchDependencies(dateObj, calcs, config, deps, manifestLookup, isHistoricalContext);
 }
 
 /**
@@ -38,8 +68,9 @@ async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps,
  * @param {Object} config - System config.
  * @param {Object} deps - System dependencies (db, logger).
  * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
+ * @param {boolean} allowMissing - If true, missing/empty dependencies are allowed (for historical/lookback scenarios).
  */
-async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
+async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}, allowMissing = false) {
     const { db, logger } = deps;
     const dStr = date.toISOString().slice(0, 10);
     
@@ -65,27 +96,87 @@ async function fetchDependencies(date, calcs, config, deps, manifestLookup = {})
     
     if (needed.size === 0) return {};
     
-    logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for ${dStr}`);
+    const calcNames = calcs.map(c => c.name || c.constructor?.name || 'unknown').join(', ');
+    logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for computation(s): ${calcNames} (date: ${dStr})`);
     
     const results = {};
+    const missingDeps = [];
+    const emptyDeps = [];
+    
+    // Helper to build path string
+    const buildPath = (category, normName) => {
+        return `${config.resultsCollection || 'computation_results'}/${dStr}/${config.resultsSubcollection || 'results'}/${category}/${config.computationsSubcollection || 'computations'}/${normName}`;
+    };
+    
     // CHANGED: Iterate over the entries to access both normalized and original names
     const promises = Array.from(needed.entries()).map(async ([normName, originalName]) => {
+        // Resolve Category from Lookup, default to 'analytics' if unknown
+        // Note: manifestLookup keys are expected to be normalized
+        const category = manifestLookup[normName] || 'analytics';
+        const path = buildPath(category, normName);
+        
         try {
-            // Resolve Category from Lookup, default to 'analytics' if unknown
-            // Note: manifestLookup keys are expected to be normalized
-            const category = manifestLookup[normName] || 'analytics'; 
+            // Pass logger in config for fetchSingleResult
+            const fetchConfig = { ...config, logger };
             
             // Fetch using the normalized name (system standard)
-            const data = await fetchSingleResult(db, config, dStr, normName, category);
+            const data = await fetchSingleResult(db, fetchConfig, dStr, normName, category);
             
-            // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
-            if (data) results[originalName] = data;
+            // CRITICAL: Validate that dependency exists and has data
+            if (!data) {
+                missingDeps.push({ name: originalName, normalizedName: normName, path });
+                // Log level depends on context - ERROR for current date, INFO for historical
+                if (allowMissing) {
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Missing dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
+                } else {
+                    logger.log('ERROR', `[DependencyFetcher] ❌ Missing required dependency '${originalName}' (${normName}) from: ${path}`);
+                }
+            } else if (isDataEmpty(data)) {
+                emptyDeps.push({ name: originalName, normalizedName: normName, path });
+                // Log level depends on context - ERROR for current date, INFO for historical
+                if (allowMissing) {
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
+                } else {
+                    logger.log('ERROR', `[DependencyFetcher] ❌ Empty dependency '${originalName}' (${normName}) from: ${path} - Document exists but contains no usable data`);
+                }
+            } else {
+                // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
+                results[originalName] = data;
+            }
         } catch (e) {
-            logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${originalName}: ${e.message}`);
+            missingDeps.push({ name: originalName, normalizedName: normName, path, error: e.message });
+            // Log level depends on context - ERROR for current date, INFO for historical
+            if (allowMissing) {
+                logger.log('INFO', `[DependencyFetcher] ⚠️ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message} (Historical context - allowed)`);
+            } else {
+                logger.log('ERROR', `[DependencyFetcher] ❌ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message}`);
+            }
         }
     });
     
     await Promise.all(promises);
+    
+    // CRITICAL: Fail if any required dependencies are missing or empty
+    // EXCEPTION: For historical/lookback scenarios, missing dependencies are permissible
+    if ((missingDeps.length > 0 || emptyDeps.length > 0) && !allowMissing) {
+        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path}${d.error ? `, error: ${d.error}` : ''})`).join(', ');
+        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        
+        const errorMsg = `[DependencyFetcher] ❌ CRITICAL: Cannot proceed - Required dependencies missing or empty for computation(s): ${calcNames}\n` +
+                        `Missing dependencies (${missingDeps.length}): ${missingList}\n` +
+                        (emptyDeps.length > 0 ? `Empty dependencies (${emptyDeps.length}): ${emptyList}\n` : '') +
+                        `Date: ${dStr}\n` +
+                        `This computation will FAIL and no results will be saved.`;
+        
+        logger.log('ERROR', errorMsg);
+        throw new Error(errorMsg);
+    } else if (missingDeps.length > 0 || emptyDeps.length > 0) {
+        // Historical/lookback context - log but allow missing dependencies
+        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        logger.log('INFO', `[DependencyFetcher] ⚠️ Historical/Lookback context: Missing/empty dependencies allowed for ${calcNames} on ${dStr}. Missing: ${missingList}${emptyDeps.length > 0 ? `, Empty: ${emptyList}` : ''}`);
+    }
+    
     return results;
 }
 
@@ -110,7 +201,7 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
     // Initialize structure
     calcNames.forEach(name => { results[normalizeName(name)] = {}; });
     
-    logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} calcs over ${lookbackDays} days.`);
+    logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} computation dependencies over ${lookbackDays} days: ${calcNames.join(', ')}`);
 
     const fetchOps = [];
     
@@ -120,11 +211,19 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
             const category = manifestLookup[normName] || 'analytics';
             
             fetchOps.push(async () => {
-                const val = await fetchSingleResult(db, config, dateStr, rawName, category);
-                if (val) {
+                const fetchConfig = { ...config, logger };
+                const val = await fetchSingleResult(db, fetchConfig, dateStr, rawName, category);
+                // CRITICAL: For series/lookback, we allow missing dates (historical lookback may have gaps)
+                // This is expected behavior - not all historical dates will have data
+                // But we still validate that the data isn't empty if it exists
+                if (val && !isDataEmpty(val)) {
                     if (!results[normName]) results[normName] = {};
                     results[normName][dateStr] = val;
+                } else if (val && isDataEmpty(val)) {
+                    // Log but don't fail - series can have gaps, empty data is treated as missing
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${rawName}' found at ${dateStr} in series (allowing gap - historical lookback)`);
                 }
+                // If val is null, that's fine - missing dates in historical series are permissible
             });
         }
     }
@@ -142,18 +241,49 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
  * Core Helper: Fetches a single result, handles Sharding & Compression.
  */
 async function fetchSingleResult(db, config, dateStr, name, category) {
-    const docRef = db.collection(config.resultsCollection)
+    const resultsCollection = config.resultsCollection || 'computation_results';
+    const resultsSubcollection = config.resultsSubcollection || 'results';
+    const computationsSubcollection = config.computationsSubcollection || 'computations';
+    
+    const path = `${resultsCollection}/${dateStr}/${resultsSubcollection}/${category}/${computationsSubcollection}/${name}`;
+    
+    // Log path - use console.log if logger not available (for backward compatibility)
+    if (config.logger) {
+        config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
+    } else {
+        console.log(`[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
+    }
+    
+    const docRef = db.collection(resultsCollection)
                      .doc(dateStr)
-                     .collection(config.resultsSubcollection)
+                     .collection(resultsSubcollection)
                      .doc(category)
-                     .collection(config.computationsSubcollection)
+                     .collection(computationsSubcollection)
                      .doc(name);
                      
     const snap = await docRef.get();
-    if (!snap.exists) return null;
+    if (!snap.exists) {
+        // Log the missing document path clearly
+        if (config.logger) {
+            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document does not exist at: ${path}`);
+        } else {
+            console.error(`[DependencyFetcher] ❌ Document does not exist at: ${path}`);
+        }
+        return null;
+    }
     
     let data = snap.data();
     
+    // Check if document exists but is effectively empty (only metadata)
+    if (isDataEmpty(data)) {
+        if (config.logger) {
+            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
+        } else {
+            console.error(`[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
+        }
+        return null; // Return null so it gets caught as missing
+    }
+    
     // 1. Handle Compression
     if (data._compressed && data.payload) {
         try {
@@ -165,27 +295,74 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             data = { ...data, ...realData }; 
             delete data.payload; 
         } catch (e) {
-            console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
+            const errorMsg = `Decompression failed for ${name}: ${e.message}`;
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
+            }
             return null;
         }
     }
     
     // 2. Handle Sharding
     if (data._sharded) {
+        const shardPath = `${path}/_shards`;
+        if (config.logger) {
+            config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
+        } else {
+            console.log(`[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
+        }
+        
         const shardCol = docRef.collection('_shards');
         const shardSnaps = await shardCol.get();
         
-        if (!shardSnaps.empty) {
-            shardSnaps.forEach(shard => {
-                const shardData = shard.data();
-                // Merge shard contents, ignoring internal metadata if it clashes
-                Object.entries(shardData).forEach(([k, v]) => {
-                    if (!k.startsWith('_')) {
-                        data[k] = v;
-                    }
-                });
+        if (shardSnaps.empty) {
+            // No shards found - this is a problem
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
+            }
+            return null; // Return null so it gets caught as missing
+        }
+        
+        // Merge shard contents
+        let hasData = false;
+        shardSnaps.forEach(shard => {
+            const shardData = shard.data();
+            const shardId = shard.id;
+            if (config.logger) {
+                config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
+            }
+            // Merge shard contents, ignoring internal metadata if it clashes
+            Object.entries(shardData).forEach(([k, v]) => {
+                if (!k.startsWith('_')) {
+                    data[k] = v;
+                    hasData = true;
+                }
             });
+        });
+        
+        // If shards contained no actual data, treat as empty
+        if (!hasData) {
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
+            }
+            return null;
+        }
+    }
+    
+    // Final validation: ensure we have usable data after all processing
+    if (isDataEmpty(data)) {
+        if (config.logger) {
+            config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
+        } else {
+            console.error(`[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
         }
+        return null;
     }
     
     return data;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.644",
+  "version": "1.0.647",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [