bulltrackers-module 1.0.661 → 1.0.662

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -133,15 +133,70 @@ async function getStableDateSession(config, dependencies, pass, dateLimitStr, fo
 // 2. NEW SNAPSHOT HANDLER
 async function handleSnapshot(config, dependencies, reqBody) {
     const { logger } = dependencies;
-    const date = reqBody.date;
+    const targetDate = reqBody.date; // Optional: if provided, only process up to this date
     
-    if (!date) throw new Error('Snapshot action requires a "date"');
-
     try {
-        logger.log('INFO', `[Dispatcher] 📸 Triggering Snapshot Service for ${date}`);
-        // Calls the service we created earlier
-        const result = await generateDailySnapshots(date, config, dependencies);
-        return result; 
+        // Get earliest available root data date
+        const earliestDates = await getEarliestDataDates(config, dependencies);
+        const earliestDate = earliestDates.absoluteEarliest;
+        
+        if (!earliestDate) {
+            throw new Error('Could not determine earliest available root data date');
+        }
+        
+        // Determine end date: use targetDate if provided, otherwise use today
+        const endDate = targetDate ? new Date(targetDate + 'T00:00:00Z') : new Date();
+        endDate.setUTCHours(0, 0, 0, 0);
+        
+        // Generate all dates from earliest to end date
+        const startDate = new Date(earliestDate);
+        startDate.setUTCHours(0, 0, 0, 0);
+        
+        const dateStrings = getExpectedDateStrings(startDate, endDate);
+        
+        if (dateStrings.length === 0) {
+            logger.log('WARN', '[Dispatcher] No dates to process for snapshot');
+            return { status: 'OK', processed: 0, skipped: 0 };
+        }
+        
+        logger.log('INFO', `[Dispatcher] 📸 Processing snapshots for ${dateStrings.length} dates from ${dateStrings[0]} to ${dateStrings[dateStrings.length - 1]}`);
+        
+        // Process each date (snapshot service will skip if already exists)
+        const results = [];
+        const BATCH_SIZE = 5; // Process 5 dates in parallel to avoid overwhelming the system
+        
+        for (let i = 0; i < dateStrings.length; i += BATCH_SIZE) {
+            const batch = dateStrings.slice(i, i + BATCH_SIZE);
+            const batchResults = await Promise.allSettled(
+                batch.map(dateStr => generateDailySnapshots(dateStr, config, dependencies))
+            );
+            
+            batchResults.forEach((result, idx) => {
+                const dateStr = batch[idx];
+                if (result.status === 'fulfilled') {
+                    const value = result.value;
+                    results.push({ date: dateStr, status: value.status || 'OK' });
+                } else {
+                    logger.log('ERROR', `[Dispatcher] Snapshot failed for ${dateStr}: ${result.reason?.message || result.reason}`);
+                    results.push({ date: dateStr, status: 'ERROR', error: result.reason?.message || String(result.reason) });
+                }
+            });
+        }
+        
+        const successful = results.filter(r => r.status === 'OK').length;
+        const skipped = results.filter(r => r.status === 'SKIPPED').length;
+        const failed = results.filter(r => r.status === 'ERROR').length;
+        
+        logger.log('INFO', `[Dispatcher] 📸 Snapshot batch complete: ${successful} processed, ${skipped} skipped, ${failed} failed out of ${results.length} total`);
+        
+        return { 
+            status: failed === 0 ? 'OK' : 'PARTIAL', 
+            processed: successful,
+            skipped: skipped,
+            failed: failed,
+            total: results.length,
+            results: results
+        };
     } catch (e) {
         logger.log('ERROR', `[Dispatcher] Snapshot failed: ${e.message}`);
         // Return error object so workflow can see failure

package/functions/computation-system/persistence/ResultCommitter.js

@@ -247,6 +247,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
             if (calc.manifest.class.getSchema && flushMode !== 'INTERMEDIATE') {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
+                // Ensure ttlDays is set to the resolved value (defaults to 90 if undefined)
+                safeMetadata.ttlDays = ttlDays;
                 schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata });
             }
             if (calc.manifest.previousCategory && calc.manifest.previousCategory !== calc.manifest.category && flushMode !== 'INTERMEDIATE') {

package/functions/computation-system/services/SnapshotService.js

@@ -10,11 +10,30 @@ const dataLoader = require('../utils/data_loader');
 
 async function generateDailySnapshots(dateStr, config, deps) {
     const { logger } = deps;
-    logger.log('INFO', `[SnapshotService] 📸 Starting Full System Snapshot for ${dateStr}`);
-
     const bucketName = config.gcsBucketName || 'bulltrackers';
     const bucket = storage.bucket(bucketName);
 
+    // Quick check: if all main snapshots exist, skip entirely
+    const mainFiles = [
+        `${dateStr}/snapshots/portfolios.json.gz`,
+        `${dateStr}/snapshots/social.json.gz`,
+        `${dateStr}/snapshots/history.jsonl.gz`,
+        `${dateStr}/snapshots/ratings.json.gz`,
+        `${dateStr}/snapshots/rankings.json.gz`
+    ];
+    
+    if (!config.forceSnapshot) {
+        const existenceChecks = await Promise.all(mainFiles.map(path => bucket.file(path).exists()));
+        const allExist = existenceChecks.every(([exists]) => exists);
+        
+        if (allExist) {
+            logger.log('INFO', `[SnapshotService] ⏭️  All snapshots already exist for ${dateStr}, skipping`);
+            return { status: 'SKIPPED', date: dateStr, reason: 'all_exist' };
+        }
+    }
+
+    logger.log('INFO', `[SnapshotService] 📸 Starting Full System Snapshot for ${dateStr}`);
+
     // parallelize independent fetches
     await Promise.all([
         snapshotPortfolios(dateStr, bucket, config, deps),  // Heavy
@@ -26,7 +45,7 @@ async function generateDailySnapshots(dateStr, config, deps) {
         snapshotMetadata(dateStr, bucket, config, deps)     // Small Docs (Insights, Alerts, Watchlist)
     ]);
 
-    logger.log('INFO', `[SnapshotService] ✅ Full System Snapshot Complete.`);
+    logger.log('INFO', `[SnapshotService] ✅ Full System Snapshot Complete for ${dateStr}`);
     return { status: 'OK', date: dateStr };
 }
 
@@ -109,10 +128,10 @@ async function snapshotRankings(dateStr, bucket, config, deps) {
 async function snapshotMetadata(dateStr, bucket, config, deps) {
     // Bundle small files into one "metadata.json" or keep separate. Separate is safer for loaders.
     const ops = [
-        { name: 'insights', fn: () => dataLoader.loadDailyInsights(config, deps, dateStr) },
-        { name: 'page_views', fn: () => dataLoader.loadPIPageViews(config, deps, dateStr) },
-        { name: 'watchlist', fn: () => dataLoader.loadWatchlistMembership(config, deps, dateStr) },
-        { name: 'alerts', fn: () => dataLoader.loadPIAlertHistory(config, deps, dateStr) },
+        { name: 'insights',    fn: () => dataLoader.loadDailyInsights(config, deps, dateStr) },
+        { name: 'page_views',  fn: () => dataLoader.loadPIPageViews(config, deps, dateStr) },
+        { name: 'watchlist',   fn: () => dataLoader.loadWatchlistMembership(config, deps, dateStr) },
+        { name: 'alerts',      fn: () => dataLoader.loadPIAlertHistory(config, deps, dateStr) },
         { name: 'master_list', fn: () => dataLoader.loadPopularInvestorMasterList(config, deps) } // Not date bound usually, but good to snapshot state
     ];
 

package/functions/computation-system/utils/schema_capture.js

@@ -4,6 +4,35 @@
  * UPDATED: Added schema validation to prevent silent batch failures.
  */
 
+/**
+ * Recursively removes undefined values from an object.
+ * Firestore doesn't allow undefined values, so we filter them out entirely.
+ * @param {any} data - Data to sanitize
+ * @returns {any} Sanitized data with undefined values removed
+ */
+function removeUndefinedValues(data) {
+    if (data === undefined) return undefined; // Will be filtered out
+    if (data === null) return null;
+    if (data instanceof Date) return data;
+    
+    if (Array.isArray(data)) {
+        return data.map(item => removeUndefinedValues(item)).filter(item => item !== undefined);
+    }
+    
+    if (typeof data === 'object') {
+        const sanitized = {};
+        for (const [key, value] of Object.entries(data)) {
+            const sanitizedValue = removeUndefinedValues(value);
+            if (sanitizedValue !== undefined) {
+                sanitized[key] = sanitizedValue;
+            }
+        }
+        return sanitized;
+    }
+    
+    return data;
+}
+
 /**
  * Validates a schema object before storage.
  * Checks for circular references and size limits.
@@ -58,13 +87,17 @@ async function batchStoreSchemas(dependencies, config, schemas) {
             const docRef = db.collection(schemaCollection).doc(item.name);
             
             // Critical: Always overwrite 'lastUpdated' to now
-            batch.set(docRef, {
+            // Sanitize metadata to remove undefined values (Firestore doesn't allow undefined)
+            const sanitizedMetadata = item.metadata ? removeUndefinedValues(item.metadata) : {};
+            const docData = removeUndefinedValues({
                 computationName: item.name,
                 category:        item.category,
                 schema:          item.schema,
-                metadata:        item.metadata || {},
+                metadata:        sanitizedMetadata,
                 lastUpdated:     new Date() 
-            }, { merge: true });
+            });
+            
+            batch.set(docRef, docData, { merge: true });
             
             validCount++;
 

package/functions/core/utils/pubsub_utils.js

@@ -42,14 +42,21 @@ class PubSubUtils {
 
     /**
      * [NEW] Publishes a single JSON message to a topic.
+     * Includes timeout to prevent hanging on network issues.
      */
-    async publish(topicName, message) {
+    async publish(topicName, message, timeoutMs = 10000) {
         const { pubsub, logger } = this.dependencies;
         const topic = pubsub.topic(topicName);
         const dataBuffer = Buffer.from(JSON.stringify(message));
         
         try {
-            await topic.publishMessage({ data: dataBuffer });
+            // Wrap publish in a timeout promise
+            const publishPromise = topic.publishMessage({ data: dataBuffer });
+            const timeoutPromise = new Promise((_, reject) => 
+                setTimeout(() => reject(new Error(`Publish timeout after ${timeoutMs}ms`)), timeoutMs)
+            );
+            
+            await Promise.race([publishPromise, timeoutPromise]);
         } catch (error) {
             logger.log('ERROR', `[Core Utils] Failed to publish message to ${topicName}`, { error: error.message });
             throw error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.661",
+  "version": "1.0.662",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [