bulltrackers-module 1.0.17 → 1.0.19

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -0,0 +1,95 @@
+/**
+ * @fileoverview Core logic for fetching eToro instrument insights.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * Fetches insights data using header and proxy managers and stores it in Firestore.
+ *
+ * @param {object} firestore - An initialized Firestore client instance.
+ * @param {object} logger - A logger instance.
+ * @param {object} headerManager - An initialized IntelligentHeaderManager instance.
+ * @param {object} proxyManager - An initialized IntelligentProxyManager instance.
+ * @param {object} config - Configuration object.
+ * @param {string} config.etoroInsightsUrl - The URL for the eToro insights API.
+ * @param {string} config.insightsCollectionName - Firestore collection name for results.
+ * @returns {Promise<{success: boolean, message: string, instrumentCount?: number}>}
+ */
+exports.fetchAndStoreInsights = async (firestore, logger, headerManager, proxyManager, config) => {
+    logger.log('INFO', '[FetchInsightsHelpers] Starting eToro insights data fetch...');
+    let selectedHeader = null;
+    let wasSuccessful = false;
+
+    try {
+        // Validate config
+        if (!config.etoroInsightsUrl || !config.insightsCollectionName) {
+            throw new Error("Missing required configuration: etoroInsightsUrl or insightsCollectionName.");
+        }
+
+        selectedHeader = await headerManager.selectHeader();
+        if (!selectedHeader) {
+            throw new Error("Could not select a header for the request.");
+        }
+
+        logger.log('INFO', `[FetchInsightsHelpers] Using header ID: ${selectedHeader.id}`, {
+            userAgent: selectedHeader.header['User-Agent'] // Access header data correctly
+        });
+
+        // Make the API request via the proxy manager
+        // Pass only the necessary parts of the selected header object
+        const fetchOptions = {
+             headers: selectedHeader.header, // Pass the actual header object
+             timeout: 30000 // 30 second timeout - Note: Proxy manager might have its own timeout logic
+        };
+        const { response } = await proxyManager.fetch(config.etoroInsightsUrl, fetchOptions);
+
+        if (!response.ok) {
+            const errorText = await response.text();
+            // No need to update header performance here, proxy manager handles failures implicitly if needed
+            throw new Error(`API request failed via proxy with status ${response.status}: ${errorText}`);
+        }
+
+        // If fetch succeeded via proxy, mark header as successful
+        wasSuccessful = true; // Mark success for header update
+
+        const insightsData = await response.json();
+
+        if (!Array.isArray(insightsData) || insightsData.length === 0) {
+            throw new Error('API returned empty or invalid data.');
+        }
+
+        // Store in Firestore
+        const today = new Date().toISOString().slice(0, 10);
+        const docRef = firestore.collection(config.insightsCollectionName).doc(today);
+
+        const firestorePayload = {
+            fetchedAt: FieldValue.serverTimestamp(),
+            instrumentCount: insightsData.length,
+            insights: insightsData
+        };
+
+        await docRef.set(firestorePayload);
+
+        const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`;
+        logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, {
+            documentId: today,
+            instrumentCount: insightsData.length
+        });
+        return { success: true, message: successMsg, instrumentCount: insightsData.length };
+
+    } catch (error) {
+        logger.log('ERROR', '[FetchInsightsHelpers] Error fetching eToro insights', {
+            errorMessage: error.message,
+            errorStack: error.stack,
+            headerId: selectedHeader ? selectedHeader.id : 'not-selected'
+        });
+        // Let the calling function handle header update on failure
+        throw error; // Re-throw the error to be caught by the handler
+    } finally {
+         // Update header performance based on the success *of the target API call*
+         if (selectedHeader) {
+            await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
+         }
+    }
+};

package/functions/fetch-insights/index.js

@@ -0,0 +1,57 @@
+/**
+ * @fileoverview Exports the FetchInsights handler creator.
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const { logger } = require("sharedsetup")(__filename);
+const { IntelligentHeaderManager, IntelligentProxyManager } = require('../core/utils');
+const { fetchAndStoreInsights } = require('./helpers/handler_helpers');
+
+/**
+ * Creates the Cloud Function handler for fetching eToro insights.
+ * @param {object} config - The configuration object loaded from the calling function's context.
+ * @returns {Function} The async Cloud Function handler (Express middleware format).
+ */
+function createFetchInsightsHandler(config) {
+    // Initialize clients once when the function instance starts
+    const firestore = new Firestore();
+    const headerManager = new IntelligentHeaderManager(firestore, logger, config); // Pass dependencies
+    const proxyManager = new IntelligentProxyManager(firestore, logger, config); // Pass dependencies
+
+    return async (req, res) => {
+        logger.log('INFO', '🚀 Fetch eToro Insights triggered via module...');
+        try {
+            // Delegate all logic to the helper
+            const result = await fetchAndStoreInsights(firestore, logger, headerManager, proxyManager, config);
+
+            // Send success response
+            res.status(200).send(result.message);
+
+        } catch (error) {
+            logger.log('ERROR', 'FATAL Error in Fetch eToro Insights (Module)', { errorMessage: error.message, errorStack: error.stack });
+            // Header performance update on failure is handled within the helper's finally block if header was selected
+
+             // Ensure header performance flush happens even on fatal error before responding
+            try {
+                await headerManager.flushPerformanceUpdates();
+            } catch (flushError) {
+                 logger.log('ERROR', 'Error flushing header performance during error handling', { flushError: flushError.message });
+            }
+
+            // Send error response
+            res.status(500).send(`An internal error occurred: ${error.message}`);
+        } finally {
+             // Always attempt to flush performance updates at the end of the function execution
+             try {
+                await headerManager.flushPerformanceUpdates();
+             } catch (flushError) {
+                 logger.log('ERROR', 'Error flushing header performance in final finally block', { flushError: flushError.message });
+             }
+        }
+    };
+}
+
+module.exports = {
+    createFetchInsightsHandler,
+    helpers: { fetchAndStoreInsights } // Export helpers if needed directly
+};

package/functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers.js

@@ -0,0 +1,179 @@
+/**
+ * @fileoverview Core logic for the speculator cleanup orchestrator.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * Cleans up stale entries from the pending speculators collection.
+ * @param {object} firestore - An initialized Firestore client.
+ * @param {object} logger - A logger instance.
+ * @param {object} config - Configuration object.
+ * @param {string} config.pendingSpeculatorsCollectionName - Name of the pending collection.
+ * @param {number} config.pendingGracePeriodHours - Hours before a pending entry is considered stale.
+ * @returns {Promise<{ batch: object, count: number }>} - The batch object with updates and the count of removed users.
+ */
+async function cleanupPendingSpeculators(firestore, logger, config) {
+    logger.log('INFO', '[CleanupHelpers] Starting pending speculator cleanup...');
+    const batch = firestore.batch();
+    let stalePendingUsersRemoved = 0;
+    const pendingCollectionRef = firestore.collection(config.pendingSpeculatorsCollectionName);
+    const staleThreshold = new Date();
+    staleThreshold.setHours(staleThreshold.getHours() - config.pendingGracePeriodHours);
+
+    try {
+        const pendingSnapshot = await pendingCollectionRef.get();
+        if (pendingSnapshot.empty) {
+            logger.log('INFO', '[CleanupHelpers] Pending speculators collection is empty.');
+            return { batch, count: 0 };
+        }
+
+        for (const doc of pendingSnapshot.docs) {
+            const pendingData = doc.data().users || {};
+            const updates = {};
+            let updatesInDoc = 0;
+
+            for (const userId in pendingData) {
+                // Ensure timestamp exists and is a Firestore Timestamp before converting
+                const timestamp = pendingData[userId]?.toDate ? pendingData[userId].toDate() : null;
+                if (timestamp && timestamp < staleThreshold) {
+                    updates[`users.${userId}`] = FieldValue.delete();
+                    stalePendingUsersRemoved++;
+                    updatesInDoc++;
+                }
+            }
+
+            if (updatesInDoc > 0) {
+                 logger.log('TRACE', `[CleanupHelpers] Marking ${updatesInDoc} users for removal from pending doc ${doc.id}`);
+                batch.update(doc.ref, updates);
+            }
+        }
+        logger.log('INFO', `[CleanupHelpers] Marked ${stalePendingUsersRemoved} total stale pending users for removal.`);
+    } catch (error) {
+         logger.log('ERROR', '[CleanupHelpers] Error cleaning pending speculators', { errorMessage: error.message });
+         throw error; // Re-throw to be caught by the main handler
+    }
+    return { batch, count: stalePendingUsersRemoved };
+}
+
+/**
+ * Cleans up stale speculators from the main blocks based on inactivity grace period.
+ * @param {object} firestore - An initialized Firestore client.
+ * @param {object} logger - A logger instance.
+ * @param {object} config - Configuration object.
+ * @param {string} config.speculatorBlocksCollectionName - Name of the speculator blocks collection.
+ * @param {string} config.speculatorBlockCountsDocPath - Path to the block counts document.
+ * @param {number} config.activityGracePeriodDays - Days of inactivity before removal.
+ * @param {object} batch - The Firestore batch object to add updates to.
+ * @returns {Promise<{ batch: object, count: number }>} - The batch object with updates and the count of removed users.
+ */
+async function cleanupStaleSpeculators(firestore, logger, config, batch) {
+    logger.log('INFO', '[CleanupHelpers] Starting stale speculator cleanup from blocks...');
+    let totalUsersRemoved = 0;
+    const blocksCollectionRef = firestore.collection(config.speculatorBlocksCollectionName);
+    const gracePeriodDate = new Date();
+    gracePeriodDate.setDate(gracePeriodDate.getDate() - config.activityGracePeriodDays);
+    const blockCountsUpdate = {}; // To track decrements per instrument/block
+
+    try {
+        const blocksSnapshot = await blocksCollectionRef.get();
+        if (blocksSnapshot.empty) {
+             logger.log('INFO', '[CleanupHelpers] Speculator blocks collection is empty.');
+            return { batch, count: 0 };
+        }
+
+        for (const doc of blocksSnapshot.docs) {
+            const blockId = doc.id; // e.g., "19000000"
+            const blockData = doc.data();
+            // Firestore map keys cannot contain '.', so the keys are likely 'users.123456'
+            const users = blockData.users || {};
+            let usersRemovedFromBlock = 0;
+            const updates = {}; // Updates specific to this block document
+
+            for (const userKey in users) { // userKey is like "users.123456"
+                // Extract CID correctly
+                const userId = userKey.split('.')[1];
+                if (!userId) continue; // Skip malformed keys
+
+                const userData = users[userKey];
+                // Ensure timestamp exists and is a Firestore Timestamp before converting
+                const lastHeldTimestamp = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : null;
+
+                // Check if the user is stale based on the grace period
+                if (lastHeldTimestamp && lastHeldTimestamp < gracePeriodDate) {
+                    updates[userKey] = FieldValue.delete(); // Delete the specific user field
+                    usersRemovedFromBlock++;
+
+                    // Decrement block counts for each instrument this user held in this block
+                    if (userData.instruments && Array.isArray(userData.instruments)) {
+                        userData.instruments.forEach(instrumentId => {
+                            const instrumentBlockKey = `${instrumentId}_${blockId}`;
+                            if (!blockCountsUpdate[instrumentBlockKey]) {
+                                blockCountsUpdate[instrumentBlockKey] = 0;
+                            }
+                            blockCountsUpdate[instrumentBlockKey]--; // Decrement count
+                        });
+                    }
+                }
+            }
+
+            // If users were marked for removal in this block, add the update to the batch
+            if (usersRemovedFromBlock > 0) {
+                logger.log('TRACE', `[CleanupHelpers] Marking ${usersRemovedFromBlock} users for removal from block ${blockId}.`);
+                batch.update(doc.ref, updates);
+                totalUsersRemoved += usersRemovedFromBlock;
+            }
+        }
+
+        // After iterating all blocks, if any users were removed, add the block count decrements to the batch
+        if (totalUsersRemoved > 0 && Object.keys(blockCountsUpdate).length > 0) {
+            const countsRef = firestore.doc(config.speculatorBlockCountsDocPath);
+            const finalCountUpdates = {};
+            for (const key in blockCountsUpdate) {
+                // Use dot notation for nested map fields
+                finalCountUpdates[`counts.${key}`] = FieldValue.increment(blockCountsUpdate[key]);
+            }
+             logger.log('TRACE', '[CleanupHelpers] Staging block count decrements.', { updates: finalCountUpdates });
+            batch.set(countsRef, finalCountUpdates, { merge: true }); // Use set with merge: true
+        }
+
+        logger.log('INFO', `[CleanupHelpers] Marked ${totalUsersRemoved} total stale speculators for removal from blocks.`);
+
+    } catch (error) {
+        logger.log('ERROR', '[CleanupHelpers] Error cleaning stale speculators', { errorMessage: error.message });
+        throw error; // Re-throw
+    }
+
+    return { batch, count: totalUsersRemoved };
+}
+
+/**
+ * Orchestrates the cleanup process.
+ * @param {object} firestore - An initialized Firestore client.
+ * @param {object} logger - A logger instance.
+ * @param {object} config - Configuration object.
+ */
+exports.runCleanup = async (firestore, logger, config) => {
+    logger.log('INFO', '[CleanupHelpers] Running cleanup orchestrator...');
+
+    try {
+        // Start cleanup for pending users
+        const { batch: batchAfterPending, count: pendingRemoved } = await cleanupPendingSpeculators(firestore, logger, config);
+
+        // Continue with the same batch for stale speculators
+        const { batch: finalBatch, count: staleRemoved } = await cleanupStaleSpeculators(firestore, logger, config, batchAfterPending);
+
+        // Commit the batch if any changes were made
+        if (pendingRemoved > 0 || staleRemoved > 0) {
+            await finalBatch.commit();
+            logger.log('SUCCESS', `[CleanupHelpers] Cleanup commit successful. Removed ${pendingRemoved} pending, ${staleRemoved} stale speculators.`);
+            return { pendingRemoved, staleRemoved };
+        } else {
+            logger.log('SUCCESS', '[CleanupHelpers] No stale users found in pending or blocks.');
+            return { pendingRemoved: 0, staleRemoved: 0 };
+        }
+    } catch (error) {
+        logger.log('ERROR', '[CleanupHelpers] FATAL error during cleanup orchestration', { errorMessage: error.message, errorStack: error.stack });
+        throw error; // Re-throw for the main handler
+    }
+};

package/functions/speculator-cleanup-orchestrator/index.js

@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Exports the SpeculatorCleanupOrchestrator handler creator.
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const { logger } = require("sharedsetup")(__filename);
+const { runCleanup } = require('./helpers/cleanup_helpers');
+
+/**
+ * Creates the Cloud Function handler for speculator cleanup.
+ * @param {object} config - The configuration object loaded from the calling function's context.
+ * @returns {Function} The async Cloud Function handler (Express middleware format).
+ */
+function createSpeculatorCleanupHandler(config) {
+    // Initialize Firestore client once when the function instance starts
+    const firestore = new Firestore();
+
+    return async (req, res) => {
+        logger.log('INFO', '🚀 Speculator Cleanup Orchestrator triggered via module...');
+        try {
+            // Validate essential config
+             if (!config || !config.speculatorBlocksCollectionName || !config.speculatorBlockCountsDocPath || !config.pendingSpeculatorsCollectionName) {
+                 throw new Error("Speculator Cleanup Orchestrator received invalid configuration.");
+             }
+
+            // Delegate all logic to the helper
+            const { pendingRemoved, staleRemoved } = await runCleanup(firestore, logger, config);
+
+            // Send success response
+            res.status(200).send(`Cleanup complete via module. Removed ${staleRemoved} stale speculators, ${pendingRemoved} stale pending users.`);
+
+        } catch (error) {
+            logger.log('ERROR', 'FATAL Error in Speculator Cleanup Orchestrator (Module)', { errorMessage: error.message, errorStack: error.stack });
+            // Send error response
+            res.status(500).send("An internal cleanup error occurred (via module).");
+        }
+    };
+}
+
+module.exports = {
+    createSpeculatorCleanupHandler,
+    helpers: { runCleanup } // Export helpers if needed directly
+};

package/index.js

@@ -11,6 +11,8 @@ const ComputationSystem = require('./functions/computation-system');
 const GenericAPI = require('./functions/generic-api'); // <-- ADD THIS
 const Dispatcher = require('./functions/dispatcher'); // <-- ADD THIS
 const InvalidSpeculatorHandler = require('./functions/invalid-speculator-handler'); // <-- ADD THIS
+const SpeculatorCleanupOrchestrator = require('./functions/speculator-cleanup-orchestrator'); // <-- ADD THIS
+const FetchInsights = require('./functions/fetch-insights'); // <-- ADD THIS
 
 module.exports = {
   core,
@@ -20,4 +22,6 @@ module.exports = {
   GenericAPI, // <-- AND ADD THIS
   Dispatcher, // <-- AND ADD THIS
   InvalidSpeculatorHandler, // <-- AND ADD THIS
+  SpeculatorCleanupOrchestrator, // <-- AND ADD THIS
+  FetchInsights, // <-- AND ADD THIS
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -11,7 +11,9 @@
     "functions/computation-system/",
     "functions/generic-api/",
     "functions/dispatcher/",
-    "functions/invalid-speculator-handler/"
+    "functions/invalid-speculator-handler/",
+    "functions/speculator-cleanup-orchestrator/",
+    "functions/fetch-insights/"
   ],
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"