bulltrackers-module 1.0.14 → 1.0.16

package/functions/core/utils/index.js

@@ -5,9 +5,13 @@
 const firestoreUtils = require('./firestore_utils');
 const pubsubUtils = require('./pubsub_utils');
 const loggingWrapper = require('./logging_wrapper');
+const { IntelligentHeaderManager } = require('./intelligent_header_manager'); // <-- ADD
+const { IntelligentProxyManager } = require('./intelligent_proxy_manager'); // <-- ADD
 
 module.exports = {
   firestore: firestoreUtils,
   pubsub: pubsubUtils,
   logging: loggingWrapper,
+  IntelligentHeaderManager, // <-- ADD
+  IntelligentProxyManager, // <-- ADD
 };

package/functions/core/utils/intelligent_header_manager.js

@@ -0,0 +1,185 @@
+/**
+ * @fileoverview
+ * Manages a pool of different browser headers. It selects headers based on their historical success rate
+ * to mimic real user traffic and avoid being blocked. Performance data for each header
+ * is stored and retrieved from Firestore.
+ * * This module is designed to be reusable and receives all dependencies
+ * (firestore, logger) and configuration via its constructor.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+class IntelligentHeaderManager {
+    /**
+     * @param {object} firestore - An initialized Firestore instance.
+     * @param {object} logger - A logger instance (e.g., from sharedsetup).
+     * @param {object} config - Configuration object.
+     * @param {string} config.headersCollectionName - The name of the Firestore collection for headers.
+     * @param {number} config.cacheDurationMs - How long to cache headers in memory (in ms).
+     * @param {string} config.fallbackUserAgent - A fallback User-Agent if loading fails.
+     */
+    constructor(firestore, logger, config) {
+        if (!firestore || !logger || !config) {
+            throw new Error("IntelligentHeaderManager requires firestore, logger, and config objects.");
+        }
+        if (!config.headersCollectionName || !config.cacheDurationMs || !config.fallbackUserAgent) {
+             throw new Error("IntelligentHeaderManager config is missing required keys (headersCollectionName, cacheDurationMs, fallbackUserAgent).");
+        }
+
+        this.firestore = firestore;
+        this.logger = logger;
+
+        // Load from config
+        this.collectionName = config.headersCollectionName;
+        this.cacheDuration = config.cacheDurationMs;
+        this.fallbackUserAgent = config.fallbackUserAgent;
+
+        // Internal state
+        this.headers = [];
+        this.lastFetched = null;
+        this.performanceUpdates = {};
+    }
+
+    /**
+     * Fetches and caches header documents from Firestore. If the cache is fresh, it does nothing.
+     * @private
+     * @returns {Promise<void>}
+     */
+    async _loadHeaders() {
+        const now = new Date();
+        if (this.lastFetched && (now - this.lastFetched < this.cacheDuration) && this.headers.length > 0) {
+            return; // Cache is fresh
+        }
+
+        try {
+            this.logger.log('INFO', '[HeaderManager] Refreshing header performance data from Firestore...');
+            const snapshot = await this.firestore.collection(this.collectionName).get();
+            
+            if (snapshot.empty) {
+                throw new Error(`No documents found in headers collection: ${this.collectionName}`);
+            }
+
+            this.headers = snapshot.docs.map(doc => ({
+                id: doc.id,
+                data: doc.data().header, // Assuming header object is stored in 'header' field
+                performance: {
+                    total: doc.data().totalRequests || 0,
+                    success: doc.data().successfulRequests || 0
+                }
+            }));
+
+            if (this.headers.length === 0) {
+                throw new Error("Header collection was queried but returned no usable headers.");
+            }
+
+            this.lastFetched = new Date();
+            this.logger.log('INFO', `[HeaderManager] Successfully loaded ${this.headers.length} headers.`);
+        } catch (error) {
+            this.logger.log('ERROR', '[HeaderManager] Failed to load headers from Firestore. Using fallback.', { 
+                errorMessage: error.message,
+                collection: this.collectionName
+            });
+            // Ensure fallback
+            if (this.headers.length === 0) {
+                this.headers = [{
+                    id: 'fallback',
+                    data: { 'User-Agent': this.fallbackUserAgent },
+                    performance: { total: 1, success: 1 }
+                }];
+            }
+        }
+    }
+
+    /**
+     * Selects the best available header based on success rate.
+     * Uses a weighted random selection to balance exploration and exploitation.
+     * @returns {Promise<{id: string, header: object}>}
+     */
+    async selectHeader() {
+        await this._loadHeaders();
+
+        if (!this.headers || this.headers.length === 0) {
+            this.logger.log('WARN', '[HeaderManager] No headers available, returning fallback.');
+            return { id: 'fallback', header: { 'User-Agent': this.fallbackUserAgent } };
+        }
+
+        // Calculate total score for weighted random selection
+        let totalScore = 0;
+        const weightedHeaders = this.headers.map(h => {
+            const successRate = (h.performance.total === 0) ? 0.5 : (h.performance.success / h.performance.total);
+            // Add-1 smoothing to avoid zero probability for new headers
+            const score = (successRate + 1) * 10; 
+            totalScore += score;
+            return { ...h, score };
+        });
+
+        // Select a random value
+        let random = Math.random() * totalScore;
+        
+        for (const h of weightedHeaders) {
+            if (random < h.score) {
+                return { id: h.id, header: h.data };
+            }
+            random -= h.score;
+        }
+
+        // Fallback in case of rounding errors
+        const fallbackHeader = this.headers[0];
+        return { id: fallbackHeader.id, header: fallbackHeader.data };
+    }
+
+    /**
+     * Records the performance of a header request in memory.
+     * @param {string} headerId - The ID of the header used.
+     * @param {boolean} success - Whether the request was successful.
+     */
+    updatePerformance(headerId, success) {
+        if (headerId === 'fallback') return; // Do not track performance of the fallback
+
+        if (!this.performanceUpdates[headerId]) {
+            this.performanceUpdates[headerId] = { successes: 0, failures: 0 };
+        }
+        if (success) {
+            this.performanceUpdates[headerId].successes++;
+        } else {
+            this.performanceUpdates[headerId].failures++;
+        }
+    }
+
+    /**
+     * Flushes the aggregated performance updates to Firestore.
+     * @returns {Promise<void>}
+     */
+    async flushPerformanceUpdates() {
+        const updatesToFlush = this.performanceUpdates;
+        this.performanceUpdates = {}; // Reset the local cache
+
+        if (Object.keys(updatesToFlush).length === 0) {
+            return;
+        }
+
+        this.logger.log('INFO', `[HeaderManager] Flushing performance updates for ${Object.keys(updatesToFlush).length} headers.`);
+        const batch = this.firestore.batch();
+
+        for (const headerId in updatesToFlush) {
+            const updates = updatesToFlush[headerId];
+            const docRef = this.firestore.collection(this.collectionName).doc(headerId);
+            batch.update(docRef, {
+                totalRequests: FieldValue.increment(updates.successes + updates.failures),
+                successfulRequests: FieldValue.increment(updates.successes),
+                lastUsed: FieldValue.serverTimestamp()
+            });
+        }
+
+        try {
+            await batch.commit();
+            this.logger.log('SUCCESS', '[HeaderManager] Successfully flushed header performance updates to Firestore.');
+        } catch (error) {
+            this.logger.log('ERROR', '[HeaderManager] Failed to commit header performance batch.', { error: error.message });
+            // Put updates back in memory to try again next time
+            this.performanceUpdates = updatesToFlush; 
+        }
+    }
+}
+
+module.exports = { IntelligentHeaderManager };

package/functions/core/utils/intelligent_proxy_manager.js

@@ -0,0 +1,238 @@
+/**
+ * @fileoverview Manages a pool of proxies (AppScript URLs) using a simple locking mechanism.
+ * It selects an available (unlocked) proxy for each request and locks it upon failure.
+ * * This module is designed to be reusable and receives all dependencies
+ * (firestore, logger) and configuration via its constructor.
+ */
+const { FieldValue } = require('@google-cloud/firestore');
+
+class IntelligentProxyManager {
+    /**
+     * @param {object} firestore - An initialized Firestore instance.
+     * @param {object} logger - A logger instance (e.g., from sharedsetup).
+     * @param {object} config - Configuration object.
+     * @param {string[]} config.proxyUrls - An array of AppScript proxy URLs.
+     * @param {number} config.cacheDurationMs - How long to cache proxy status in memory (in ms).
+     * @param {string} config.proxiesCollectionName - Firestore collection to check for locks.
+     * @param {string} config.proxyPerformanceDocPath - Firestore doc path for performance tracking.
+     */
+    constructor(firestore, logger, config) {
+        if (!firestore || !logger || !config) {
+            throw new Error("IntelligentProxyManager requires firestore, logger, and config objects.");
+        }
+        if (!config.proxyUrls || !config.cacheDurationMs || !config.proxiesCollectionName || !config.proxyPerformanceDocPath) {
+            throw new Error("IntelligentProxyManager config is missing required keys (proxyUrls, cacheDurationMs, proxiesCollectionName, proxyPerformanceDocPath).");
+        }
+        
+        this.firestore = firestore;
+        this.logger = logger;
+        
+        // Load from config
+        this.proxyUrls = config.proxyUrls.filter(Boolean); // Filter out any empty/null URLs
+        this.CONFIG_CACHE_DURATION_MS = config.cacheDurationMs;
+        this.PROXIES_COLLECTION = config.proxiesCollectionName;
+        this.PERFORMANCE_DOC_PATH = config.proxyPerformanceDocPath;
+
+        // Internal state
+        this.proxies = {}; // Stores { owner, url, status ('unlocked', 'locked') }
+        this.configLastLoaded = 0;
+
+        if (this.proxyUrls.length === 0) {
+            this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
+        } else {
+             this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and Locking Mechanism.`);
+        }
+    }
+
+    /**
+     * Loads proxy configuration and lock status from Firestore.
+     * Caches the configuration to reduce frequent database reads.
+     */
+    async _loadConfig() {
+        if (Date.now() - this.configLastLoaded < this.CONFIG_CACHE_DURATION_MS) {
+            return; // Cache is fresh
+        }
+        if (this.proxyUrls.length === 0) {
+            return; // No proxies to load
+        }
+
+        this.logger.log('INFO', "[ProxyManager] Refreshing proxy configuration and lock status...");
+        try {
+            const tempProxyStatus = {};
+
+            // 1. Initialize all known proxies from config
+            for (const url of this.proxyUrls) {
+                const owner = new URL(url).hostname; // Or derive a unique ID differently
+                tempProxyStatus[owner] = { owner, url, status: 'unlocked' }; // Default to unlocked
+            }
+            
+            // 2. Load performance doc to get lock statuses
+            const doc = await this.firestore.doc(this.PERFORMANCE_DOC_PATH).get();
+            if (doc.exists) {
+                const data = doc.data();
+                if (data.locks) {
+                    for (const owner in data.locks) {
+                        if (tempProxyStatus[owner] && data.locks[owner].locked === true) {
+                            tempProxyStatus[owner].status = 'locked';
+                        }
+                    }
+                }
+            }
+
+            this.proxies = tempProxyStatus;
+            this.configLastLoaded = Date.now();
+            this.logger.log('SUCCESS', `[ProxyManager] Refreshed ${Object.keys(this.proxies).length} proxy statuses.`);
+            
+        } catch (error) {
+            this.logger.log('ERROR', '[ProxyManager] Failed to load proxy config from Firestore.', { 
+                errorMessage: error.message,
+                path: this.PERFORMANCE_DOC_PATH
+            });
+        }
+    }
+
+    /**
+     * Selects an available (unlocked) proxy from the in-memory cache.
+     * @returns {Promise<{owner: string, url: string}>}
+     */
+    async _selectProxy() {
+        await this._loadConfig();
+
+        const unlockedProxies = Object.values(this.proxies).filter(p => p.status === 'unlocked');
+
+        if (unlockedProxies.length === 0) {
+            this.logger.log('ERROR', '[ProxyManager] All proxies are locked. No proxy available.');
+            throw new Error("All proxies are locked. Cannot make request.");
+        }
+
+        // Return a random unlocked proxy
+        const selected = unlockedProxies[Math.floor(Math.random() * unlockedProxies.length)];
+        return { owner: selected.owner, url: selected.url };
+    }
+
+    /**
+     * Locks a proxy by setting its status in memory and writing to Firestore.
+     * @param {string} owner - The owner/ID of the proxy to lock.
+     */
+    async lockProxy(owner) {
+        // 1. Update in-memory cache immediately
+        if (this.proxies[owner]) {
+            this.proxies[owner].status = 'locked';
+        }
+
+        this.logger.log('WARN', `[ProxyManager] Locking proxy: ${owner}`);
+        
+        // 2. Update Firestore
+        try {
+            const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH);
+            // Use dot notation to update a specific field in the 'locks' map
+            await docRef.set({
+                locks: {
+                    [owner]: {
+                        locked: true,
+                        lastLocked: FieldValue.serverTimestamp()
+                    }
+                }
+            }, { merge: true });
+        } catch (error) {
+            this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message });
+        }
+    }
+
+    /**
+     * Makes a fetch request using a selected proxy.
+     * @param {string} targetUrl - The URL to fetch.
+     * @param {object} options - Fetch options (e.g., headers).
+     * @returns {Promise<object>} A mock Response object.
+     */
+    async fetch(targetUrl, options = {}) {
+        let proxy = null;
+        try {
+            // 1. Select Proxy
+            proxy = await this._selectProxy();
+        } catch (error) {
+            // No proxies available
+            return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
+        }
+        
+        // 2. Make Request
+        const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
+        
+        // 3. Handle Proxy Failure (e.g., quota error, network error)
+        if (!response.ok && response.isUrlFetchError) {
+             // isUrlFetchError is a custom flag from _fetchViaAppsScript
+            await this.lockProxy(proxy.owner);
+        }
+        
+        return response;
+    }
+
+    /**
+     * Internal function to call the Google AppScript proxy.
+     * @private
+     */
+    async _fetchViaAppsScript(proxyUrl, targetUrl, options) {
+        const payload = {
+            url: targetUrl,
+            options: options
+        };
+
+        try {
+            const response = await fetch(proxyUrl, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(payload)
+            });
+
+            if (!response.ok) {
+                 const errorText = await response.text();
+                 this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, { 
+                     status: response.status, 
+                     proxy: proxyUrl, 
+                     error: errorText 
+                 });
+                return { 
+                    ok: false, 
+                    status: response.status,
+                    isUrlFetchError: true, // Flag this as a proxy infrastructure error
+                    error: { message: `Proxy infrastructure failed with status ${response.status}` },
+                    headers: response.headers,
+                    text: () => Promise.resolve(errorText)
+                };
+            }
+            
+            const proxyResponse = await response.json();
+
+            if (proxyResponse.error) {
+                const errorMsg = proxyResponse.error.message || '';
+                // Check for Google-side quota errors
+                if (errorMsg.toLowerCase().includes('service invoked too many times')) {
+                    this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error });
+                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, headers: new Headers() };
+                }
+                // Other errors returned by the AppScript
+                return { ok: false, status: 500, error: proxyResponse.error, headers: new Headers(), text: () => Promise.resolve(errorMsg) };
+            }
+
+            // Success
+            return {
+                ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300,
+                status: proxyResponse.statusCode,
+                headers: new Headers(proxyResponse.headers || {}),
+                json: () => Promise.resolve(JSON.parse(proxyResponse.body)),
+                text: () => Promise.resolve(proxyResponse.body),
+            };
+        } catch (networkError) {
+             this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message });
+            return {
+                ok: false,
+                status: 0, // Network errors don't have a status
+                isUrlFetchError: true, // Flag this as a proxy infrastructure error
+                error: { message: `Network error: ${networkError.message}` },
+                headers: new Headers()
+            };
+        }
+    }
+}
+
+module.exports = { IntelligentProxyManager };

package/functions/dispatcher/helpers/dispatch_helpers.js

@@ -0,0 +1,50 @@
+/**
+ * @fileoverview Contains the core logic for dispatching tasks in batches.
+ */
+const { logger } = require("sharedsetup")(__filename); // Assuming sharedsetup is available
+
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Publishes tasks in batches with delays.
+ * @param {Array} tasks - Array of tasks to publish.
+ * @param {object} pubsubClient - Initialized Google Cloud PubSub client.
+ * @param {object} config - Configuration object.
+ * @param {string} config.topicName - Target Pub/Sub topic name.
+ * @param {number} config.batchSize - Number of tasks per batch.
+ * @param {number} config.batchDelayMs - Delay between batches in milliseconds.
+ */
+async function dispatchTasksInBatches(tasks, pubsubClient, config) {
+    const { topicName, batchSize, batchDelayMs } = config;
+    const topic = pubsubClient.topic(topicName);
+    let totalTasksQueued = 0;
+
+    logger.log('INFO', `[Module Dispatcher] Received ${tasks.length} tasks. Creating batches...`);
+
+    for (let i = 0; i < tasks.length; i += batchSize) {
+        const batch = tasks.slice(i, i + batchSize);
+
+        try {
+            // Publish messages for the current batch
+            await Promise.all(batch.map(task => topic.publishMessage({ json: task })));
+            totalTasksQueued += batch.length;
+            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks.`);
+
+            // Apply delay if it's not the last batch
+            if (i + batchSize < tasks.length) {
+                await sleep(batchDelayMs);
+            }
+        } catch (publishError) {
+            logger.log('ERROR', `[Module Dispatcher] Failed to publish batch ${Math.ceil((i + 1) / batchSize)}. Error: ${publishError.message}`, { errorStack: publishError.stack });
+            // Decide on error handling: continue to next batch or throw?
+            // For now, log and continue, but this might need adjustment based on requirements.
+        }
+    }
+
+    logger.log('SUCCESS', `[Module Dispatcher] Successfully dispatched ${totalTasksQueued} tasks in ${Math.ceil(tasks.length / batchSize)} batches.`);
+    return totalTasksQueued; // Return count for confirmation
+}
+
+module.exports = {
+    dispatchTasksInBatches
+};

package/functions/dispatcher/index.js

@@ -0,0 +1,52 @@
+/**
+ * @fileoverview Main entry point for the Dispatcher function logic within the module.
+ */
+const { logger } = require("sharedsetup")(__filename); // Assuming sharedsetup is available
+const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
+
+/**
+ * Creates the Pub/Sub triggered Cloud Function handler for the Dispatcher.
+ * @param {object} pubsubClient - Initialized Google Cloud PubSub client.
+ * @param {object} config - Configuration object loaded from the calling function's context.
+ * @returns {Function} The Cloud Function handler.
+ */
+function createDispatcherHandler(pubsubClient, config) {
+    return async (message, context) => {
+        try {
+            // 1. Decode Message
+            if (!message.data) {
+                logger.log('WARN', '[Module Dispatcher] Received message without data.');
+                return; // Acknowledge and exit gracefully
+            }
+            const decodedMessage = JSON.parse(Buffer.from(message.data, 'base64').toString());
+            const { tasks } = decodedMessage;
+
+            if (!tasks || !Array.isArray(tasks) || tasks.length === 0) {
+                logger.log('WARN', '[Module Dispatcher] Received message with no valid tasks. Nothing to do.');
+                return; // Acknowledge and exit
+            }
+
+            // 2. Validate Config (Basic check)
+            if (!config || !config.topicName || !config.batchSize || !config.batchDelayMs) {
+                logger.log('ERROR', '[Module Dispatcher] Invalid configuration provided.', { config });
+                throw new Error("Dispatcher module received invalid configuration.");
+            }
+
+            // 3. Dispatch Tasks
+            await dispatchTasksInBatches(tasks, pubsubClient, config);
+
+            // If we reach here, the process (including logging success/errors) is handled within dispatchTasksInBatches
+
+        } catch (error) {
+            // Catch errors during decoding or major setup issues
+            logger.log('ERROR', '[Module Dispatcher] FATAL error processing message', { errorMessage: error.message, errorStack: error.stack });
+            // Re-throw to signal Pub/Sub to potentially retry (depending on subscription settings)
+            throw error;
+        }
+    };
+}
+
+module.exports = {
+    createDispatcherHandler,
+    helpers: { dispatchTasksInBatches } // Export helpers if needed elsewhere
+};

package/index.js

@@ -9,6 +9,7 @@ const Orchestrator = require('./functions/orchestrator');
 const TaskEngine = require('./functions/task-engine');
 const ComputationSystem = require('./functions/computation-system');
 const GenericAPI = require('./functions/generic-api'); // <-- ADD THIS
+const Dispatcher = require('./functions/dispatcher'); // <-- ADD THIS
 
 module.exports = {
   core,
@@ -16,4 +17,5 @@ module.exports = {
   TaskEngine,
   ComputationSystem,
   GenericAPI, // <-- AND ADD THIS
+  Dispatcher, // <-- AND ADD THI
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -9,7 +9,8 @@
     "functions/task-engine/",
     "functions/core/",
     "functions/computation-system/",
-    "functions/generic-api/"
+    "functions/generic-api/",
+    "functions/dispatcher/"
   ],
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"