bulltrackers-module 1.0.151 → 1.0.153

package/functions/core/utils/intelligent_header_manager.js

@@ -19,41 +19,24 @@ class IntelligentHeaderManager {
      * @param {string} config.fallbackUserAgent - A fallback User-Agent if loading fails.
      */
     constructor(firestore, logger, config) {
-        if (!firestore || !logger || !config) {
-            throw new Error("IntelligentHeaderManager requires firestore, logger, and config objects.");
-        }
-        if (!config.headersCollectionName || !config.cacheDurationMs || !config.fallbackUserAgent) {
-             throw new Error("IntelligentHeaderManager config is missing required keys (headersCollectionName, cacheDurationMs, fallbackUserAgent).");
-        }
-
+        if (!firestore || !logger || !config) { throw new Error("IntelligentHeaderManager requires firestore, logger, and config objects."); }
+        if (!config.headersCollectionName || !config.cacheDurationMs || !config.fallbackUserAgent) { throw new Error("IntelligentHeaderManager config is missing required keys (headersCollectionName, cacheDurationMs, fallbackUserAgent)."); }
         this.firestore = firestore;
         this.logger = logger;
-
-        // Load from config
         this.collectionName = config.headersCollectionName;
         this.cacheDuration = config.cacheDurationMs;
         this.fallbackUserAgent = config.fallbackUserAgent;
-
-        // Internal state
         this.headers = [];
         this.lastFetched = null;
         this.performanceUpdates = {};
-        
-        // Initialize with fallback to ensure we always have at least one header
-        this._setFallbackHeader();
-    }
+        this._setFallbackHeader(); }
 
     /**
      * Sets the fallback header as the only available header.
      * @private
      */
     _setFallbackHeader() {
-        this.headers = [{
-            id: 'fallback',
-            data: { 'User-Agent': this.fallbackUserAgent },
-            performance: { total: 1, success: 1 }
-        }];
-    }
+        this.headers = [{ id: 'fallback', data: { 'User-Agent': this.fallbackUserAgent }, performance: { total: 1, success: 1 } }]; }
 
     /**
      * Fetches and caches header documents from Firestore. If the cache is fresh, it does nothing.
@@ -62,58 +45,22 @@ class IntelligentHeaderManager {
      */
     async _loadHeaders() {
         const now = new Date();
-        if (this.lastFetched && (now - this.lastFetched < this.cacheDuration) && this.headers.length > 0) {
-            return; // Cache is fresh
-        }
-
+        if (this.lastFetched && (now - this.lastFetched < this.cacheDuration) && this.headers.length > 0) { return;  }
         try {
             this.logger.log('INFO', '[HeaderManager] Refreshing header performance data from Firestore...');
             const snapshot = await this.firestore.collection(this.collectionName).get();
-            
-            if (snapshot.empty) {
-                this.logger.log('WARN', `[HeaderManager] No documents found in headers collection: ${this.collectionName}. Using fallback.`);
-                this._setFallbackHeader();
-                this.lastFetched = new Date();
-                return;
-            }
-
+            if (snapshot.empty) { this.logger.log('WARN', `[HeaderManager] No documents found in headers collection: ${this.collectionName}. Using fallback.`); this._setFallbackHeader(); this.lastFetched = new Date(); return; }
             const loadedHeaders = snapshot.docs.map(doc => {
                 const docData = doc.data();
-                
-                // Validate that the document has the required 'headers' field
-                if (!docData.headers) {
-                    this.logger.log('WARN', `[HeaderManager] Document ${doc.id} is missing 'headers' field. Skipping.`);
-                    return null;
-                }
-                
-                return {
-                    id: doc.id,
-                    data: docData.headers,  // Changed from 'header' to 'headers'
-                    performance: {
-                        total: docData.totalRequests || 0,
-                        success: docData.successfulRequests || 0
-                    }
-                };
-            }).filter(Boolean); // Remove null entries
-
-            if (loadedHeaders.length === 0) {
-                this.logger.log('WARN', "[HeaderManager] No valid headers found in collection. Using fallback.");
-                this._setFallbackHeader();
-            } else {
-                this.headers = loadedHeaders;
-                this.logger.log('INFO', `[HeaderManager] Successfully loaded ${this.headers.length} headers.`);
-            }
-
+                if (!docData.headers) { this.logger.log('WARN', `[HeaderManager] Document ${doc.id} is missing 'headers' field. Skipping.`); return null; }
+                return { id: doc.id, data: docData.headers, performance: { total: docData.totalRequests || 0, success: docData.successfulRequests || 0 } };
+            }).filter(Boolean); 
+            if (loadedHeaders.length === 0) { this.logger.log('WARN', "[HeaderManager] No valid headers found in collection. Using fallback."); this._setFallbackHeader();
+            } else { this.headers = loadedHeaders; this.logger.log('INFO', `[HeaderManager] Successfully loaded ${this.headers.length} headers.`); }
             this.lastFetched = new Date();
-            
         } catch (error) {
-            this.logger.log('ERROR', '[HeaderManager] Failed to load headers from Firestore. Using fallback.', { 
-                errorMessage: error.message,
-                errorStack: error.stack,
-                collection: this.collectionName
-            });
-            this._setFallbackHeader();
-        }
+            this.logger.log('ERROR', '[HeaderManager] Failed to load headers from Firestore. Using fallback.', {  errorMessage: error.message, errorStack: error.stack, collection: this.collectionName });
+            this._setFallbackHeader(); }
     }
 
     /**
@@ -123,34 +70,11 @@ class IntelligentHeaderManager {
      */
     async selectHeader() {
         await this._loadHeaders();
-
-        // This should never happen now because we always have fallback, but keep as safety check
-        if (!this.headers || this.headers.length === 0) {
-            this.logger.log('ERROR', '[HeaderManager] No headers available after load attempt. Returning fallback.');
-            return { id: 'fallback', header: { 'User-Agent': this.fallbackUserAgent } };
-        }
-
-        // Calculate total score for weighted random selection
+        if (!this.headers || this.headers.length === 0) { this.logger.log('ERROR', '[HeaderManager] No headers available after load attempt. Returning fallback.'); return { id: 'fallback', header: { 'User-Agent': this.fallbackUserAgent } }; }
         let totalScore = 0;
-        const weightedHeaders = this.headers.map(h => {
-            const successRate = (h.performance.total === 0) ? 0.5 : (h.performance.success / h.performance.total);
-            // Add-1 smoothing to avoid zero probability for new headers
-            const score = (successRate + 1) * 10; 
-            totalScore += score;
-            return { ...h, score };
-        });
-
-        // Select a random value
+        const weightedHeaders = this.headers.map(h => { const successRate = (h.performance.total === 0) ? 0.5 : (h.performance.success / h.performance.total); const score = (successRate + 1) * 10;  totalScore += score; return { ...h, score }; });
         let random = Math.random() * totalScore;
-        
-        for (const h of weightedHeaders) {
-            if (random < h.score) {
-                return { id: h.id, header: h.data };
-            }
-            random -= h.score;
-        }
-
-        // Fallback in case of rounding errors
+        for (const h of weightedHeaders) { if (random < h.score) { return { id: h.id, header: h.data }; } random -= h.score; }
         const fallbackHeader = this.headers[0];
         return { id: fallbackHeader.id, header: fallbackHeader.data };
     }
@@ -161,16 +85,10 @@ class IntelligentHeaderManager {
      * @param {boolean} success - Whether the request was successful.
      */
     updatePerformance(headerId, success) {
-        if (headerId === 'fallback') return; // Do not track performance of the fallback
-
-        if (!this.performanceUpdates[headerId]) {
-            this.performanceUpdates[headerId] = { successes: 0, failures: 0 };
-        }
-        if (success) {
-            this.performanceUpdates[headerId].successes++;
-        } else {
-            this.performanceUpdates[headerId].failures++;
-        }
+        if (headerId === 'fallback') return; 
+        if (!this.performanceUpdates[headerId]) { this.performanceUpdates[headerId] = { successes: 0, failures: 0 }; }
+        if (success) { this.performanceUpdates[headerId].successes++;
+        } else { this.performanceUpdates[headerId].failures++; }
     }
 
     /**
@@ -179,34 +97,14 @@ class IntelligentHeaderManager {
      */
     async flushPerformanceUpdates() {
         const updatesToFlush = this.performanceUpdates;
-        this.performanceUpdates = {}; // Reset the local cache
-
-        if (Object.keys(updatesToFlush).length === 0) {
-            return;
-        }
-
+        this.performanceUpdates = {}; 
+        if (Object.keys(updatesToFlush).length === 0) { return; }
         this.logger.log('INFO', `[HeaderManager] Flushing performance updates for ${Object.keys(updatesToFlush).length} headers.`);
         const batch = this.firestore.batch();
-
-        for (const headerId in updatesToFlush) {
-            const updates = updatesToFlush[headerId];
-            const docRef = this.firestore.collection(this.collectionName).doc(headerId);
-            batch.update(docRef, {
-                totalRequests: FieldValue.increment(updates.successes + updates.failures),
-                successfulRequests: FieldValue.increment(updates.successes),
-                lastUsed: FieldValue.serverTimestamp()
-            });
-        }
-
-        try {
-            await batch.commit();
-            this.logger.log('SUCCESS', '[HeaderManager] Successfully flushed header performance updates to Firestore.');
-        } catch (error) {
-            this.logger.log('ERROR', '[HeaderManager] Failed to commit header performance batch.', { error: error.message });
-            // Put updates back in memory to try again next time
-            this.performanceUpdates = updatesToFlush; 
-        }
-    }
+        for (const headerId in updatesToFlush) { const updates = updatesToFlush[headerId]; const docRef = this.firestore.collection(this.collectionName).doc(headerId);
+            batch.update(docRef, { totalRequests: FieldValue.increment(updates.successes + updates.failures), successfulRequests: FieldValue.increment(updates.successes), lastUsed: FieldValue.serverTimestamp() }); }
+        try { await batch.commit(); this.logger.log('SUCCESS', '[HeaderManager] Successfully flushed header performance updates to Firestore.');
+        } catch (error) { this.logger.log('ERROR', '[HeaderManager] Failed to commit header performance batch.', { error: error.message }); this.performanceUpdates = updatesToFlush;  } }
 }
 
 module.exports = { IntelligentHeaderManager };

package/functions/core/utils/intelligent_proxy_manager.js

@@ -18,35 +18,19 @@ class IntelligentProxyManager {
      * @param {boolean} [config.proxyLockingEnabled=true] - Whether to enable the proxy locking mechanism.
      */
     constructor(firestore, logger, config) {
-        if (!firestore || !logger || !config) {
-            throw new Error("IntelligentProxyManager requires firestore, logger, and config objects.");
-        }
-        if (!config.proxyUrls || !config.cacheDurationMs || !config.proxiesCollectionName || !config.proxyPerformanceDocPath) {
-            throw new Error("IntelligentProxyManager config is missing required keys (proxyUrls, cacheDurationMs, proxiesCollectionName, proxyPerformanceDocPath).");
-        }
-        
+        if (!firestore || !logger || !config) { throw new Error("IntelligentProxyManager requires firestore, logger, and config objects."); }
+        if (!config.proxyUrls || !config.cacheDurationMs || !config.proxiesCollectionName || !config.proxyPerformanceDocPath) { throw new Error("IntelligentProxyManager config is missing required keys (proxyUrls, cacheDurationMs, proxiesCollectionName, proxyPerformanceDocPath)."); }
         this.firestore = firestore;
         this.logger = logger;
-        
-        // Load from config
-        this.proxyUrls = config.proxyUrls.filter(Boolean); // Filter out any empty/null URLs
+        this.proxyUrls = config.proxyUrls.filter(Boolean);
         this.CONFIG_CACHE_DURATION_MS = config.cacheDurationMs;
         this.PROXIES_COLLECTION = config.proxiesCollectionName;
         this.PERFORMANCE_DOC_PATH = config.proxyPerformanceDocPath;
-        // NEW: Check for the locking enabled flag, default to true
         this.proxyLockingEnabled = config.proxyLockingEnabled !== false;
-
-        // Internal state
-        this.proxies = {}; // Stores { owner, url, status ('unlocked', 'locked') }
+        this.proxies = {}; 
         this.configLastLoaded = 0;
-
-        if (this.proxyUrls.length === 0) {
-            this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
-        } else {
-             // UPDATED: Log locking status
-             const lockingStatus = this.proxyLockingEnabled ? "Locking Mechanism Enabled" : "Locking Mechanism DISABLED";
-             this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and ${lockingStatus}.`);
-        }
+        if (this.proxyUrls.length === 0) { this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
+        } else { const lockingStatus = this.proxyLockingEnabled ? "Locking Mechanism Enabled" : "Locking Mechanism DISABLED"; this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and ${lockingStatus}.`); }
     }
 
     /**
@@ -54,50 +38,19 @@ class IntelligentProxyManager {
      * Caches the configuration to reduce frequent database reads.
      */
     async _loadConfig() {
-        if (Date.now() - this.configLastLoaded < this.CONFIG_CACHE_DURATION_MS) {
-            return; // Cache is fresh
-        }
-        if (this.proxyUrls.length === 0) {
-            return; // No proxies to load
-        }
-
+        if (Date.now() - this.configLastLoaded < this.CONFIG_CACHE_DURATION_MS) { return;  }
+        if (this.proxyUrls.length === 0) { return; }
         this.logger.log('INFO', "[ProxyManager] Refreshing proxy configuration and lock status...");
-        try {
-            const tempProxyStatus = {};
-
-            // 1. Initialize all known proxies from config
-            for (const url of this.proxyUrls) {
-                const owner = new URL(url).hostname; 
-                tempProxyStatus[owner] = { owner, url, status: 'unlocked' }; // Default to unlocked
-            }
-            
-            // 2. Load performance doc to get lock statuses - ONLY if locking is enabled
-            if (this.proxyLockingEnabled) {
-                const doc = await this.firestore.doc(this.PERFORMANCE_DOC_PATH).get();
-                if (doc.exists) {
-                    const data = doc.data();
-                    if (data.locks) {
-                        for (const owner in data.locks) {
-                            if (tempProxyStatus[owner] && data.locks[owner].locked === true) {
-                                tempProxyStatus[owner].status = 'locked';
-                            }
-                        }
-                    }
-                }
-            } else {
-                 this.logger.log('TRACE', '[ProxyManager] Proxy locking is disabled, skipping lock status check.');
-            }
-
+        try { const tempProxyStatus = {};
+            for (const url of this.proxyUrls) { const owner = new URL(url).hostname;  tempProxyStatus[owner] = { owner, url, status: 'unlocked' };  }
+            if (this.proxyLockingEnabled) { const doc = await this.firestore.doc(this.PERFORMANCE_DOC_PATH).get();
+                if (doc.exists) { const data = doc.data(); if (data.locks) { for (const owner in data.locks) { if (tempProxyStatus[owner] && data.locks[owner].locked === true) { tempProxyStatus[owner].status = 'locked'; } } } }
+            } else { this.logger.log('TRACE', '[ProxyManager] Proxy locking is disabled, skipping lock status check.'); }
             this.proxies = tempProxyStatus;
             this.configLastLoaded = Date.now();
             this.logger.log('SUCCESS', `[ProxyManager] Refreshed ${Object.keys(this.proxies).length} proxy statuses.`);
-            
         } catch (error) {
-            this.logger.log('ERROR', '[ProxyManager] Failed to load proxy config from Firestore.', { 
-                errorMessage: error.message,
-                path: this.PERFORMANCE_DOC_PATH
-            });
-        }
+            this.logger.log('ERROR', '[ProxyManager] Failed to load proxy config from Firestore.', { errorMessage: error.message, path: this.PERFORMANCE_DOC_PATH }); }
     }
 
     /**
@@ -107,20 +60,9 @@ class IntelligentProxyManager {
     async _selectProxy() {
         await this._loadConfig();
 
-        // UPDATED: If locking is disabled, all proxies are available. If enabled, filter.
-        const availableProxies = this.proxyLockingEnabled
-            ? Object.values(this.proxies).filter(p => p.status === 'unlocked')
-            : Object.values(this.proxies);
-
-        if (availableProxies.length === 0) {
-            const errorMsg = this.proxyLockingEnabled 
-                ? "All proxies are locked. No proxy available."
-                : "No proxies are loaded. Cannot make request.";
-            this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`);
-            throw new Error(errorMsg);
-        }
-
-        // Return a random available proxy
+        const availableProxies = this.proxyLockingEnabled ? Object.values(this.proxies).filter(p => p.status === 'unlocked') : Object.values(this.proxies);
+        if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled  ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request."; 
+            this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
         const selected = availableProxies[Math.floor(Math.random() * availableProxies.length)];
         return { owner: selected.owner, url: selected.url };
     }
@@ -130,34 +72,12 @@ class IntelligentProxyManager {
      * @param {string} owner - The owner/ID of the proxy to lock.
      */
     async lockProxy(owner) {
-        // NEW: Check if locking is enabled.
-        if (!this.proxyLockingEnabled) {
-            this.logger.log('TRACE', `[ProxyManager] Locking skipped for ${owner} (locking is disabled).`);
-            return;
-        }
-
-        // 1. Update in-memory cache immediately
-        if (this.proxies[owner]) {
-            this.proxies[owner].status = 'locked';
-        }
-
+        if (!this.proxyLockingEnabled) { this.logger.log('TRACE', `[ProxyManager] Locking skipped for ${owner} (locking is disabled).`); return; }
+        if (this.proxies[owner]) { this.proxies[owner].status = 'locked'; }
         this.logger.log('WARN', `[ProxyManager] Locking proxy: ${owner}`);
-        
-        // 2. Update Firestore
-        try {
-            const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH);
-            // Use dot notation to update a specific field in the 'locks' map
-            await docRef.set({
-                locks: {
-                    [owner]: {
-                        locked: true,
-                        lastLocked: FieldValue.serverTimestamp()
-                    }
-                }
-            }, { merge: true });
-        } catch (error) {
-            this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message });
-        }
+        try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH);
+            await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
+        } catch (error) { this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message }); }
     }
 
     /**
@@ -168,24 +88,10 @@ class IntelligentProxyManager {
      */
     async fetch(targetUrl, options = {}) {
         let proxy = null;
-        try {
-            // 1. Select Proxy
-            proxy = await this._selectProxy();
-        } catch (error) {
-            // No proxies available
-            return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
-        }
-        
-        // 2. Make Request
+        try { proxy = await this._selectProxy();
+        } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
         const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
-        
-        // 3. Handle Proxy Failure (e.g., quota error, network error)
-        if (!response.ok && response.isUrlFetchError) {
-             // isUrlFetchError is a custom flag from _fetchViaAppsScript
-             // This call is now gated by the flag inside lockProxy()
-            await this.lockProxy(proxy.owner);
-        }
-        
+        if (!response.ok && response.isUrlFetchError) { await this.lockProxy(proxy.owner); }
         return response;
     }
 
@@ -194,68 +100,24 @@ class IntelligentProxyManager {
      * @private
      */
     async _fetchViaAppsScript(proxyUrl, targetUrl, options) {
-        // Spread the 'options' object (which contains headers, method, body, etc.)
-        // into the top-level payload.
-        const payload = {
-            url: targetUrl,
-            ...options
-        };
-
+        const payload = { url: targetUrl, ...options };
         try {
-            const response = await fetch(proxyUrl, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify(payload)
-            });
-
+            const response = await fetch(proxyUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
             if (!response.ok) {
                  const errorText = await response.text();
-                 this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, { 
-                     status: response.status, 
-                     proxy: proxyUrl, 
-                     error: errorText 
-                 });
-                return { 
-                    ok: false, 
-                    status: response.status,
-                    isUrlFetchError: true, // Flag this as a proxy infrastructure error
-                    error: { message: `Proxy infrastructure failed with status ${response.status}` },
-                    headers: response.headers,
-                    text: () => Promise.resolve(errorText)
-                };
-            }
-            
+                 this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, {  status: response.status,  proxy: proxyUrl,  error: errorText  });
+                return {  ok: false,  status: response.status, isUrlFetchError: true,  error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) }; }
             const proxyResponse = await response.json();
-
             if (proxyResponse.error) {
                 const errorMsg = proxyResponse.error.message || '';
-                // Check for Google-side quota errors
                 if (errorMsg.toLowerCase().includes('service invoked too many times')) {
                     this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error });
-                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, headers: new Headers() };
-                }
-                // Other errors returned by the AppScript
-                return { ok: false, status: 500, error: proxyResponse.error, headers: new Headers(), text: () => Promise.resolve(errorMsg) };
-            }
-
-            // Success
-            return {
-                ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300,
-                status: proxyResponse.statusCode,
-                headers: new Headers(proxyResponse.headers || {}),
-                json: () => Promise.resolve(JSON.parse(proxyResponse.body)),
-                text: () => Promise.resolve(proxyResponse.body),
-            };
+                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, headers: new Headers() }; }
+                return { ok: false, status: 500, error: proxyResponse.error, headers: new Headers(), text: () => Promise.resolve(errorMsg) }; }
+            return { ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300, status: proxyResponse.statusCode, headers: new Headers(proxyResponse.headers || {}), json: () => Promise.resolve(JSON.parse(proxyResponse.body)), text: () => Promise.resolve(proxyResponse.body), };
         } catch (networkError) {
              this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message });
-            return {
-                ok: false,
-                status: 0, // Network errors don't have a status
-                isUrlFetchError: true, // Flag this as a proxy infrastructure error
-                error: { message: `Network error: ${networkError.message}` },
-                headers: new Headers()
-            };
-        }
+            return { ok: false, status: 0,  isUrlFetchError: true, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() }; }
     }
 }
 

package/functions/core/utils/pubsub_utils.js

@@ -18,36 +18,19 @@
 async function batchPublishTasks(dependencies, config) {
     const { pubsub, logger } = dependencies;
     const { topicName, tasks, taskType, maxPubsubBatchSize = 500 } = config;
-
-    if (!tasks || tasks.length === 0) {
-        logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`);
-        return;
-    }
+    if (!tasks || tasks.length === 0) { logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`); return; }
     logger.log('INFO',`[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
     const topic = pubsub.topic(topicName);
     let messagesPublished = 0;
-
     try {
-        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) {
-            const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
-            const batchPromises = batchTasks.map(task => {
-                const dataBuffer = Buffer.from(JSON.stringify(task));
-                return topic.publishMessage({ data: dataBuffer })
-                    .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task }));
-            });
+        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) { const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
+            const batchPromises = batchTasks.map(task => { const dataBuffer = Buffer.from(JSON.stringify(task));
+                return topic.publishMessage({ data: dataBuffer }) .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task })); });
             await Promise.all(batchPromises);
             messagesPublished += batchTasks.length;
-            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`);
-        }
-
+            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`); }
         logger.log('SUCCESS', `[Core Utils] Finished publishing ${messagesPublished} ${taskType} tasks to ${topicName}.`);
-
-    } catch (error) {
-        logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message });
-        throw error;
-    }
+    } catch (error) { logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message }); throw error; }
 }
 
-module.exports = {
-  batchPublishTasks,
-};
+module.exports = { batchPublishTasks };

package/functions/dispatcher/helpers/dispatch_helpers.js

@@ -1,5 +1,5 @@
 /**
- * @fileoverview Sub-pipe for dispatching tasks.
+ * @fileoverview Sub-pipe for dispatching tasks. Handles the speed that tasks are submitted to the task engine for scaling optimisation.
  * REFACTORED: Now stateless and receives dependencies.
  *
  * --- MODIFICATION ---
@@ -24,38 +24,17 @@ async function dispatchTasksInBatches(tasks, dependencies, config) {
     const { topicName, batchSize, batchDelayMs } = config;
     const topic = pubsub.topic(topicName);
     let totalTasksQueued = 0;
-
     logger.log('INFO', `[Module Dispatcher] Received ${tasks.length} tasks. Creating batches...`);
-
     for (let i = 0; i < tasks.length; i += batchSize) {
-        const batch = tasks.slice(i, i + batchSize); // e.g., batch is [task1, task2, ... task100]
-
-        try {
-            // --- START MODIFICATION ---
-            // Wrap the entire batch of tasks into a single object payload.
-            // This is the payload the Task Engine will now receive.
-            const messagePayload = { tasks: batch };
-            
-            // Publish the entire batch as a single message
-            await topic.publishMessage({ json: messagePayload });
-            
-            // --- END MODIFICATION ---
-
-            totalTasksQueued += batch.length;
-            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks as a single message.`);
-
-            if (i + batchSize < tasks.length) {
-                await sleep(batchDelayMs);
-            }
-        } catch (publishError) {
-            logger.log('ERROR', `[Module Dispatcher] Failed to publish batch ${Math.ceil((i + 1) / batchSize)}. Error: ${publishError.message}`, { errorStack: publishError.stack });
-        }
-    }
-
+        const batch = tasks.slice(i, i + batchSize); 
+        try { const messagePayload = { tasks: batch }; 
+        await topic.publishMessage({ json: messagePayload });
+        totalTasksQueued += batch.length;
+        logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks as a single message.`);
+        if (i + batchSize < tasks.length) { await sleep(batchDelayMs); }
+        } catch (publishError) { logger.log('ERROR', `[Module Dispatcher] Failed to publish batch ${Math.ceil((i + 1) / batchSize)}. Error: ${publishError.message}`, { errorStack: publishError.stack }); } }
     logger.log('SUCCESS', `[Module Dispatcher] Successfully dispatched ${totalTasksQueued} tasks in ${Math.ceil(tasks.length / batchSize)} batches.`);
     return totalTasksQueued;
 }
 
-module.exports = {
-    dispatchTasksInBatches
-};
+module.exports = { dispatchTasksInBatches };

package/functions/dispatcher/index.js

@@ -1,12 +1,10 @@
 /**
  * @fileoverview Main entry point for the Dispatcher function.
  * REFACTORED: This file now contains the main pipe function 'handleRequest'.
+ * This is built to handle the large load of tasks produced by the orchestrator, to prevent a flood of requests which would cause problematic rate limiting within GCP scaling. This is effectively a cost efficiency script.
  */
 
-// --- 1. REMOVE the circular require ---
-// const { pipe } = require('../../index'); // <<< REMOVE THIS LINE
 
-// --- 2. ADD direct require for the specific sub-pipe needed ---
 const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
 
 
@@ -20,34 +18,13 @@ const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
  */
 async function handleRequest(message, context, config, dependencies) {
     const { logger } = dependencies;
-    try {
-        if (!message.data) {
-            logger.log('WARN', '[Module Dispatcher] Received message without data.');
-            return;
-        }
+    try { if (!message.data) { logger.log('WARN', '[Module Dispatcher] Received message without data.'); return; }
         const decodedMessage = JSON.parse(Buffer.from(message.data, 'base64').toString());
         const { tasks } = decodedMessage;
-
-        if (!tasks || !Array.isArray(tasks) || tasks.length === 0) {
-            logger.log('WARN', '[Module Dispatcher] Received message with no valid tasks. Nothing to do.');
-            return;
-        }
-
-        if (!config || !config.topicName || !config.batchSize || !config.batchDelayMs) {
-            logger.log('ERROR', '[Module Dispatcher] Invalid configuration provided.', { config });
-            throw new Error("Dispatcher module received invalid configuration.");
-        }
-
-        // --- 3. Use the directly required function ---
-        // Call the sub-pipe -> becomes dispatchTasksInBatches
-        await dispatchTasksInBatches(tasks, dependencies, config); // <<< USE DIRECTLY
-
-    } catch (error) {
-        logger.log('ERROR', '[Module Dispatcher] FATAL error processing message', { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    }
+        if (!tasks || !Array.isArray(tasks) || tasks.length === 0) { logger.log('WARN', '[Module Dispatcher] Received message with no valid tasks. Nothing to do.'); return; }
+        if (!config || !config.topicName || !config.batchSize || !config.batchDelayMs) { logger.log('ERROR', '[Module Dispatcher] Invalid configuration provided.', { config }); throw new Error("Dispatcher module received invalid configuration.");}
+        await dispatchTasksInBatches(tasks, dependencies, config); 
+    } catch (error) { logger.log('ERROR', '[Module Dispatcher] FATAL error processing message', { errorMessage: error.message, errorStack: error.stack }); throw error; }
 }
 
-module.exports = {
-    handleRequest,
-};
+module.exports = { handleRequest };