bulltrackers-module 1.0.762 → 1.0.764

package/functions/computation-system-v2/framework/execution/RemoteTaskRunner.js

@@ -13,6 +13,7 @@
  * - Fault isolation - one entity failure doesn't affect others
  * - Cost efficient - workers scale to zero between runs
  * - RESILIENCE: Implements Circuit Breaker to prevent Retry Cost Spirals [Fix #2]
+ * * UPDATE: Implemented Hybrid Batching (Chunks of 50) to reduce invocations [Optimization].
  */
 
 const { Storage } = require('@google-cloud/storage');
@@ -30,13 +31,17 @@ class RemoteTaskRunner {
         const poolConfig = config.workerPool || {};
         this.bucketName = poolConfig.tempBucket || 'bulltrackers-worker-staging';
         this.workerUrl = poolConfig.workerUrl;
-        this.concurrency = poolConfig.concurrency || 100;
-        this.timeout = poolConfig.timeout || 60000; // 60s default
+        
+        // Optimization: Batch Size for Hybrid Execution
+        this.batchSize = poolConfig.workerBatchSize || 50; 
+        
+        this.concurrency = poolConfig.concurrency || 20; // Lower concurrency since each task does more work
+        this.timeout = poolConfig.timeout || 120000; // Increased timeout for batches (2m)
         this.retries = poolConfig.retries || 2;
         
         // Circuit Breaker Config [Fix #2]
         this.cbConfig = {
-            minInvocations: 20,       // Minimum calls before checking rate
+            minInvocations: 10,       // Minimum calls before checking rate
             failureThreshold: 0.30,   // Trip if failure rate > 30%
             ...poolConfig.circuitBreaker
         };
@@ -58,17 +63,10 @@ class RemoteTaskRunner {
 
     /**
      * Execute a batch of entities remotely (or locally for testing)
-     * * @param {Object} entry - Manifest entry for the computation
-     * @param {string} dateStr - Target date (YYYY-MM-DD)
-     * @param {Object} baseContext - Shared context (references, config)
-     * @param {string[]} entityIds - Entity IDs to process
-     * @param {Map<string, Object>} entityDataMap - Pre-filtered data per entity
-     * @param {Object} depResults - Pre-loaded dependency results
-     * @returns {Promise<{results: Object, errors: Array}>}
      */
     async runBatch(entry, dateStr, baseContext, entityIds, entityDataMap, depResults) {
         const startTime = Date.now();
-        this._log('INFO', `Starting batch: ${entityIds.length} entities for ${entry.name}`);
+        this._log('INFO', `Starting run: ${entityIds.length} entities for ${entry.name}`);
         
         if (this.localMode) {
             return this._runBatchLocal(entry, dateStr, baseContext, entityIds, entityDataMap, depResults);
@@ -79,32 +77,38 @@ class RemoteTaskRunner {
 
     /**
      * Local execution mode - runs workers in-process
-     * Perfect for testing without GCS or network overhead
      */
     async _runBatchLocal(entry, dateStr, baseContext, entityIds, entityDataMap, depResults) {
+        // For local execution, we still process one by one to keep debugging simple,
+        // or we could simulate batching. Sticking to simple pLimit for now.
         const limit = pLimit(this.concurrency);
         const results = {};
         const errors = [];
 
         const tasks = entityIds.map(entityId => limit(async () => {
             try {
-                const contextPackage = this._buildContextPackage(
+                // Simulate the "Batch" structure but for single local execution
+                const contextPackage = this._buildBatchContextPackage(
                     entry, 
-                    entityId, 
-                    entityDataMap.get(entityId),
+                    [entityId], 
+                    entityDataMap,
                     baseContext,
                     depResults
                 );
 
-                const { result } = await executeLocal({
+                // We use the same context structure, but pass it to local handler
+                const response = await executeLocal({
                     computationName: entry.originalName || entry.name,
-                    entityId,
+                    entityIds: [entityId],
                     date: dateStr,
                     contextPackage
                 });
 
-                if (result !== null && result !== undefined) {
-                    results[entityId] = result;
+                if (response.batchResults && response.batchResults[entityId]) {
+                    results[entityId] = response.batchResults[entityId];
+                }
+                if (response.batchErrors && response.batchErrors.length > 0) {
+                    errors.push(...response.batchErrors);
                 }
 
             } catch (e) {
@@ -115,87 +119,97 @@ class RemoteTaskRunner {
 
         await Promise.all(tasks);
 
-        this._log('INFO', `Local batch complete: ${Object.keys(results).length} results, ${errors.length} errors`);
+        this._log('INFO', `Local run complete: ${Object.keys(results).length} results, ${errors.length} errors`);
         return { results, errors };
     }
 
     /**
      * Remote execution mode - invokes Cloud Functions via HTTP
+     * NOW USES CHUNKING (Hybrid Batching)
      */
     async _runBatchRemote(entry, dateStr, baseContext, entityIds, entityDataMap, depResults) {
-        const uploadLimit = pLimit(50); // Concurrent uploads to GCS
+        // 1. Chunk the entities
+        const chunks = this._chunk(entityIds, this.batchSize);
+        this._log('INFO', `Split ${entityIds.length} entities into ${chunks.length} worker batches (size: ${this.batchSize})`);
+
+        const uploadLimit = pLimit(20); // Concurrent uploads
         const invokeLimit = pLimit(this.concurrency); // Concurrent worker invocations
         
-        const results = {};
-        const errors = [];
-        const uploadedPaths = [];
+        const finalResults = {};
+        const finalErrors = [];
+        const uploadedBatches = [];
 
-        // Circuit Breaker Stats (scoped to this batch)
-        const batchStats = {
-            invocations: 0,
-            failures: 0,
-            tripped: false
-        };
+        // Circuit Breaker Stats
+        const batchStats = { invocations: 0, failures: 0, tripped: false };
 
-        // Phase 1: Upload context packages to GCS
-        this._log('INFO', 'Uploading context packages to GCS...');
+        // Phase 1: Upload context packages (One per chunk)
+        this._log('INFO', 'Uploading batch contexts to GCS...');
         const uploadStart = Date.now();
         
-        const uploadTasks = entityIds.map(entityId => uploadLimit(async () => {
-            // Check tripped status early to save uploads if massive failure occurring
+        const uploadTasks = chunks.map((chunkIds, index) => uploadLimit(async () => {
             if (batchStats.tripped) return;
 
-            const contextPackage = this._buildContextPackage(
+            const contextPackage = this._buildBatchContextPackage(
                 entry,
-                entityId,
-                entityDataMap.get(entityId),
+                chunkIds,
+                entityDataMap,
                 baseContext,
                 depResults
             );
 
-            const path = `${dateStr}/${entry.name}/${entityId}.json`;
+            // Use the first entity ID + index to make unique but identifiable path
+            const batchId = `${chunkIds[0]}_batch_${index}`;
+            const path = `${dateStr}/${entry.name}/batches/${batchId}.json`;
             
             try {
                 await this._uploadToGCS(path, contextPackage);
-                uploadedPaths.push({ entityId, path });
+                uploadedBatches.push({ chunkIds, path, batchId });
             } catch (e) {
-                errors.push({ entityId, error: `Upload failed: ${e.message}` });
+                // If upload fails, all entities in chunk fail
+                chunkIds.forEach(id => finalErrors.push({ entityId: id, error: `Upload failed: ${e.message}` }));
             }
         }));
 
         await Promise.all(uploadTasks);
-        this._log('INFO', `Uploaded ${uploadedPaths.length} packages in ${Date.now() - uploadStart}ms`);
+        this._log('INFO', `Uploaded ${uploadedBatches.length} batches in ${Date.now() - uploadStart}ms`);
 
-        // Phase 2: Invoke workers in parallel
+        // Phase 2: Invoke workers
         this._log('INFO', 'Invoking workers...');
         const invokeStart = Date.now();
 
-        const invokeTasks = uploadedPaths.map(({ entityId, path }) => 
+        const invokeTasks = uploadedBatches.map(({ chunkIds, path, batchId }) => 
             invokeLimit(async () => {
-                // FAIL FAST: If circuit tripped, do not invoke worker
                 if (batchStats.tripped) {
-                    errors.push({ entityId, error: 'Skipped: Circuit Breaker Tripped due to high failure rate' });
+                    chunkIds.forEach(id => finalErrors.push({ entityId: id, error: 'Circuit Breaker Tripped' }));
                     return;
                 }
 
                 try {
                     const response = await this._invokeWorkerWithRetry({
                         computationName: entry.originalName || entry.name,
-                        entityId,
+                        entityIds: chunkIds, // Pass the LIST of IDs
                         date: dateStr,
                         dataUri: { bucket: this.bucketName, path }
-                    }, 1, batchStats); // Pass stats object to retry logic
-
-                    if (response.status === 'success' && response.result !== null) {
-                        results[entityId] = response.result;
-                    } else if (response.status === 'error') {
-                        errors.push({ entityId, error: response.error });
+                    }, 1, batchStats);
+
+                    // Process Bulk Response
+                    if (response.status === 'success' || response.status === 'partial') {
+                        // Merge Results
+                        if (response.batchResults) {
+                            Object.assign(finalResults, response.batchResults);
+                        }
+                        // Merge Errors
+                        if (response.batchErrors && Array.isArray(response.batchErrors)) {
+                            finalErrors.push(...response.batchErrors);
+                        }
+                    } else {
+                        throw new Error(`Worker returned status: ${response.status}`);
                     }
-                    // status === 'success' with result === null means skipped (filtered out)
                     
                 } catch (e) {
-                    // Circuit Breaker errors are thrown here
-                    errors.push({ entityId, error: e.message });
+                    // Entire batch failed (network error, timeout, 500)
+                    this._log('WARN', `Batch ${batchId} failed completely: ${e.message}`);
+                    chunkIds.forEach(id => finalErrors.push({ entityId: id, error: `Batch failure: ${e.message}` }));
                 }
             })
         );
@@ -203,40 +217,49 @@ class RemoteTaskRunner {
         await Promise.all(invokeTasks);
         
         if (batchStats.tripped) {
-            this._log('ERROR', `Batch ABORTED by Circuit Breaker. Stats: ${batchStats.failures} failures / ${batchStats.invocations} invocations.`);
+            this._log('ERROR', `Run ABORTED by Circuit Breaker.`);
         }
         
         this._log('INFO', `Invocations complete in ${Date.now() - invokeStart}ms`);
 
-        // Phase 3: Cleanup GCS (fire and forget)
-        this._cleanupGCS(uploadedPaths.map(p => p.path)).catch(e => {
-            this._log('WARN', `GCS cleanup failed: ${e.message}`);
-        });
+        // Phase 3: Cleanup
+        this._cleanupGCS(uploadedBatches.map(b => b.path)).catch(() => {});
 
-        return { results, errors };
+        return { results: finalResults, errors: finalErrors };
     }
 
     /**
-     * Build the context package for a single entity
+     * Build a combined context package for multiple entities
      */
-    _buildContextPackage(entry, entityId, entityData, baseContext, depResults) {
-        // Extract only this entity's dependencies
-        const entityDeps = {};
+    _buildBatchContextPackage(entry, chunkIds, entityDataMap, baseContext, depResults) {
+        // 1. Extract Data for chunk members
+        const batchDataMap = {};
+        chunkIds.forEach(id => {
+            if (entityDataMap.has(id)) {
+                batchDataMap[id] = entityDataMap.get(id);
+            }
+        });
+
+        // 2. Extract Dependencies for chunk members
+        const batchDeps = {};
         for (const [depName, allResults] of Object.entries(depResults || {})) {
-            if (allResults === null) continue; // Large dependency not preloaded
+            if (allResults === null) continue;
             
             if (typeof allResults === 'object') {
-                // If it's a map of entity -> result, extract this entity's
-                if (allResults[entityId]) {
-                    entityDeps[depName] = { [entityId]: allResults[entityId] };
-                }
+                batchDeps[depName] = {};
+                // Only include dependency data for entities in this chunk
+                chunkIds.forEach(id => {
+                    if (allResults[id] !== undefined) {
+                        batchDeps[depName][id] = allResults[id];
+                    }
+                });
             }
         }
 
         return {
-            entityData: entityData || {},
+            entityDataMap: batchDataMap, // Changed from single entityData
             references: baseContext.references || {},
-            dependencies: entityDeps,
+            dependencies: batchDeps,
             computationMeta: {
                 name: entry.name,
                 originalName: entry.originalName,
@@ -248,131 +271,88 @@ class RemoteTaskRunner {
     }
 
     /**
-     * Upload a context package to GCS
+     * Helper to split array into chunks
      */
+    _chunk(array, size) {
+        const chunks = [];
+        for (let i = 0; i < array.length; i += size) {
+            chunks.push(array.slice(i, i + size));
+        }
+        return chunks;
+    }
+
+    // ... [Rest of methods: _uploadToGCS, _invokeWorkerWithRetry, _checkCircuitBreaker, _invokeWorker, _isRetryableError, _cleanupGCS, _log] ...
+    // These remain unchanged from previous version, just ensure they are included in the class.
+    
     async _uploadToGCS(path, data) {
         const file = this.storage.bucket(this.bucketName).file(path);
-        
         await file.save(JSON.stringify(data), {
             contentType: 'application/json',
-            resumable: false, // Faster for small files
-            metadata: {
-                cacheControl: 'no-cache' // Don't cache temp files
-            }
+            resumable: false,
+            metadata: { cacheControl: 'no-cache' }
         });
     }
 
-    /**
-     * Invoke a worker with retry logic and Circuit Breaker
-     */
     async _invokeWorkerWithRetry(payload, attempt = 1, stats = null) {
-        // Track Invocation (Cost)
         if (stats) stats.invocations++;
-
         try {
             return await this._invokeWorker(payload);
         } catch (e) {
-            // Track Failure
             if (stats) {
                 stats.failures++;
                 this._checkCircuitBreaker(stats);
             }
-
-            const isRetryable = this._isRetryableError(e);
-            
-            if (isRetryable && attempt < this.retries) {
-                // Exponential backoff
+            if (this._isRetryableError(e) && attempt < this.retries) {
                 const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000);
                 await new Promise(r => setTimeout(r, delay));
-                
-                // Re-check circuit before retrying (another thread might have tripped it)
                 if (stats) this._checkCircuitBreaker(stats);
-
-                this._log('DEBUG', `Retrying ${payload.entityId} (attempt ${attempt + 1})`);
                 return this._invokeWorkerWithRetry(payload, attempt + 1, stats);
             }
-            
             throw e;
         }
     }
 
-    /**
-     * Check circuit breaker status and throw if tripped
-     */
     _checkCircuitBreaker(stats) {
-        if (stats.tripped) {
-            throw new Error('Circuit Breaker: Batch aborted due to high failure rate');
-        }
-
-        // Only check after minimum invocations (warmup)
+        if (stats.tripped) throw new Error('Circuit Breaker: Run aborted');
         if (stats.invocations >= this.cbConfig.minInvocations) {
             const failureRate = stats.failures / stats.invocations;
-            
             if (failureRate > this.cbConfig.failureThreshold) {
                 stats.tripped = true;
-                const msg = `🚨 CIRCUIT BREAKER TRIPPED! Failure rate ${(failureRate * 100).toFixed(1)}% ` +
-                            `(${stats.failures}/${stats.invocations}) exceeds threshold of ${(this.cbConfig.failureThreshold * 100)}%`;
-                
-                this._log('ERROR', msg);
-                throw new Error(msg);
+                this._log('ERROR', `🚨 CIRCUIT BREAKER TRIPPED! Rate: ${(failureRate * 100).toFixed(1)}%`);
+                throw new Error('Circuit Breaker Tripped');
             }
         }
     }
 
-    /**
-     * Invoke a single worker via HTTP
-     */
     async _invokeWorker(payload) {
-        // Lazy-load auth client
         if (!this._authClient) {
             const { GoogleAuth } = require('google-auth-library');
             const auth = new GoogleAuth();
             this._authClient = await auth.getIdTokenClient(this.workerUrl);
         }
-
         const response = await this._authClient.request({
             url: this.workerUrl,
             method: 'POST',
             data: payload,
             timeout: this.timeout,
-            headers: {
-                'Content-Type': 'application/json'
-            }
+            headers: { 'Content-Type': 'application/json' }
         });
-
         return response.data;
     }
 
-    /**
-     * Check if an error is retryable
-     */
     _isRetryableError(error) {
-        // Network errors
         if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT') return true;
-        
-        // HTTP 5xx errors (server errors)
         if (error.response && error.response.status >= 500) return true;
-        
-        // Rate limiting
         if (error.response && error.response.status === 429) return true;
-        
         return false;
     }
 
-    /**
-     * Cleanup uploaded files from GCS
-     */
     async _cleanupGCS(paths) {
-        // Batch delete
         const bucket = this.storage.bucket(this.bucketName);
-        
-        // GCS batch delete has limits, process in chunks
         const chunkSize = 100;
         for (let i = 0; i < paths.length; i += chunkSize) {
             const chunk = paths.slice(i, i + chunkSize);
-            await Promise.all(chunk.map(path => 
-                bucket.file(path).delete().catch(() => {})
-            ));
+            await Promise.all(chunk.map(path => bucket.file(path).delete().catch(() => {})));
         }
     }
 

package/functions/computation-system-v2/handlers/scheduler.js

@@ -247,7 +247,10 @@ async function dispatchPlannedTasks(tasks) {
                     url: dispatcherUrl,
                     headers: { 'Content-Type': 'application/json' },
                     body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { serviceAccountEmail }
+                    oidcToken: { 
+                        serviceAccountEmail,
+                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
+                    }
                 },
                 scheduleTime: { seconds: t.runAtSeconds },
                 name: taskName
@@ -291,7 +294,10 @@ async function dispatchRecoveryTasks(tasks) {
                     url: dispatcherUrl,
                     headers: { 'Content-Type': 'application/json' },
                     body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { serviceAccountEmail }
+                    oidcToken: { 
+                        serviceAccountEmail,
+                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
+                    }
                 },
                 // Run Immediately (no scheduleTime)
                 name: taskName