bulltrackers-module 1.0.306 → 1.0.307

package/functions/computation-system/helpers/computation_worker.js

@@ -1,314 +1,102 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
- * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Added "Contention-Aware Retry" for the Batch Counter to fix ABORTED errors.
+ * UPDATED: Removed redundant Callback and Sentinel logic.
+ * The system now relies on Dispatcher cursor satiation.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
 const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
-const https                   = require('https'); 
-const { GoogleAuth }          = require('google-auth-library'); 
 const { normalizeName }       = require('../utils/utils');
+const os                      = require('os');
 
 let calculationPackage;
-try { calculationPackage = require('aiden-shared-calculations-unified');
-} catch (e) {console.error("FATAL: Could not load 'aiden-shared-calculations-unified'."); throw e; }
+try { calculationPackage = require('aiden-shared-calculations-unified'); } catch (e) { throw e; }
 const calculations = calculationPackage.calculations;
 
 const MAX_RETRIES = 3;
 
-/**
- * [UPDATED] Heartbeat now returns a closure to get the PEAK memory.
- * This acts as a "Black Box Recorder".
- */
+/** Black Box Recorder for Peak Memory. */
 function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
     let peakRss = 0;
-    
-    const getMemStats = () => {
-        const mem = process.memoryUsage();
-        const rssMB = Math.round(mem.rss / 1024 / 1024);
-        if (rssMB > peakRss) peakRss = rssMB;
-        
-        return {
-            rssMB: rssMB,
-            heapUsedMB: Math.round(mem.heapUsed / 1024 / 1024),
-            timestamp: new Date()
-        };
-    };
-
     const timer = setInterval(async () => {
-        try {
-            const stats = getMemStats();
-            // Use update() to minimize payload size and avoid overwriting status
-            await db.doc(ledgerPath).update({
-                'telemetry.lastMemory': stats,
-                'telemetry.lastHeartbeat': new Date()
-            }).catch(() => {}); // Ignore write errors to prevent crashing the worker
-        } catch (e) {
-            // Silently fail on telemetry errors
-        }
+        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
+        if (rssMB > peakRss) peakRss = rssMB;
+        await db.doc(ledgerPath).update({ 'telemetry.lastMemoryMB': rssMB, 'telemetry.lastHeartbeat': new Date() }).catch(() => {});
     }, intervalMs);
-
-    // Unref so this timer doesn't prevent the process from exiting naturally
     timer.unref();
-
-    return {
-        timer,
-        getPeak: () => peakRss
-    };
-}
-
-/**
- * Helper: Fires the webhook back to Google Cloud Workflows.
- */
-async function triggerWorkflowCallback(url, status, logger) {
-    if (!url) return;
-    logger.log('INFO', `[Worker] 🔔 BATCH COMPLETE! Triggering Workflow Callback: ${status}`);
-    
-    try {
-        const auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
-        const client = await auth.getClient();
-        const accessToken = await client.getAccessToken();
-        const token = accessToken.token;
-
-        return new Promise((resolve, reject) => {
-            const body = JSON.stringify({ status: status, timestamp: new Date().toISOString() });
-            
-            const req = https.request(url, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Content-Length': Buffer.byteLength(body),
-                    'Authorization': `Bearer ${token}`
-                }
-            }, (res) => {
-                if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } 
-                else { logger.log('WARN', `Callback responded with ${res.statusCode}`); resolve(); }
-            });
-
-            req.on('error', (e) => { logger.log('ERROR', `Failed to trigger callback: ${e.message}`); resolve(); });
-            req.write(body);
-            req.end();
-        });
-    } catch (e) {
-        logger.log('ERROR', `Failed to generate auth token for callback: ${e.message}`);
-    }
-}
-
-/**
- * [UPDATED] Helper: Decrements 'remainingTasks' in Firestore.
- * NOW INCLUDES CONTENTION RETRY LOGIC (The "Sentinel" Fix)
- */
-async function decrementAndCheck(db, metaStatePath, logger) {
-    if (!metaStatePath) return null;
-    
-    const MAX_CONTENTION_RETRIES = 10;
-    let attempt = 0;
-
-    while (attempt < MAX_CONTENTION_RETRIES) {
-        try {
-            const result = await db.runTransaction(async (t) => {
-                const ref = db.doc(metaStatePath);
-                const doc = await t.get(ref);
-                if (!doc.exists) return null;
-                
-                const data = doc.data();
-                // Safety: Don't decrement below zero
-                const currentRemaining = data.remainingTasks || 0;
-                if (currentRemaining <= 0) return { remaining: 0, callbackUrl: data.callbackUrl };
-
-                const newRemaining = currentRemaining - 1;
-                t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
-                
-                return { remaining: newRemaining, callbackUrl: data.callbackUrl };
-            });
-
-            // Success! Check if we are the "Sentinel" (the last one)
-            if (result && result.remaining <= 0) return result.callbackUrl;
-            return null; // We decremented successfully, but weren't the last one.
-
-        } catch (e) {
-            // Check if it's a contention error (ABORTED/10 or DEADLINE_EXCEEDED/4)
-            const isContention = e.code === 10 || e.code === 4 || (e.message && e.message.includes('contention'));
-            
-            if (isContention) {
-                attempt++;
-                // JITTER: Random delay between 50ms and 500ms to desynchronize the herd
-                const delay = Math.floor(Math.random() * 450) + 50;
-                logger.log('WARN', `[Worker] Batch counter contention (Attempt ${attempt}/${MAX_CONTENTION_RETRIES}). Retrying in ${delay}ms...`);
-                await new Promise(r => setTimeout(r, delay));
-            } else {
-                // Fatal error (permission, etc)
-                logger.log('ERROR', `[Worker] Fatal error decrementing batch counter: ${e.message}`);
-                return null;
-            }
-        }
-    }
-    
-    logger.log('ERROR', `[Worker] Failed to decrement batch counter after ${MAX_CONTENTION_RETRIES} attempts. The count will be inaccurate.`);
-    return null;
+    return { timer, getPeak: () => peakRss };
 }
 
 async function handleComputationTask(message, config, dependencies) {
-    const systemLogger = new StructuredLogger({ minLevel: config.minLevel || 'INFO', enableStructured: true, ...config });
-    const runDependencies = { ...dependencies, logger: systemLogger };
-    const { logger, db }  = runDependencies;
+    const logger = new StructuredLogger({ minLevel: config.minLevel || 'INFO', enableStructured: true, ...config });
+    const runDeps = { ...dependencies, logger };
+    const db = dependencies.db;
     let data;
 
     try {
-        if (message.data && message.data.message && message.data.message.data) { data = JSON.parse(Buffer.from(message.data.message.data, 'base64').toString()); 
-        } else if (message.data && typeof message.data === 'string')           { data = JSON.parse(Buffer.from(message.data, 'base64').toString()); 
-        } else if (message.json) { data = message.json; 
-        } else { data = message; }
-    } catch (parseError) { logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message }); return; }
+        const raw = message.data?.message?.data || message.data || message.json;
+        data = (typeof raw === 'string') ? JSON.parse(Buffer.from(raw, 'base64').toString()) : raw;
+    } catch (e) { return; }
 
-    if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
+    if (!data || data.action !== 'RUN_COMPUTATION_DATE') return;
     
-    // [UPDATED] Extract 'resources' from payload (set by Dispatcher)
-    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, metaStatePath, resources } = data;
-    const resourceTier = resources || 'standard'; // Default to standard
-    
-    if (!date || !pass || !computation) { logger.log('ERROR', `[Worker] Invalid payload.`, data); return; }
-
-    const retryCount = message.deliveryAttempt || 1;
+    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, resources } = data;
+    const resourceTier = resources || 'standard';
     const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computation}`;
 
-    // --- POISON MESSAGE HANDLING (DLQ) ---
-    if (retryCount > MAX_RETRIES) {
-            logger.log('ERROR', `[Worker] ☠️ Task POISONED. Moved to DLQ: ${computation}`);
-            try {
-                await db.collection('computation_dead_letter_queue').add({
-                    originalData: data,
-                    dispatchId: dispatchId,
-                    error: { message: 'Max Retries Exceeded', stack: 'PubSub delivery limit reached' },
-                    finalAttemptAt: new Date(),
-                    failureReason: 'MAX_RETRIES_EXCEEDED'
-                });
-
-                await db.doc(ledgerPath).set({
-                    status: 'FAILED',
-                    error: 'Max Retries Exceeded (Poison Message)',
-                    failedAt: new Date()
-                }, { merge: true });
+    logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}]`);
 
-                const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
-                return;
-            } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
-    }
+    // 1. Audit Lease
+    await db.doc(ledgerPath).set({
+        status: 'IN_PROGRESS',
+        workerId: process.env.K_REVISION || os.hostname(),
+        startedAt: new Date(),
+        dispatchId
+    }, { merge: true });
 
-    logger.log('INFO', `[Worker] 📥 Received Task: ${computation} (${date}) [Attempt ${retryCount}/${MAX_RETRIES}] [Tier: ${resourceTier}]`);
-
-    // 1. Update Status to IN_PROGRESS & Initialize Telemetry
-    try {
-        await db.doc(ledgerPath).set({
-            status: 'IN_PROGRESS',
-            workerId: process.env.K_REVISION || 'unknown',
-            startedAt: new Date(),
-            dispatchId: dispatchId,
-            telemetry: { startTime: new Date(), lastMemory: null } // Init for heartbeat
-        }, { merge: true });
-    } catch (leaseErr) {}
-
-    // 2. START HEARTBEAT (The Flight Recorder)
-    // [UPDATED] Using new logic to track peak
-    const heartbeatControl = startMemoryHeartbeat(db, ledgerPath, 2000);
-
-    let computationManifest;
-    try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
-    } catch (manifestError) { 
-        clearInterval(heartbeatControl.timer); // Stop if we fail early
-        logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
-        return; 
-    }
+    const heartbeat = startMemoryHeartbeat(db, ledgerPath);
 
     try {
+        const manifest = getManifest(config.activeProductLines || [], calculations, runDeps);
         const startTime = Date.now();
-        const result    = await executeDispatchTask(
-            date, pass, computation, config, runDependencies, 
-            computationManifest, previousCategory, dependencyResultHashes
+        
+        const result = await executeDispatchTask(
+            date, pass, computation, config, runDeps, 
+            manifest, previousCategory, dependencyResultHashes
         );
-        const duration = Date.now() - startTime;
-
-        // STOP HEARTBEAT ON SUCCESS
-        clearInterval(heartbeatControl.timer);
 
-        const failureReport  = result?.updates?.failureReport  || [];
+        clearInterval(heartbeat.timer);
+        const failureReport = result?.updates?.failureReport || [];
         const successUpdates = result?.updates?.successUpdates || {};
 
-        if (failureReport.length > 0) {
-            const failReason = failureReport[0]; 
-            throw new Error(failReason.error.message || 'Computation Logic Failed');
-        } 
-        else {
-            if (Object.keys(successUpdates).length > 0) { logger.log('INFO', `[Worker] ✅ Stored: ${computation}`); } 
-            else { logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`); }
-            
-            // Extract the metrics from the success update for the recorder
-            const calcUpdate = successUpdates[normalizeName(computation)] || {};
-            const finalMetrics = {
-                durationMs: duration,
-                peakMemoryMB: heartbeatControl.getPeak(),
-                io: calcUpdate.metrics?.io,
-                storage: calcUpdate.metrics?.storage,
-                execution: calcUpdate.metrics?.execution,
-                validation: calcUpdate.metrics?.validation,
-                composition: calcUpdate.composition
-            };
-
-            await db.doc(ledgerPath).update({
-                status: 'COMPLETED',
-                completedAt: new Date()
-            }).catch(() => {});
+        if (failureReport.length > 0) throw new Error(failureReport[0].error.message);
+
+        const calcUpdate = successUpdates[normalizeName(computation)] || {};
+        const metrics = {
+            durationMs: Date.now() - startTime,
+            peakMemoryMB: heartbeat.getPeak(),
+            io: calcUpdate.metrics?.io,
+            storage: calcUpdate.metrics?.storage,
+            execution: calcUpdate.metrics?.execution,
+            validation: calcUpdate.metrics?.validation,
+            composition: calcUpdate.composition
+        };
 
-            // [UPDATED] Pass resourceTier and metrics to recordRunAttempt
-            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, finalMetrics, triggerReason, resourceTier);
+        await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
+        await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
 
-            const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-            if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
-        }
     } catch (err) { 
-        // STOP HEARTBEAT ON ERROR
-        clearInterval(heartbeatControl.timer);
-
-        // --- ERROR HANDLING ---
-        const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
-                                     err.stage === 'QUALITY_CIRCUIT_BREAKER' ||
-                                     err.stage === 'SEMANTIC_GATE' ||
-                                     (err.message && (err.message.includes('INVALID_ARGUMENT') || err.message.includes('Transaction too big')));
+        clearInterval(heartbeat.timer);
+        const isDeterministic = ['SHARDING_LIMIT_EXCEEDED', 'QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(err.stage);
 
-        if (isDeterministicError) {
-             logger.log('ERROR', `[Worker] 🛑 Permanent Failure: ${err.message}`);
-             try {
-                 await db.collection('computation_dead_letter_queue').add({
-                     originalData: data,
-                     dispatchId: dispatchId,
-                     error: { message: err.message, stack: err.stack, stage: err.stage || 'UNKNOWN' },
-                     finalAttemptAt: new Date(),
-                     failureReason: 'PERMANENT_DETERMINISTIC_ERROR'
-                 });
-                 
-                 await db.doc(ledgerPath).set({
-                    status: 'FAILED',
-                    error: err.message || 'Permanent Deterministic Error',
-                    failedAt: new Date()
-                }, { merge: true });
-
-                 await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0, peakMemoryMB: heartbeatControl.getPeak() }, triggerReason, resourceTier);
-                 
-                 const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                 if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
-                 return; 
-             } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
+        if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
+            await db.doc(ledgerPath).set({ status: 'FAILED', error: err.message, failedAt: new Date() }, { merge: true });
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeat.getPeak() }, triggerReason, resourceTier);
+            return; // Don't throw, we've handled the permanent failure
         }
-
-        if (retryCount >= MAX_RETRIES) { throw err; }
-
-        logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
-        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0, peakMemoryMB: heartbeatControl.getPeak() }, triggerReason, resourceTier);
-        throw err; 
+        throw err; // Trigger Pub/Sub retry
     }
 }