bulltrackers-module 1.0.642 → 1.0.643

package/functions/computation-system/helpers/computation_worker.js

@@ -22,6 +22,28 @@ try { calculationPackage = require('aiden-shared-calculations-unified'); } catch
 const calculations = calculationPackage.calculations;
 
 const MAX_RETRIES = 3;
+const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; // 15 minutes
+
+// =============================================================================
+// HELPER: Firestore Timestamp Conversion
+// =============================================================================
+/**
+ * Converts a Firestore Timestamp or Date to milliseconds.
+ * Firestore stores Date objects as Timestamp objects, which have a .toDate() method.
+ * This function handles both cases correctly.
+ * @param {any} field - Firestore Timestamp, Date object, or string
+ * @returns {number} Milliseconds since epoch, or 0 if invalid
+ */
+function getMillis(field) {
+    if (!field) return 0;
+    // Handle Firestore Timestamp (has .toDate() method)
+    if (field.toDate && typeof field.toDate === 'function') {
+        return field.toDate().getTime();
+    }
+    // Handle standard Date object or string
+    const date = new Date(field);
+    return isNaN(date.getTime()) ? 0 : date.getTime();
+}
 
 // [NEW] Helper to push metric to Google Cloud
 async function pushMetric(type, value, labels) {
@@ -123,8 +145,20 @@ async function checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerI
                     return { shouldRun: false, reason: 'Duplicate delivery: Task already IN_PROGRESS with same ID.' };
                 }
                 if (data.status === 'IN_PROGRESS' && !onDemand) {
-                    // On-demand can break locks if needed, but regular requests should wait
-                    return { shouldRun: false, reason: 'Collision: Task currently IN_PROGRESS by another worker.' };
+                    // CRITICAL FIX: Check if the lock is stale before blocking
+                    // If the task has been IN_PROGRESS for more than 15 minutes, break the lock
+                    const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
+                    const lockAge = Date.now() - lastActivityTime;
+                    
+                    if (lockAge > STALE_LOCK_THRESHOLD_MS) {
+                        // Stale lock detected - break it and allow this worker to claim it
+                        // The transaction will overwrite the stale lock below
+                        // Note: We can't log here because we're in a transaction, but the caller will log
+                        // Continue to claim lease below
+                    } else {
+                        // Valid active lock - wait for it to complete
+                        return { shouldRun: false, reason: 'Collision: Task currently IN_PROGRESS by another worker.' };
+                    }
                 }
             }
 
@@ -179,10 +213,29 @@ async function handleComputationTask(message, config, dependencies) {
     const db = dependencies.db;
 
     // --- STEP 1: IDEMPOTENCY CHECK (with on-demand override) ---
+    // Check for stale locks before the transaction (for logging purposes)
+    const preCheckSnap = await db.doc(ledgerPath).get();
+    let wasStaleLock = false;
+    if (preCheckSnap.exists) {
+        const preCheckData = preCheckSnap.data();
+        if (preCheckData.status === 'IN_PROGRESS' && preCheckData.workerId !== workerId) {
+            const lastActivityTime = getMillis(preCheckData.telemetry?.lastHeartbeat) || getMillis(preCheckData.startedAt);
+            const lockAge = Date.now() - lastActivityTime;
+            if (lockAge > STALE_LOCK_THRESHOLD_MS) {
+                wasStaleLock = true;
+                logger.log('WARN', `[Worker] 🧟 Breaking stale lock for ${computation}. Previous worker ${preCheckData.workerId} was inactive for ${Math.round(lockAge / 1000 / 60)} minutes.`);
+            }
+        }
+    }
+    
     const gate = await checkIdempotencyAndClaimLease(db, ledgerPath, dispatchId, workerId, onDemand);
     if (!gate.shouldRun) {
         logger.log('WARN', `[Worker] 🛑 Idempotency Gate: Skipping ${computation}. Reason: ${gate.reason}`);
-        return; 
+        return;
+    }
+    
+    if (wasStaleLock) {
+        logger.log('INFO', `[Worker] ✅ Successfully claimed lease for ${computation} after breaking stale lock.`);
     }
     
     if (onDemand) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.642",
+  "version": "1.0.643",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [