bulltrackers-module 1.0.765 → 1.0.766

package/functions/computation-system-v2/framework/execution/Orchestrator.js

@@ -287,116 +287,126 @@ class Orchestrator {
     async _executeStreaming(entry, dateStr, depResults, previousResult, options) {
         const checkpointer = new Checkpointer(this.config, this.storageManager);
         let cp = null;
-        if (!options.dryRun) {
-            const forceLock = options.force || this.config.bypassLocks || process.env.NODE_ENV === 'test';
-            cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0, entry.hash, forceLock); 
-
-            if (cp && cp.isLocked) throw new Error(`⚠️ Computation ${entry.name} is currently LOCKED.`);
-            if (cp && cp.skipped) {
-                this._log('INFO', `⏭️ Skipping ${entry.name}: ${cp.reason}`);
-                return { count: 0, hash: 'skipped_dead_letter', skipped: true };
-            }
-            if (cp?.isCompleted) return { count: 0, hash: 'cached', skipped: true };
-            if (cp?.isResumed) this._log('INFO', `Resuming ${entry.name} from checkpoint...`);
-        }
-
-        // DECISION: Use remote workers or local execution?
-        const useRemote = this._shouldUseRemoteWorkers(entry, options);
         
-        if (useRemote) {
-            this._log('INFO', `Using REMOTE worker pool for ${entry.name}`);
-            return this._executeStreamingRemote(entry, dateStr, depResults, previousResult, options, checkpointer, cp);
-        }
+        try {
+            if (!options.dryRun) {
+                const forceLock = options.force || this.config.bypassLocks || process.env.NODE_ENV === 'test';
+                cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0, entry.hash, forceLock); 
 
-        // LOCAL EXECUTION PATH (Original Logic)
-        const driverTable = this._getDriverTable(entry.requires);
-        const driverEntityField = this.config.tables[driverTable]?.entityField;
-        const { batchRequires, globalRequires } = this._splitRequirements(entry.requires, driverTable);
+                if (cp && cp.isLocked) throw new Error(`⚠️ Computation ${entry.name} is currently LOCKED.`);
+                if (cp && cp.skipped) {
+                    this._log('INFO', `⏭️ Skipping ${entry.name}: ${cp.reason}`);
+                    return { count: 0, hash: 'skipped_dead_letter', skipped: true };
+                }
+                if (cp?.isCompleted) return { count: 0, hash: 'cached', skipped: true };
+                if (cp?.isResumed) this._log('INFO', `Resuming ${entry.name} from checkpoint...`);
+            }
 
-        let globalData = {};
-        if (Object.keys(globalRequires).length > 0) {
-            globalData = await this.dataFetcher.fetchForComputation(globalRequires, dateStr);
-        }
+            // DECISION: Use remote workers or local execution?
+            const useRemote = this._shouldUseRemoteWorkers(entry, options);
+            
+            if (useRemote) {
+                this._log('INFO', `Using REMOTE worker pool for ${entry.name}`);
+                return await this._executeStreamingRemote(entry, dateStr, depResults, previousResult, options, checkpointer, cp);
+            }
 
-        const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
-        const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
-        
-        const rollingHash = crypto.createHash('sha256');
-        let totalCount = 0;
-        let batchIndex = 0;
-        const concurrency = this.config.execution?.entityConcurrency || DEFAULT_CONCURRENCY;
-        const limit = pLimit(concurrency);
+            // LOCAL EXECUTION PATH (Original Logic)
+            const driverTable = this._getDriverTable(entry.requires);
+            const driverEntityField = this.config.tables[driverTable]?.entityField;
+            const { batchRequires, globalRequires } = this._splitRequirements(entry.requires, driverTable);
 
-        for await (const batch of batchStream) {
-            if (cp && cp.completedBatches && cp.completedBatches.has(batchIndex)) {
-                batchIndex++; continue;
+            let globalData = {};
+            if (Object.keys(globalRequires).length > 0) {
+                globalData = await this.dataFetcher.fetchForComputation(globalRequires, dateStr);
             }
 
-            const { data: batchLocalData, entityIds } = batch;
-            const combinedData = { ...batchLocalData, ...globalData };
+            const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
+            const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
             
-            // STRICT FIX: Prefetch dependencies for the batch.
-            const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
-            
-            const { rules } = this.ruleInjector.createContext(); 
-            const batchResults = {};
-            
-            await Promise.all(entityIds.map(entityId => limit(async () => {
-                const instance = new entry.class();
-                const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
+            const rollingHash = crypto.createHash('sha256');
+            let totalCount = 0;
+            let batchIndex = 0;
+            const concurrency = this.config.execution?.entityConcurrency || DEFAULT_CONCURRENCY;
+            const limit = pLimit(concurrency);
+
+            for await (const batch of batchStream) {
+                if (cp && cp.completedBatches && cp.completedBatches.has(batchIndex)) {
+                    batchIndex++; continue;
+                }
+
+                const { data: batchLocalData, entityIds } = batch;
+                const combinedData = { ...batchLocalData, ...globalData };
+                
+                // STRICT FIX: Prefetch dependencies for the batch.
+                const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
                 
-                const context = {
-                    computation: entry, date: dateStr, entityId, data: entityData,
+                const { rules } = this.ruleInjector.createContext(); 
+                const batchResults = {};
+                
+                await Promise.all(entityIds.map(entityId => limit(async () => {
+                    const instance = new entry.class();
+                    const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
                     
-                    // STRICT FIX: No fallback to _lazyLoadDependency.
-                    getDependency: (depName, targetId) => {
-                        const id = targetId || entityId;
-                        
-                        // 1. Look in Batch-Prefetched Dependencies (Priority)
-                        if (batchDeps[depName] && batchDeps[depName].has(id)) {
-                            return batchDeps[depName].get(id);
-                        }
+                    const context = {
+                        computation: entry, date: dateStr, entityId, data: entityData,
                         
-                        // 2. Look in Global/Preloaded Dependencies
-                        if (depResults[depName]) {
-                             if (depResults[depName][id] !== undefined) return depResults[depName][id];
-                        }
+                        // STRICT FIX: No fallback to _lazyLoadDependency.
+                        getDependency: (depName, targetId) => {
+                            const id = targetId || entityId;
+                            
+                            // 1. Look in Batch-Prefetched Dependencies (Priority)
+                            if (batchDeps[depName] && batchDeps[depName].has(id)) {
+                                return batchDeps[depName].get(id);
+                            }
+                            
+                            // 2. Look in Global/Preloaded Dependencies
+                            if (depResults[depName]) {
+                                if (depResults[depName][id] !== undefined) return depResults[depName][id];
+                            }
+                            
+                            // 3. STRICT MODE: Throw Error
+                            throw new Error(
+                                `[Strict Dependency] Dependency '${depName}' (ID: ${id}) not found in batch context. ` +
+                                `Ensure '${depName}' is listed in ${entry.name}.getConfig().dependencies.`
+                            );
+                        },
                         
-                        // 3. STRICT MODE: Throw Error
-                        throw new Error(
-                            `[Strict Dependency] Dependency '${depName}' (ID: ${id}) not found in batch context. ` +
-                            `Ensure '${depName}' is listed in ${entry.name}.getConfig().dependencies.`
-                        );
-                    },
-                    
-                    previousResult, rules, references: this.referenceDataCache,
-                    config: this.config, dataFetcher: this.dataFetcher
-                };
+                        previousResult, rules, references: this.referenceDataCache,
+                        config: this.config, dataFetcher: this.dataFetcher
+                    };
+
+                    const result = await this.runner.run(instance, context);
+                    if (result !== undefined) {
+                        batchResults[entityId] = result;
+                        this._updateRollingHash(rollingHash, result);
+                    }
+                })));
 
-                const result = await this.runner.run(instance, context);
-                if (result !== undefined) {
-                    batchResults[entityId] = result;
-                    this._updateRollingHash(rollingHash, result);
+                if (!options.dryRun) {
+                    await this.storageManager.commitResults(dateStr, entry, batchResults, {}); 
+                    const lastId = entityIds[entityIds.length - 1];
+                    await checkpointer.markBatchComplete(dateStr, entry.name, cp?.id, batchIndex, batchSize, lastId);
                 }
-            })));
 
-            if (!options.dryRun) {
-                await this.storageManager.commitResults(dateStr, entry, batchResults, {}); 
-                const lastId = entityIds[entityIds.length - 1];
-                await checkpointer.markBatchComplete(dateStr, entry.name, cp?.id, batchIndex, batchSize, lastId);
+                totalCount += Object.keys(batchResults).length;
+                batchIndex++;
             }
 
-            totalCount += Object.keys(batchResults).length;
-            batchIndex++;
-        }
+            if (!options.dryRun) {
+                await this.storageManager.finalizeResults(dateStr, entry);
+                if (cp) await checkpointer.complete(dateStr, entry.name, cp.id);
+            }
 
-        if (!options.dryRun) {
-            await this.storageManager.finalizeResults(dateStr, entry);
-            if (cp) await checkpointer.complete(dateStr, entry.name, cp.id);
+            // FIX: Return valid object shape including skipped: false
+            return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16), skipped: false };
+            
+        } catch (error) {
+            if (cp && cp.id) {
+                this._log('ERROR', `Streaming failed, marking checkpoint ${cp.id} as failed.`);
+                await this.storageManager.failCheckpoint(cp.id, error.message);
+            }
+            throw error;
         }
-
-        // FIX: Return valid object shape including skipped: false
-        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16), skipped: false };
     }
 
     /**

package/functions/computation-system-v2/framework/storage/StorageManager.js

@@ -6,6 +6,7 @@
  * - finalizeResults: Loads all GCS files to a temp table and performs a single MERGE
  * * V2.2 UPDATE: Added saveCheckpoint for Append-Only Checkpointer support.
  * * V2.3 UPDATE: Parallelized GCS and Firestore writes in commitResults.
+ * * V2.4 UPDATE: Added Attempt Tracking for Zombie Detection.
  * * FIX: Switched to bigquery.createJob for GCS imports to prevent local file path interpretation errors.
  * * FIX: Improved error logging to catch swallowed BigQuery insert errors.
  * * FIX: finalizeResults now checks for file existence to prevent "Not found" errors on empty results.
@@ -351,8 +352,9 @@ class StorageManager {
             // FIX: Use subquery with ROW_NUMBER to find the TRUE latest state per computation.
             // We only count it as a zombie if the LATEST row is 'running'.
             // This ignores 'running' rows that have a newer (or same-time) 'completed' sibling.
+            // UPDATE: Added attempts to the selection
             const query = `
-                SELECT computation_name, date, checkpoint_id, last_updated
+                SELECT computation_name, date, checkpoint_id, last_updated, attempts
                 FROM (
                     SELECT 
                         computation_name, 
@@ -360,6 +362,7 @@ class StorageManager {
                         checkpoint_id, 
                         last_updated, 
                         status,
+                        attempts,
                         ROW_NUMBER() OVER (
                             PARTITION BY computation_name, date 
                             ORDER BY last_updated DESC,
@@ -377,7 +380,12 @@ class StorageManager {
                 LIMIT 50
             `;
             const [rows] = await this.bigquery.query({ query, params: { minutes: minutesThreshold }, location: this.config.bigquery.location });
-            return rows.map(r => ({ name: r.computation_name, date: r.date.value || r.date, checkpointId: r.checkpoint_id }));
+            return rows.map(r => ({ 
+                name: r.computation_name, 
+                date: r.date.value || r.date, 
+                checkpointId: r.checkpoint_id,
+                attempts: r.attempts 
+            }));
         } catch (e) { 
             console.error(`[Storage] findZombies failed: ${e.message}`);
             return []; 

package/functions/computation-system-v2/handlers/scheduler.js

@@ -121,17 +121,27 @@ async function runWatchdog(req, res) {
         // 1. Find Zombies
         const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
         
-        if (zombies.length === 0) {
-            return res.status(200).send('No zombies detected.');
+        // Filter out excessive attempts
+        const actionableZombies = [];
+        for (const z of zombies) {
+            if ((z.attempts || 0) >= 3) {
+                console.warn(`[Watchdog] Ignoring zombie ${z.name} (Checkpoint: ${z.checkpointId}) - Max attempts reached (${z.attempts})`);
+                continue;
+            }
+            actionableZombies.push(z);
+        }
+
+        if (actionableZombies.length === 0) {
+            return res.status(200).send('No recoverable zombies.');
         }
 
-        console.log(`[Watchdog] 🧟 Found ${zombies.length} zombies. Initiating recovery...`);
+        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Initiating recovery...`);
 
         // 2. Claim & Recover
         // We claim them first so the next watchdog doesn't grab them while we are dispatching
-        await Promise.all(zombies.map(z => storageManager.claimZombie(z.checkpointId)));
+        await Promise.all(actionableZombies.map(z => storageManager.claimZombie(z.checkpointId)));
 
-        const recoveryTasks = zombies.map(z => {
+        const recoveryTasks = actionableZombies.map(z => {
             const entry = manifest.find(m => m.name === z.name);
             if (!entry) {
                 console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.765",
+  "version": "1.0.766",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [