bulltrackers-module 1.0.734 → 1.0.736

package/functions/computation-system-v2/framework/monitoring/Profiler.js

@@ -1,79 +1,81 @@
 /**
  * @fileoverview Computation Profiler
  * Tracks execution metrics (duration, memory) for computations.
+ * * UPDATE: Now aggregates metrics in memory to allow single-row reporting.
  */
 
 class ComputationProfiler {
     constructor() {
-        this.profiles = new Map();
+        this.activeProfiles = new Map();
+        this.completedStats = new Map(); // Stores arrays of durations/memory per computation
     }
     
     startProfile(computationName, entityId = null) {
-        const key = entityId ? `${computationName}:${entityId}` : computationName;
+        const id = entityId ? `${computationName}:${entityId}` : `${computationName}:${Date.now()}:${Math.random()}`;
         
-        this.profiles.set(key, {
+        this.activeProfiles.set(id, {
+            name: computationName,
             startTime: Date.now(),
-            startMemory: process.memoryUsage().heapUsed,
-            queryCount: 0,
-            bytesProcessed: 0
+            startMemory: process.memoryUsage().heapUsed
         });
         
-        return key;
+        return id;
     }
     
-    endProfile(key, metadata = {}) {
-        const profile = this.profiles.get(key);
-        if (!profile) return;
+    endProfile(id, metadata = {}) {
+        const profile = this.activeProfiles.get(id);
+        if (!profile) return null;
         
         const endTime = Date.now();
         const endMemory = process.memoryUsage().heapUsed;
-        
+        const duration = endTime - profile.startTime;
+        const memoryDelta = endMemory - profile.startMemory;
+
+        // Store for aggregation
+        if (!this.completedStats.has(profile.name)) {
+            this.completedStats.set(profile.name, { durations: [], memories: [] });
+        }
+        const stats = this.completedStats.get(profile.name);
+        stats.durations.push(duration);
+        stats.memories.push(memoryDelta);
+
+        this.activeProfiles.delete(id);
+
         return {
-            duration: endTime - profile.startTime,
-            memoryDelta: endMemory - profile.startMemory,
-            queriesExecuted: profile.queryCount,
-            bytesProcessed: profile.bytesProcessed,
+            duration,
+            memoryDelta,
             ...metadata
         };
     }
-    
-    async generateReport(dateStr, manifest) {
-        const report = {
-            date: dateStr,
-            totalDuration: 0,
-            totalQueries: 0,
-            totalBytes: 0,
-            computations: []
+
+    /**
+     * Retrieves aggregated statistics for a specific computation and clears them.
+     */
+    getAndClearStats(computationName) {
+        const stats = this.completedStats.get(computationName);
+        if (!stats || stats.durations.length === 0) return null;
+
+        const count = stats.durations.length;
+        const totalDuration = stats.durations.reduce((a, b) => a + b, 0);
+        const avgDuration = totalDuration / count;
+        const avgMemory = stats.memories.reduce((a, b) => a + b, 0) / count;
+
+        // Calculate P95
+        const sorted = [...stats.durations].sort((a, b) => a - b);
+        const p95 = sorted[Math.ceil(count * 0.95) - 1];
+
+        // Clear memory
+        this.completedStats.delete(computationName);
+
+        return {
+            entityCount: count,
+            totalDuration,
+            avgDuration,
+            p95Duration: p95,
+            minDuration: sorted[0],
+            maxDuration: sorted[count - 1],
+            avgMemoryDelta: avgMemory
         };
-        
-        for (const entry of manifest) {
-            const compProfiles = Array.from(this.profiles.entries())
-                .filter(([k]) => k.startsWith(entry.name));
-            
-            if (compProfiles.length === 0) continue;
-            
-            const durations = compProfiles.map(([, p]) => Date.now() - p.startTime);
-            const memories = compProfiles.map(([, p]) => process.memoryUsage().heapUsed - p.startMemory);
-            
-            report.computations.push({
-                name: entry.name,
-                entityCount: compProfiles.length,
-                avgDuration: durations.reduce((a, b) => a + b, 0) / durations.length,
-                maxDuration: Math.max(...durations),
-                p95Duration: this._percentile(durations, 0.95),
-                avgMemory: memories.reduce((a, b) => a + b, 0) / memories.length,
-                totalDuration: durations.reduce((a, b) => a + b, 0)
-            });
-        }
-        
-        report.computations.sort((a, b) => b.totalDuration - a.totalDuration);
-        return report;
-    }
-    
-    _percentile(arr, p) {
-        const sorted = [...arr].sort((a, b) => a - b);
-        const index = Math.ceil(sorted.length * p) - 1;
-        return sorted[index];
     }
 }
 

package/functions/computation-system-v2/framework/resilience/Checkpointer.js

@@ -1,42 +1,145 @@
 /**
- * @fileoverview Computation Checkpointer
- * Manages save/resume states for long-running computations using lightweight batch tracking.
+ * @fileoverview Computation Checkpointer (Append-Only Version)
+ * Manages save/resume states for long-running computations.
+ * Implements "Heartbeat & Steal" using an APPEND-ONLY pattern to avoid BigQuery Streaming Buffer locks.
+ * * UPDATE: Added 'force' parameter to initCheckpoint for local testing bypass.
  */
 
 const crypto = require('crypto');
 
+// Max number of times the Scheduler can revive a Zombie before we give up.
+const MAX_ATTEMPTS = 3;
+
 class Checkpointer {
     constructor(config, storage) {
         this.config = config;
         this.storage = storage;
         this.enabled = config.checkpointing?.enabled !== false;
+        
+        // Identity of this specific process/worker
+        this.workerId = crypto.randomUUID();
+        this.heartbeatInterval = null;
+        
+        // Local cache of the current state (for append-only writes)
+        this.currentState = null;
     }
     
-    async initCheckpoint(dateStr, computationName, totalEntities = 0) {
+    /**
+     * Initialize or Resume a checkpoint.
+     * Handles zombie detection, atomic lock stealing, and version-aware dead-letter logic.
+     * @param {string} dateStr - Target date
+     * @param {string} computationName - Name of the computation
+     * @param {number} totalEntities - Total entities to process (approx)
+     * @param {string} codeHash - Hash of the current running code
+     * @param {boolean} force - If true, bypasses lock checks (steals immediately). Useful for testing.
+     */
+    async initCheckpoint(dateStr, computationName, totalEntities = 0, codeHash, force = false) {
         if (!this.enabled) return null;
 
-        // Check if there is an existing running checkpoint we can resume
+        // 1. Get the LATEST checkpoint row (Snapshot)
+        // StorageManager must order by last_updated DESC limit 1 to handle append-only logs
         const existing = await this.storage.getLatestCheckpoint(dateStr, computationName);
         
-        if (existing && existing.status === 'running') {
-            return {
-                id: existing.checkpoint_id,
-                processedCount: existing.processed_count || 0,
-                completedBatches: new Set(existing.completed_batches || []),
-                lastEntityId: existing.last_entity_id,
-                isResumed: true
-            };
-        } else if (existing && existing.status === 'completed') {
-            // Already fully done
+        // FIX: Force Re-Run (Bypass everything and start fresh)
+        // This ensures even "completed" or "up-to-date" tasks are re-run when testing.
+        if (force) {
+             console.log(`[Checkpointer] ⚡ FORCE mode active. Resetting checkpoint for ${computationName}...`);
+             return this._createNew(dateStr, computationName, totalEntities, codeHash);
+        }
+
+        // LOGIC: Version Check
+        // If code changed, we generally want to invalidate previous partial progress 
+        // unless it was fully completed.
+        if (existing && existing.code_hash && existing.code_hash !== codeHash && existing.status !== 'completed') {
+             console.log(`[Checkpointer] Code change detected (${existing.code_hash} -> ${codeHash}). Resetting checkpoint.`);
+             return this._createNew(dateStr, computationName, totalEntities, codeHash);
+        }
+
+        // CASE 1: No checkpoint exists -> Create New
+        if (!existing) {
+            return this._createNew(dateStr, computationName, totalEntities, codeHash);
+        }
+
+        // CASE 2: Already Completed -> Return cached
+        if (existing.status === 'completed') {
             return { isCompleted: true };
         }
 
-        // Start new
-        const checkpointId = crypto.randomUUID();
-        await this.storage.initCheckpoint(dateStr, computationName, checkpointId, totalEntities);
+        // CASE 3: Zombie / Stale Lock Detection
+        // If status is 'running' but heartbeat is old (> 15 mins), it's a Zombie.
+        const lastUpdate = new Date(existing.last_updated).getTime();
+        const diffMinutes = (Date.now() - lastUpdate) / 1000 / 60;
+        const isStale = diffMinutes > 15; // 15 min timeout
+
+        // If it is running and valid (and not forced), we cannot touch it.
+        if (existing.status === 'running' && !isStale) {
+            return { isLocked: true, workerId: existing.worker_instance_id };
+        }
+
+        // CASE 4: Dead Letter Queue (Too many retries)
+        if (existing.attempts >= MAX_ATTEMPTS) {
+            console.warn(`[Checkpointer] ${computationName} exceeded max attempts (${existing.attempts}). Skipping.`);
+            return { skipped: true, reason: 'max_attempts_exceeded' };
+        }
+
+        // CASE 5: Resume / Steal Lock
+        // It's either 'failed', 'running' (stale), or 'running' (force). We steal it.
+        const reason = isStale ? 'ZOMBIE_DETECTED' : 'RESUME';
+        console.log(`[Checkpointer] Stealing lock for ${computationName} [${reason}] (Attempt ${existing.attempts + 1}/${MAX_ATTEMPTS})...`);
         
+        // Prepare new state based on previous
+        this.currentState = {
+            date: dateStr,
+            computation_name: computationName,
+            checkpoint_id: existing.checkpoint_id, // Keep ID to link history
+            worker_instance_id: this.workerId,
+            status: 'running',
+            processed_count: existing.processed_count || 0,
+            total_entities: totalEntities || existing.total_entities,
+            last_entity_id: existing.last_entity_id,
+            completed_batches: existing.completed_batches || [], // Keep previous progress
+            attempts: (existing.attempts || 1) + 1,
+            code_hash: codeHash,
+            started_at: existing.started_at,
+            last_updated: new Date().toISOString()
+        };
+
+        await this._persistState();
+        this._startHeartbeat();
+
         return {
-            id: checkpointId,
+            id: this.currentState.checkpoint_id,
+            processedCount: this.currentState.processed_count,
+            completedBatches: new Set(this.currentState.completed_batches),
+            lastEntityId: this.currentState.last_entity_id,
+            isResumed: true
+        };
+    }
+
+    async _createNew(dateStr, computationName, totalEntities, codeHash) {
+        const id = crypto.randomUUID();
+        
+        this.currentState = {
+            date: dateStr,
+            computation_name: computationName,
+            checkpoint_id: id,
+            worker_instance_id: this.workerId,
+            status: 'running',
+            processed_count: 0,
+            total_entities: totalEntities,
+            last_entity_id: null,
+            completed_batches: [],
+            attempts: 1,
+            code_hash: codeHash,
+            started_at: new Date().toISOString(),
+            last_updated: new Date().toISOString()
+        };
+
+        await this._persistState();
+        this._startHeartbeat();
+
+        return {
+            id,
             processedCount: 0,
             completedBatches: new Set(),
             lastEntityId: null,
@@ -45,21 +148,64 @@ class Checkpointer {
     }
     
     async markBatchComplete(dateStr, computationName, checkpointId, batchIndex, batchSize, lastEntityId) {
-        if (!this.enabled || !checkpointId) return;
+        if (!this.enabled || !this.currentState) return;
 
-        // We update processed count approximately based on batch size
+        // Update local state
         const processedCount = (batchIndex + 1) * batchSize;
+        const batches = new Set(this.currentState.completed_batches || []);
+        batches.add(batchIndex);
+
+        this.currentState.processed_count = processedCount;
+        this.currentState.last_entity_id = lastEntityId;
+        this.currentState.completed_batches = Array.from(batches); // Store as Array for BigQuery
+        this.currentState.last_updated = new Date().toISOString();
 
-        await this.storage.updateCheckpoint(dateStr, computationName, checkpointId, {
-            processedCount,
-            lastEntityId,
-            batchIndex
-        });
+        // Append new state row
+        await this._persistState();
     }
     
     async complete(dateStr, computationName, checkpointId) {
-        if (!this.enabled || !checkpointId) return;
-        await this.storage.completeCheckpoint(dateStr, computationName, checkpointId);
+        this._stopHeartbeat();
+        if (!this.enabled || !this.currentState) return;
+
+        this.currentState.status = 'completed';
+        this.currentState.last_updated = new Date().toISOString();
+
+        await this._persistState();
+        this.currentState = null; // Done
+    }
+
+    async _persistState() {
+        if (!this.storage.saveCheckpoint) {
+            if (this.storage.createCheckpoint) {
+                return this.storage.createCheckpoint(this.currentState);
+            }
+            throw new Error("StorageManager is missing 'saveCheckpoint' or 'createCheckpoint' method for Append-Only logic.");
+        }
+        return this.storage.saveCheckpoint(this.currentState);
+    }
+
+    _startHeartbeat() {
+        if (this.heartbeatInterval) clearInterval(this.heartbeatInterval);
+        
+        // Pulse every 2 minutes
+        this.heartbeatInterval = setInterval(async () => {
+            if (this.currentState) {
+                this.currentState.last_updated = new Date().toISOString();
+                try {
+                    await this._persistState();
+                } catch (e) {
+                    console.warn(`[Checkpointer] Heartbeat failed: ${e.message}`);
+                }
+            }
+        }, 2 * 60 * 1000);
+    }
+
+    _stopHeartbeat() {
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = null;
+        }
     }
 }