bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/framework/scheduling/ScheduleValidator.js

@@ -1,11 +1,12 @@
 /**
  * @fileoverview Schedule Validator
- * 
- * Validates computation schedules and enforces timing rules:
+ * * Validates computation schedules and enforces timing rules:
  * 1. Parses schedule declarations
  * 2. Validates schedule format
- * 3. Checks 15-minute rule between dependent computations
- * 4. Generates warnings/errors for problematic schedules
+ * 3. Checks logical ordering (Dependent must not be scheduled BEFORE Dependency)
+ * * * UPDATE: Removed "Gap Warning" (race condition check). 
+ * * * The system's RunAnalyzer (Blocking) and Orchestrator (Cascading) 
+ * * * handle overlapping schedules safely, so 0-minute gaps are valid.
  */
 
 /**
@@ -37,6 +38,8 @@ class ScheduleValidator {
             time: '02:00',
             timezone: 'UTC'
         };
+        // dependencyGapMinutes is no longer used for validation warnings, 
+        // but kept if needed for other scheduling logic (e.g. cloud task delays)
         this.dependencyGapMinutes = config.scheduling?.dependencyGapMinutes || 15;
     }
     
@@ -261,7 +264,7 @@ class ScheduleValidator {
             const gap = this.calculateGap(depSchedule, entrySchedule);
             
             if (gap === null) {
-                // Different frequencies - can't directly compare
+                // Different frequencies - can't directly compare (Warning is still useful here as a heads up)
                 issues.push({
                     severity: 'warning',
                     computation: entry.name,
@@ -272,27 +275,22 @@ class ScheduleValidator {
                 continue;
             }
             
+            // STRICT CHECK: Dependent CANNOT run BEFORE dependency
             if (gap < 0) {
-                // Dependent runs BEFORE its dependency
                 issues.push({
                     severity: 'error',
                     computation: entry.name,
                     dependency: depName,
                     gap,
                     message: `${entry.name} is scheduled BEFORE its dependency ${depName} (${Math.abs(gap)} minutes earlier)`,
-                    suggestion: `Move ${entry.name} to at least ${this.dependencyGapMinutes} minutes after ${depName}`
+                    suggestion: `Move ${entry.name} to after ${depName} or use the default schedule.`
                 });
-            } else if (gap < this.dependencyGapMinutes) {
-                // Too close - race condition risk
-                issues.push({
-                    severity: 'warning',
-                    computation: entry.name,
-                    dependency: depName,
-                    gap,
-                    message: `${entry.name} scheduled only ${gap} minutes after dependency ${depName}`,
-                    suggestion: `Increase gap to at least ${this.dependencyGapMinutes} minutes`
-                });
-            }
+            } 
+            
+            // REMOVED: Warning for (0 <= gap < 15).
+            // Reason: In this architecture, dependents are triggered via Event Cascade (Pass 1 -> Pass 2).
+            // A 0-minute gap (or same default schedule) is safe because the RunAnalyzer will simply BLOCK
+            // the dependent until the dependency is ready, or the Orchestrator will trigger it automatically.
         }
         
         return issues;
@@ -324,4 +322,4 @@ class ScheduleValidator {
     }
 }
 
-module.exports = { ScheduleValidator };
+module.exports = { ScheduleValidator };

package/functions/computation-system-v2/framework/storage/StateRepository.js

@@ -4,6 +4,7 @@
  * 1. Loading daily execution status (hashes, timestamps)
  * 2. Loading previous results (for dependencies and history)
  * 3. Caching results for performance
+ * * * UPDATE: Added getRunDates() to support automatic backfill fan-out.
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
@@ -87,6 +88,37 @@ class StateRepository {
         return statusMap;
     }
 
+    /**
+     * Fetch all dates where a computation has previously run.
+     * Used for fan-out / backfill operations on code deployment.
+     * @param {string} computationName 
+     * @returns {Promise<string[]>} List of YYYY-MM-DD strings
+     */
+    async getRunDates(computationName) {
+        try {
+            const table = this.config.resultStore?.table || 'computation_results';
+            const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+            const query = `
+                SELECT DISTINCT date
+                FROM ${fullTable}
+                WHERE computation_name = @compName
+                ORDER BY date DESC
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { compName: computationName.toLowerCase() },
+                location: this.config.bigquery.location
+            });
+
+            return rows.map(r => r.date.value || r.date);
+        } catch (e) {
+            this._log('WARN', `Failed to fetch run dates for ${computationName}: ${e.message}`);
+            return [];
+        }
+    }
+
     /**
      * Update the local status cache after a write.
      * @param {string} dateStr 
@@ -208,8 +240,6 @@ class StateRepository {
 
     /**
      * Get multiple entity results in a single query (batch lazy load)
-     * FIXED: This solves the N+1 problem by allowing the Executor to fetch dependencies
-     * for an entire processing batch in one go.
      */
     async getBatchEntityResults(dateStr, computationName, entityIds) {
         if (!entityIds || entityIds.length === 0) return {};

package/functions/computation-system-v2/framework/storage/StorageManager.js

@@ -10,7 +10,7 @@
  * * FIX: Switched to bigquery.createJob for GCS imports to prevent local file path interpretation errors.
  * * FIX: Improved error logging to catch swallowed BigQuery insert errors.
  * * FIX: finalizeResults now checks for file existence to prevent "Not found" errors on empty results.
- * * FIX: Added SAFE.PARSE_JSON to MERGE statement for BOTH result_data and dependency_result_hashes.
+ * * FIX: Removed SAFE.PARSE_JSON from MERGE to match STRING schema types.
  */
 
 const { Firestore } = require('@google-cloud/firestore');
@@ -23,20 +23,20 @@ class StorageManager {
     constructor(config, logger = null) {
         this.config = config;
         this.logger = logger;
-        
+
         this.bigquery = new BigQuery({
             projectId: config.bigquery?.projectId,
             location: config.bigquery?.location || 'EU'
         });
-        
+
         this.storage = new Storage({
             projectId: config.bigquery?.projectId
         });
-        
+
         this._firestore = null;
         this.tableExists = new Map();
     }
-    
+
     get firestore() {
         if (!this._firestore) {
             this._firestore = new Firestore({
@@ -52,16 +52,15 @@ class StorageManager {
      */
     async claimZombie(checkpointId) {
         if (!checkpointId) return;
-        
-        // FIX: Access projectId and dataset from the config object
-        const { projectId, dataset } = this.config.bigquery; //
+
+        const { projectId, dataset } = this.config.bigquery;
 
         const query = `
             UPDATE \`${projectId}.${dataset}.computation_checkpoints\`
             SET last_updated = CURRENT_TIMESTAMP()
             WHERE checkpoint_id = @checkpointId
         `;
-        
+
         try {
             await this.bigquery.query({
                 query,
@@ -73,8 +72,8 @@ class StorageManager {
         }
     }
 
-    
-    
+
+
     // =========================================================================
     // RESULT COMMITTING (Batch -> GCS Buffer)
     // =========================================================================
@@ -87,7 +86,7 @@ class StorageManager {
     async commitResults(dateStr, entry, results, depResultHashes = {}) {
         const storageConfig = this._resolveStorageConfig(entry);
         const startTime = Date.now();
-        
+
         // Define GCS Task (Fatal on error)
         const gcsTask = async () => {
             if (storageConfig.bigquery === false) return null;
@@ -117,11 +116,11 @@ class StorageManager {
             firestoreTask()
         ]);
 
-        const writeResults = { 
-            bigquery: bigqueryResult, 
-            firestore: firestoreResult 
+        const writeResults = {
+            bigquery: bigqueryResult,
+            firestore: firestoreResult
         };
-        
+
         const duration = Date.now() - startTime;
         this._log('INFO', `Committed (Staged) ${entry.name} results in ${duration}ms`);
         return writeResults;
@@ -139,18 +138,18 @@ class StorageManager {
         const table = this.config.resultStore?.table || 'computation_results';
         const bucketName = this.config.gcs?.bucket;
         const prefix = this.config.gcs?.prefix || 'staging';
-        
+
         // 1. Define GCS path pattern: gs://bucket/prefix/date/computation/*.json
         const filePrefix = `${prefix}/${dateStr}/${entry.name}/`;
         const gcsPath = `gs://${bucketName}/${filePrefix}*.json`;
-        
+
         this._log('INFO', `Finalizing ${entry.name}...`);
 
         try {
             // FIX: Check if files actually exist before trying to load them
             // If the computation produced 0 results, no files exist, and BQ will throw "Not Found".
             const [files] = await this.storage.bucket(bucketName).getFiles({ prefix: filePrefix });
-            
+
             if (!files || files.length === 0) {
                 this._log('INFO', `No staged files found for ${entry.name}. Skipping finalization (Empty Result).`);
                 return;
@@ -162,7 +161,7 @@ class StorageManager {
             // 2. Load GCS files into a Temporary Table
             // We create the temp table with the exact schema we expect first
             await this._createTempTableForLoad(tempTableId);
-            
+
             // FIX: Use bigquery.createJob directly.
             const [job] = await this.bigquery.createJob({
                 configuration: {
@@ -235,9 +234,9 @@ class StorageManager {
             await this.bigquery.dataset(dataset).table(table).insert([row]);
         } catch (error) {
             if (error.name === 'PartialFailureError' || error.errors) {
-                 this._log('ERROR', `Checkpoint insert failed: ${JSON.stringify(error.errors)}`);
+                this._log('ERROR', `Checkpoint insert failed: ${JSON.stringify(error.errors)}`);
             } else {
-                 this._log('ERROR', `Checkpoint insert failed: ${error.message}`);
+                this._log('ERROR', `Checkpoint insert failed: ${error.message}`);
             }
             throw error;
         }
@@ -349,10 +348,6 @@ class StorageManager {
         const table = 'computation_checkpoints';
         const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
         try {
-            // FIX: Use subquery with ROW_NUMBER to find the TRUE latest state per computation.
-            // We only count it as a zombie if the LATEST row is 'running'.
-            // This ignores 'running' rows that have a newer (or same-time) 'completed' sibling.
-            // UPDATE: Added attempts to the selection
             const query = `
                 SELECT computation_name, date, checkpoint_id, last_updated, attempts
                 FROM (
@@ -380,15 +375,15 @@ class StorageManager {
                 LIMIT 50
             `;
             const [rows] = await this.bigquery.query({ query, params: { minutes: minutesThreshold }, location: this.config.bigquery.location });
-            return rows.map(r => ({ 
-                name: r.computation_name, 
-                date: r.date.value || r.date, 
+            return rows.map(r => ({
+                name: r.computation_name,
+                date: r.date.value || r.date,
                 checkpointId: r.checkpoint_id,
-                attempts: r.attempts 
+                attempts: r.attempts
             }));
-        } catch (e) { 
+        } catch (e) {
             console.error(`[Storage] findZombies failed: ${e.message}`);
-            return []; 
+            return [];
         }
     }
 
@@ -407,9 +402,6 @@ class StorageManager {
         const table = 'computation_checkpoints';
         const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
         try {
-            // FIX: Added Tie-Breaker logic to ORDER BY
-            // If timestamps are identical, 'completed' (1) comes before 'failed' (2) before 'running' (3).
-            // This ensures we never accidentally pick a "running" row when a "completed" one exists at the exact same ms.
             const query = `
                 SELECT checkpoint_id, status, processed_count, last_entity_id, completed_batches, worker_instance_id, last_updated, attempts, code_hash, started_at
                 FROM ${fullTable}
@@ -425,7 +417,7 @@ class StorageManager {
             `;
             const [rows] = await this.bigquery.query({ query, params: { date: dateStr, computationName }, location: this.config.bigquery.location });
             if (rows.length === 0) return null;
-            
+
             const r = rows[0];
             return {
                 ...r,
@@ -446,17 +438,17 @@ class StorageManager {
     async savePerformanceReport(report) {
         const table = 'computation_performance';
         const dataset = this.config.bigquery.dataset;
-        await this._ensurePerformanceTable(table);  
+        await this._ensurePerformanceTable(table);
 
         const row = {
-            run_id : report.runId || 'unknown',
-            computation_name : report.computationName,
+            run_id: report.runId || 'unknown',
+            computation_name: report.computationName,
             date: report.date,
             duration_ms: report.durationMs,
-            metrics: JSON.stringify (report.metrics || {}),
+            metrics: JSON.stringify(report.metrics || {}),
             entity_count: report.entityCount || 0,
-            status : report.status || 'completed',
-            created_at : this.bigquery.timestamp(new Date())
+            status: report.status || 'completed',
+            created_at: this.bigquery.timestamp(new Date())
         };
         try {
             await this.bigquery.dataset(dataset).table(table).insert([row]);
@@ -486,29 +478,27 @@ class StorageManager {
     async _stageToGCS(dateStr, entry, results, depResultHashes) {
         const rows = this._buildBigQueryRows(dateStr, entry, results, depResultHashes);
         if (rows.length === 0) return { rowCount: 0 };
-        
+
         const bucketName = this.config.gcs?.bucket;
         const prefix = this.config.gcs?.prefix || 'staging';
         const filename = `${prefix}/${dateStr}/${entry.name}/${crypto.randomUUID()}.json`;
-        
+
         const file = this.storage.bucket(bucketName).file(filename);
-        
+
         const ndjson = rows.map(r => JSON.stringify(r)).join('\n');
-        
+
         await file.save(ndjson, {
             contentType: 'application/json',
-            resumable: false 
+            resumable: false
         });
-        
+
         return { rowCount: rows.length, gcsUri: `gs://${bucketName}/${filename}` };
     }
 
     async _createTempTableForLoad(tableName) {
         const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
         const table = dataset.table(tableName);
-        
-        // Note: result_data and dependency_result_hashes are loaded as STRING from the JSON file
-        // They will be parsed into JSON during the merge step.
+
         const schema = [
             { name: 'date', type: 'DATE', mode: 'REQUIRED' },
             { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
@@ -528,9 +518,10 @@ class StorageManager {
     async _mergeStagedData(targetTable, tempTable) {
         const fullTarget = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${targetTable}\``;
         const fullTemp = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${tempTable}\``;
-        
+
         await this._ensureBigQueryTable(targetTable);
 
+        // FIX: Removed SAFE.PARSE_JSON() because target columns are STRING.
         const mergeQuery = `
             MERGE INTO ${fullTarget} T
             USING (
@@ -544,31 +535,69 @@ class StorageManager {
                 UPDATE SET 
                     code_hash = S.code_hash, 
                     result_hash = S.result_hash, 
-                    dependency_result_hashes = SAFE.PARSE_JSON(S.dependency_result_hashes), 
+                    dependency_result_hashes = S.dependency_result_hashes, 
                     entity_count = S.entity_count, 
-                    result_data = SAFE.PARSE_JSON(S.result_data), 
+                    result_data = S.result_data, 
                     updated_at = S.updated_at
             WHEN NOT MATCHED THEN
                 INSERT (date, computation_name, category, entity_id, code_hash, result_hash, 
                         dependency_result_hashes, entity_count, result_data, updated_at)
                 VALUES (S.date, S.computation_name, S.category, S.entity_id, S.code_hash, S.result_hash, 
-                        SAFE.PARSE_JSON(S.dependency_result_hashes), S.entity_count, SAFE.PARSE_JSON(S.result_data), S.updated_at)
+                        S.dependency_result_hashes, S.entity_count, S.result_data, S.updated_at)
         `;
-        
+
         // UPDATE: Use createQueryJob to capture DML statistics
         try {
-            const [job] = await this.bigquery.createQueryJob({ 
-                query: mergeQuery, 
-                location: this.config.bigquery.location 
+            const [job] = await this.bigquery.createQueryJob({
+                query: mergeQuery,
+                location: this.config.bigquery.location
             });
-            
+
             await job.getQueryResults(); // Wait for completion
-            
+
             const metadata = await job.getMetadata();
             const stats = metadata[0]?.statistics?.query;
             const affectedRows = stats?.numDmlAffectedRows;
 
             this._log('INFO', `Merge complete on ${targetTable}. Rows affected (Inserted/Updated): ${affectedRows}`);
+
+            // =========================================================================
+            // CRITICAL FIX: Delete stale entity rows that were NOT in the new staging data.
+            // This prevents ghost entities from previous runs from polluting results.
+            // We identify the date and computation from the temp table and remove any
+            // rows in the target that don't have a matching entity_id in the new run.
+            // =========================================================================
+            const deleteQuery = `
+                DELETE FROM ${fullTarget} T
+                WHERE EXISTS (
+                    -- Identify which (date, computation_name) pairs were just processed
+                    SELECT 1 FROM ${fullTemp} S
+                    WHERE S.date = T.date AND S.computation_name = T.computation_name
+                )
+                AND NOT EXISTS (
+                    -- Keep only entity_ids that are in the new staging data
+                    SELECT 1 FROM ${fullTemp} S
+                    WHERE S.date = T.date 
+                      AND S.computation_name = T.computation_name 
+                      AND S.entity_id = T.entity_id
+                )
+            `;
+
+            const [deleteJob] = await this.bigquery.createQueryJob({
+                query: deleteQuery,
+                location: this.config.bigquery.location
+            });
+
+            await deleteJob.getQueryResults();
+
+            const deleteMeta = await deleteJob.getMetadata();
+            const deleteStats = deleteMeta[0]?.statistics?.query;
+            const deletedRows = deleteStats?.numDmlAffectedRows;
+
+            if (deletedRows && parseInt(deletedRows, 10) > 0) {
+                this._log('INFO', `Cleanup: Deleted ${deletedRows} stale entity rows from ${targetTable}`);
+            }
+
         } catch (e) {
             this._logError(`Merge Failed on ${targetTable}`, e);
             throw e;
@@ -587,7 +616,7 @@ class StorageManager {
         const rows = [];
         const timestamp = new Date().toISOString();
         const depResultHashesJson = JSON.stringify(depResultHashes);
-        
+
         if (entry.type === 'per-entity' && typeof results === 'object') {
             for (const [entityId, data] of Object.entries(results)) {
                 rows.push({
@@ -659,7 +688,7 @@ class StorageManager {
                 { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
                 { name: 'date', type: 'DATE', mode: 'REQUIRED' },
                 { name: 'duration_ms', type: 'INTEGER', mode: 'NULLABLE' },
-                { name: 'metrics', type: 'STRING', mode: 'NULLABLE' }, // JSON string
+                { name: 'metrics', type: 'STRING', mode: 'NULLABLE' },
                 { name: 'entity_count', type: 'INTEGER', mode: 'NULLABLE' },
                 { name: 'status', type: 'STRING', mode: 'NULLABLE' },
                 { name: 'created_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
@@ -672,28 +701,28 @@ class StorageManager {
             });
             this._log('INFO', `Created table ${tableName}`);
         }
-        
+
         this.tableExists.set(tableName, true);
     }
-    
+
     async _ensureCheckpointTable(tableName) {
         if (this.tableExists.get(tableName)) return;
         const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
         const table = dataset.table(tableName);
         const [exists] = await table.exists();
-        
+
         const schema = [
             { name: 'date', type: 'DATE', mode: 'REQUIRED' },
             { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
             { name: 'checkpoint_id', type: 'STRING', mode: 'REQUIRED' },
             { name: 'worker_instance_id', type: 'STRING', mode: 'NULLABLE' },
-            { name: 'code_hash', type: 'STRING', mode: 'NULLABLE' }, 
+            { name: 'code_hash', type: 'STRING', mode: 'NULLABLE' },
             { name: 'status', type: 'STRING', mode: 'REQUIRED' },
             { name: 'processed_count', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'total_entities', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'last_entity_id', type: 'STRING', mode: 'NULLABLE' },
             { name: 'completed_batches', type: 'INTEGER', mode: 'REPEATED' },
-            { name: 'attempts', type: 'INTEGER', mode: 'NULLABLE' }, 
+            { name: 'attempts', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'started_at', type: 'TIMESTAMP', mode: 'REQUIRED' },
             { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
         ];
@@ -704,38 +733,38 @@ class StorageManager {
                 timePartitioning: { type: 'DAY', field: 'date' },
                 clustering: { fields: ['computation_name', 'status'] }
             });
-        } 
+        }
         this.tableExists.set(tableName, true);
     }
 
     _writeToFirestore(dateStr, entry, results, firestoreConfig) {
         const { path, merge, includeMetadata } = firestoreConfig;
         if (!path) throw new Error(`Firestore path not configured for ${entry.name}`);
-        
+
         const timestamp = new Date();
         const metadata = includeMetadata ? {
-            _computedAt: timestamp, _computationDate: dateStr, 
+            _computedAt: timestamp, _computationDate: dateStr,
             _computationName: entry.name, _codeHash: entry.hash
         } : {};
-        
+
         let docCount = 0;
-        
+
         if (entry.type === 'per-entity' && typeof results === 'object') {
             const batches = [];
             let currentBatch = this.firestore.batch();
             let batchCount = 0;
             const MAX_BATCH = 500;
-            
+
             for (const [entityId, data] of Object.entries(results)) {
                 const docPath = this._resolvePath(path, {
                     entityId, date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
                 });
-                
+
                 const docRef = this.firestore.doc(docPath);
                 const docData = { ...data, ...metadata };
-                
+
                 merge ? currentBatch.set(docRef, docData, { merge: true }) : currentBatch.set(docRef, docData);
-                
+
                 batchCount++; docCount++;
                 if (batchCount >= MAX_BATCH) {
                     batches.push(currentBatch);
@@ -744,10 +773,10 @@ class StorageManager {
                 }
             }
             if (batchCount > 0) batches.push(currentBatch);
-            
+
             const limit = pLimit(10);
             return Promise.all(batches.map(b => limit(() => b.commit()))).then(() => ({ docCount }));
-            
+
         } else {
             const docPath = this._resolvePath(path, {
                 entityId: '_global', date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
@@ -772,23 +801,22 @@ class StorageManager {
         const str = typeof data === 'string' ? data : JSON.stringify(data);
         return crypto.createHash('md5').update(str).digest('hex').substring(0, 16);
     }
-    
+
     _log(level, message) {
         this.logger?.log ? this.logger.log(level, `[StorageManager] ${message}`) : console.log(`[${level}] [StorageManager] ${message}`);
     }
 
     _logError(context, error) {
-        // Safe logging for BigQuery PartialFailureError which hides details in .errors
         let details = error.message;
         if (error.errors && Array.isArray(error.errors)) {
             details = JSON.stringify(error.errors, null, 2);
         } else if (error.response && error.response.insertErrors) {
             details = JSON.stringify(error.response.insertErrors, null, 2);
         }
-        
+
         this._log('ERROR', `${context}: ${details}`);
     }
-    
+
 }
 
 module.exports = { StorageManager };