bulltrackers-module 1.0.768 → 1.0.770

package/functions/computation-system-v2/framework/scheduling/ScheduleValidator.js

@@ -1,11 +1,12 @@
 /**
  * @fileoverview Schedule Validator
- * 
- * Validates computation schedules and enforces timing rules:
+ * * Validates computation schedules and enforces timing rules:
  * 1. Parses schedule declarations
  * 2. Validates schedule format
- * 3. Checks 15-minute rule between dependent computations
- * 4. Generates warnings/errors for problematic schedules
+ * 3. Checks logical ordering (Dependent must not be scheduled BEFORE Dependency)
+ * * * UPDATE: Removed "Gap Warning" (race condition check). 
+ * * * The system's RunAnalyzer (Blocking) and Orchestrator (Cascading) 
+ * * * handle overlapping schedules safely, so 0-minute gaps are valid.
  */
 
 /**
@@ -37,6 +38,8 @@ class ScheduleValidator {
             time: '02:00',
             timezone: 'UTC'
         };
+        // dependencyGapMinutes is no longer used for validation warnings, 
+        // but kept if needed for other scheduling logic (e.g. cloud task delays)
         this.dependencyGapMinutes = config.scheduling?.dependencyGapMinutes || 15;
     }
     
@@ -261,7 +264,7 @@ class ScheduleValidator {
             const gap = this.calculateGap(depSchedule, entrySchedule);
             
             if (gap === null) {
-                // Different frequencies - can't directly compare
+                // Different frequencies - can't directly compare (Warning is still useful here as a heads up)
                 issues.push({
                     severity: 'warning',
                     computation: entry.name,
@@ -272,27 +275,22 @@ class ScheduleValidator {
                 continue;
             }
             
+            // STRICT CHECK: Dependent CANNOT run BEFORE dependency
             if (gap < 0) {
-                // Dependent runs BEFORE its dependency
                 issues.push({
                     severity: 'error',
                     computation: entry.name,
                     dependency: depName,
                     gap,
                     message: `${entry.name} is scheduled BEFORE its dependency ${depName} (${Math.abs(gap)} minutes earlier)`,
-                    suggestion: `Move ${entry.name} to at least ${this.dependencyGapMinutes} minutes after ${depName}`
+                    suggestion: `Move ${entry.name} to after ${depName} or use the default schedule.`
                 });
-            } else if (gap < this.dependencyGapMinutes) {
-                // Too close - race condition risk
-                issues.push({
-                    severity: 'warning',
-                    computation: entry.name,
-                    dependency: depName,
-                    gap,
-                    message: `${entry.name} scheduled only ${gap} minutes after dependency ${depName}`,
-                    suggestion: `Increase gap to at least ${this.dependencyGapMinutes} minutes`
-                });
-            }
+            } 
+            
+            // REMOVED: Warning for (0 <= gap < 15).
+            // Reason: In this architecture, dependents are triggered via Event Cascade (Pass 1 -> Pass 2).
+            // A 0-minute gap (or same default schedule) is safe because the RunAnalyzer will simply BLOCK
+            // the dependent until the dependency is ready, or the Orchestrator will trigger it automatically.
         }
         
         return issues;
@@ -324,4 +322,4 @@ class ScheduleValidator {
     }
 }
 
-module.exports = { ScheduleValidator };
+module.exports = { ScheduleValidator };

package/functions/computation-system-v2/framework/storage/StateRepository.js

@@ -4,6 +4,7 @@
  * 1. Loading daily execution status (hashes, timestamps)
  * 2. Loading previous results (for dependencies and history)
  * 3. Caching results for performance
+ * * * UPDATE: Added getRunDates() to support automatic backfill fan-out.
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
@@ -87,6 +88,37 @@ class StateRepository {
         return statusMap;
     }
 
+    /**
+     * Fetch all dates where a computation has previously run.
+     * Used for fan-out / backfill operations on code deployment.
+     * @param {string} computationName 
+     * @returns {Promise<string[]>} List of YYYY-MM-DD strings
+     */
+    async getRunDates(computationName) {
+        try {
+            const table = this.config.resultStore?.table || 'computation_results';
+            const fullTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${table}\``;
+
+            const query = `
+                SELECT DISTINCT date
+                FROM ${fullTable}
+                WHERE computation_name = @compName
+                ORDER BY date DESC
+            `;
+
+            const [rows] = await this.bigquery.query({
+                query,
+                params: { compName: computationName.toLowerCase() },
+                location: this.config.bigquery.location
+            });
+
+            return rows.map(r => r.date.value || r.date);
+        } catch (e) {
+            this._log('WARN', `Failed to fetch run dates for ${computationName}: ${e.message}`);
+            return [];
+        }
+    }
+
     /**
      * Update the local status cache after a write.
      * @param {string} dateStr 
@@ -208,8 +240,6 @@ class StateRepository {
 
     /**
      * Get multiple entity results in a single query (batch lazy load)
-     * FIXED: This solves the N+1 problem by allowing the Executor to fetch dependencies
-     * for an entire processing batch in one go.
      */
     async getBatchEntityResults(dateStr, computationName, entityIds) {
         if (!entityIds || entityIds.length === 0) return {};

package/functions/computation-system-v2/framework/storage/StorageManager.js

@@ -23,20 +23,20 @@ class StorageManager {
     constructor(config, logger = null) {
         this.config = config;
         this.logger = logger;
-        
+
         this.bigquery = new BigQuery({
             projectId: config.bigquery?.projectId,
             location: config.bigquery?.location || 'EU'
         });
-        
+
         this.storage = new Storage({
             projectId: config.bigquery?.projectId
         });
-        
+
         this._firestore = null;
         this.tableExists = new Map();
     }
-    
+
     get firestore() {
         if (!this._firestore) {
             this._firestore = new Firestore({
@@ -52,7 +52,7 @@ class StorageManager {
      */
     async claimZombie(checkpointId) {
         if (!checkpointId) return;
-        
+
         const { projectId, dataset } = this.config.bigquery;
 
         const query = `
@@ -60,7 +60,7 @@ class StorageManager {
             SET last_updated = CURRENT_TIMESTAMP()
             WHERE checkpoint_id = @checkpointId
         `;
-        
+
         try {
             await this.bigquery.query({
                 query,
@@ -72,8 +72,8 @@ class StorageManager {
         }
     }
 
-    
-    
+
+
     // =========================================================================
     // RESULT COMMITTING (Batch -> GCS Buffer)
     // =========================================================================
@@ -86,7 +86,7 @@ class StorageManager {
     async commitResults(dateStr, entry, results, depResultHashes = {}) {
         const storageConfig = this._resolveStorageConfig(entry);
         const startTime = Date.now();
-        
+
         // Define GCS Task (Fatal on error)
         const gcsTask = async () => {
             if (storageConfig.bigquery === false) return null;
@@ -116,11 +116,11 @@ class StorageManager {
             firestoreTask()
         ]);
 
-        const writeResults = { 
-            bigquery: bigqueryResult, 
-            firestore: firestoreResult 
+        const writeResults = {
+            bigquery: bigqueryResult,
+            firestore: firestoreResult
         };
-        
+
         const duration = Date.now() - startTime;
         this._log('INFO', `Committed (Staged) ${entry.name} results in ${duration}ms`);
         return writeResults;
@@ -138,18 +138,18 @@ class StorageManager {
         const table = this.config.resultStore?.table || 'computation_results';
         const bucketName = this.config.gcs?.bucket;
         const prefix = this.config.gcs?.prefix || 'staging';
-        
+
         // 1. Define GCS path pattern: gs://bucket/prefix/date/computation/*.json
         const filePrefix = `${prefix}/${dateStr}/${entry.name}/`;
         const gcsPath = `gs://${bucketName}/${filePrefix}*.json`;
-        
+
         this._log('INFO', `Finalizing ${entry.name}...`);
 
         try {
             // FIX: Check if files actually exist before trying to load them
             // If the computation produced 0 results, no files exist, and BQ will throw "Not Found".
             const [files] = await this.storage.bucket(bucketName).getFiles({ prefix: filePrefix });
-            
+
             if (!files || files.length === 0) {
                 this._log('INFO', `No staged files found for ${entry.name}. Skipping finalization (Empty Result).`);
                 return;
@@ -161,7 +161,7 @@ class StorageManager {
             // 2. Load GCS files into a Temporary Table
             // We create the temp table with the exact schema we expect first
             await this._createTempTableForLoad(tempTableId);
-            
+
             // FIX: Use bigquery.createJob directly.
             const [job] = await this.bigquery.createJob({
                 configuration: {
@@ -234,9 +234,9 @@ class StorageManager {
             await this.bigquery.dataset(dataset).table(table).insert([row]);
         } catch (error) {
             if (error.name === 'PartialFailureError' || error.errors) {
-                 this._log('ERROR', `Checkpoint insert failed: ${JSON.stringify(error.errors)}`);
+                this._log('ERROR', `Checkpoint insert failed: ${JSON.stringify(error.errors)}`);
             } else {
-                 this._log('ERROR', `Checkpoint insert failed: ${error.message}`);
+                this._log('ERROR', `Checkpoint insert failed: ${error.message}`);
             }
             throw error;
         }
@@ -375,15 +375,15 @@ class StorageManager {
                 LIMIT 50
             `;
             const [rows] = await this.bigquery.query({ query, params: { minutes: minutesThreshold }, location: this.config.bigquery.location });
-            return rows.map(r => ({ 
-                name: r.computation_name, 
-                date: r.date.value || r.date, 
+            return rows.map(r => ({
+                name: r.computation_name,
+                date: r.date.value || r.date,
                 checkpointId: r.checkpoint_id,
-                attempts: r.attempts 
+                attempts: r.attempts
             }));
-        } catch (e) { 
+        } catch (e) {
             console.error(`[Storage] findZombies failed: ${e.message}`);
-            return []; 
+            return [];
         }
     }
 
@@ -417,7 +417,7 @@ class StorageManager {
             `;
             const [rows] = await this.bigquery.query({ query, params: { date: dateStr, computationName }, location: this.config.bigquery.location });
             if (rows.length === 0) return null;
-            
+
             const r = rows[0];
             return {
                 ...r,
@@ -438,17 +438,17 @@ class StorageManager {
     async savePerformanceReport(report) {
         const table = 'computation_performance';
         const dataset = this.config.bigquery.dataset;
-        await this._ensurePerformanceTable(table);  
+        await this._ensurePerformanceTable(table);
 
         const row = {
-            run_id : report.runId || 'unknown',
-            computation_name : report.computationName,
+            run_id: report.runId || 'unknown',
+            computation_name: report.computationName,
             date: report.date,
             duration_ms: report.durationMs,
-            metrics: JSON.stringify (report.metrics || {}),
+            metrics: JSON.stringify(report.metrics || {}),
             entity_count: report.entityCount || 0,
-            status : report.status || 'completed',
-            created_at : this.bigquery.timestamp(new Date())
+            status: report.status || 'completed',
+            created_at: this.bigquery.timestamp(new Date())
         };
         try {
             await this.bigquery.dataset(dataset).table(table).insert([row]);
@@ -478,27 +478,27 @@ class StorageManager {
     async _stageToGCS(dateStr, entry, results, depResultHashes) {
         const rows = this._buildBigQueryRows(dateStr, entry, results, depResultHashes);
         if (rows.length === 0) return { rowCount: 0 };
-        
+
         const bucketName = this.config.gcs?.bucket;
         const prefix = this.config.gcs?.prefix || 'staging';
         const filename = `${prefix}/${dateStr}/${entry.name}/${crypto.randomUUID()}.json`;
-        
+
         const file = this.storage.bucket(bucketName).file(filename);
-        
+
         const ndjson = rows.map(r => JSON.stringify(r)).join('\n');
-        
+
         await file.save(ndjson, {
             contentType: 'application/json',
-            resumable: false 
+            resumable: false
         });
-        
+
         return { rowCount: rows.length, gcsUri: `gs://${bucketName}/${filename}` };
     }
 
     async _createTempTableForLoad(tableName) {
         const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
         const table = dataset.table(tableName);
-        
+
         const schema = [
             { name: 'date', type: 'DATE', mode: 'REQUIRED' },
             { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
@@ -518,7 +518,7 @@ class StorageManager {
     async _mergeStagedData(targetTable, tempTable) {
         const fullTarget = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${targetTable}\``;
         const fullTemp = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${tempTable}\``;
-        
+
         await this._ensureBigQueryTable(targetTable);
 
         // FIX: Removed SAFE.PARSE_JSON() because target columns are STRING.
@@ -545,21 +545,59 @@ class StorageManager {
                 VALUES (S.date, S.computation_name, S.category, S.entity_id, S.code_hash, S.result_hash, 
                         S.dependency_result_hashes, S.entity_count, S.result_data, S.updated_at)
         `;
-        
+
         // UPDATE: Use createQueryJob to capture DML statistics
         try {
-            const [job] = await this.bigquery.createQueryJob({ 
-                query: mergeQuery, 
-                location: this.config.bigquery.location 
+            const [job] = await this.bigquery.createQueryJob({
+                query: mergeQuery,
+                location: this.config.bigquery.location
             });
-            
+
             await job.getQueryResults(); // Wait for completion
-            
+
             const metadata = await job.getMetadata();
             const stats = metadata[0]?.statistics?.query;
             const affectedRows = stats?.numDmlAffectedRows;
 
             this._log('INFO', `Merge complete on ${targetTable}. Rows affected (Inserted/Updated): ${affectedRows}`);
+
+            // =========================================================================
+            // CRITICAL FIX: Delete stale entity rows that were NOT in the new staging data.
+            // This prevents ghost entities from previous runs from polluting results.
+            // We identify the date and computation from the temp table and remove any
+            // rows in the target that don't have a matching entity_id in the new run.
+            // =========================================================================
+            const deleteQuery = `
+                DELETE FROM ${fullTarget} T
+                WHERE EXISTS (
+                    -- Identify which (date, computation_name) pairs were just processed
+                    SELECT 1 FROM ${fullTemp} S
+                    WHERE S.date = T.date AND S.computation_name = T.computation_name
+                )
+                AND NOT EXISTS (
+                    -- Keep only entity_ids that are in the new staging data
+                    SELECT 1 FROM ${fullTemp} S
+                    WHERE S.date = T.date 
+                      AND S.computation_name = T.computation_name 
+                      AND S.entity_id = T.entity_id
+                )
+            `;
+
+            const [deleteJob] = await this.bigquery.createQueryJob({
+                query: deleteQuery,
+                location: this.config.bigquery.location
+            });
+
+            await deleteJob.getQueryResults();
+
+            const deleteMeta = await deleteJob.getMetadata();
+            const deleteStats = deleteMeta[0]?.statistics?.query;
+            const deletedRows = deleteStats?.numDmlAffectedRows;
+
+            if (deletedRows && parseInt(deletedRows, 10) > 0) {
+                this._log('INFO', `Cleanup: Deleted ${deletedRows} stale entity rows from ${targetTable}`);
+            }
+
         } catch (e) {
             this._logError(`Merge Failed on ${targetTable}`, e);
             throw e;
@@ -578,7 +616,7 @@ class StorageManager {
         const rows = [];
         const timestamp = new Date().toISOString();
         const depResultHashesJson = JSON.stringify(depResultHashes);
-        
+
         if (entry.type === 'per-entity' && typeof results === 'object') {
             for (const [entityId, data] of Object.entries(results)) {
                 rows.push({
@@ -650,7 +688,7 @@ class StorageManager {
                 { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
                 { name: 'date', type: 'DATE', mode: 'REQUIRED' },
                 { name: 'duration_ms', type: 'INTEGER', mode: 'NULLABLE' },
-                { name: 'metrics', type: 'STRING', mode: 'NULLABLE' }, 
+                { name: 'metrics', type: 'STRING', mode: 'NULLABLE' },
                 { name: 'entity_count', type: 'INTEGER', mode: 'NULLABLE' },
                 { name: 'status', type: 'STRING', mode: 'NULLABLE' },
                 { name: 'created_at', type: 'TIMESTAMP', mode: 'REQUIRED' }
@@ -663,28 +701,28 @@ class StorageManager {
             });
             this._log('INFO', `Created table ${tableName}`);
         }
-        
+
         this.tableExists.set(tableName, true);
     }
-    
+
     async _ensureCheckpointTable(tableName) {
         if (this.tableExists.get(tableName)) return;
         const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
         const table = dataset.table(tableName);
         const [exists] = await table.exists();
-        
+
         const schema = [
             { name: 'date', type: 'DATE', mode: 'REQUIRED' },
             { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
             { name: 'checkpoint_id', type: 'STRING', mode: 'REQUIRED' },
             { name: 'worker_instance_id', type: 'STRING', mode: 'NULLABLE' },
-            { name: 'code_hash', type: 'STRING', mode: 'NULLABLE' }, 
+            { name: 'code_hash', type: 'STRING', mode: 'NULLABLE' },
             { name: 'status', type: 'STRING', mode: 'REQUIRED' },
             { name: 'processed_count', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'total_entities', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'last_entity_id', type: 'STRING', mode: 'NULLABLE' },
             { name: 'completed_batches', type: 'INTEGER', mode: 'REPEATED' },
-            { name: 'attempts', type: 'INTEGER', mode: 'NULLABLE' }, 
+            { name: 'attempts', type: 'INTEGER', mode: 'NULLABLE' },
             { name: 'started_at', type: 'TIMESTAMP', mode: 'REQUIRED' },
             { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
         ];
@@ -695,38 +733,38 @@ class StorageManager {
                 timePartitioning: { type: 'DAY', field: 'date' },
                 clustering: { fields: ['computation_name', 'status'] }
             });
-        } 
+        }
         this.tableExists.set(tableName, true);
     }
 
     _writeToFirestore(dateStr, entry, results, firestoreConfig) {
         const { path, merge, includeMetadata } = firestoreConfig;
         if (!path) throw new Error(`Firestore path not configured for ${entry.name}`);
-        
+
         const timestamp = new Date();
         const metadata = includeMetadata ? {
-            _computedAt: timestamp, _computationDate: dateStr, 
+            _computedAt: timestamp, _computationDate: dateStr,
             _computationName: entry.name, _codeHash: entry.hash
         } : {};
-        
+
         let docCount = 0;
-        
+
         if (entry.type === 'per-entity' && typeof results === 'object') {
             const batches = [];
             let currentBatch = this.firestore.batch();
             let batchCount = 0;
             const MAX_BATCH = 500;
-            
+
             for (const [entityId, data] of Object.entries(results)) {
                 const docPath = this._resolvePath(path, {
                     entityId, date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
                 });
-                
+
                 const docRef = this.firestore.doc(docPath);
                 const docData = { ...data, ...metadata };
-                
+
                 merge ? currentBatch.set(docRef, docData, { merge: true }) : currentBatch.set(docRef, docData);
-                
+
                 batchCount++; docCount++;
                 if (batchCount >= MAX_BATCH) {
                     batches.push(currentBatch);
@@ -735,10 +773,10 @@ class StorageManager {
                 }
             }
             if (batchCount > 0) batches.push(currentBatch);
-            
+
             const limit = pLimit(10);
             return Promise.all(batches.map(b => limit(() => b.commit()))).then(() => ({ docCount }));
-            
+
         } else {
             const docPath = this._resolvePath(path, {
                 entityId: '_global', date: dateStr, computationName: entry.name, category: entry.category || 'uncategorized'
@@ -763,7 +801,7 @@ class StorageManager {
         const str = typeof data === 'string' ? data : JSON.stringify(data);
         return crypto.createHash('md5').update(str).digest('hex').substring(0, 16);
     }
-    
+
     _log(level, message) {
         this.logger?.log ? this.logger.log(level, `[StorageManager] ${message}`) : console.log(`[${level}] [StorageManager] ${message}`);
     }
@@ -775,10 +813,10 @@ class StorageManager {
         } else if (error.response && error.response.insertErrors) {
             details = JSON.stringify(error.response.insertErrors, null, 2);
         }
-        
+
         this._log('ERROR', `${context}: ${details}`);
     }
-    
+
 }
 
 module.exports = { StorageManager };

package/functions/computation-system-v2/framework/testing/ComputationTester.js

@@ -4,7 +4,7 @@
  * - Redirects output to a test table.
  * - Resolves and executes dependency chains.
  * - Finds valid runnable dates based on raw data availability.
- * * * UPDATE: Removed SQL output fetching support.
+ * * * UPDATE: Fixed dependency reading to look at test table instead of production.
  */
 
 const fs = require('fs');
@@ -28,14 +28,21 @@ class IntegrationTester {
     _createTestConfig(config) {
         const testConfig = JSON.parse(JSON.stringify(config)); // Deep clone
         
-        // 1. Redirect Storage
+        // 1. Redirect Storage (Writes)
         if (!testConfig.resultStore) testConfig.resultStore = {};
         testConfig.resultStore.table = 'computation_results_test';
+
+        // 2. Redirect Reads for Intermediate Results (Crucial for Chained Computations)
+        // We must ensure that when a computation asks for 'computation_results', 
+        // it reads from the TEST table where the upstream dependency just wrote.
+        if (testConfig.tables && testConfig.tables['computation_results']) {
+            testConfig.tables['computation_results'].tableName = 'computation_results_test';
+        }
         
-        // 2. Disable Side Effects
+        // 3. Disable Side Effects
         testConfig.cloudTasks = null; // Don't trigger downstream
         
-        // 3. Force Local Execution for simplified debugging
+        // 4. Force Local Execution for simplified debugging
         if (testConfig.workerPool) {
             testConfig.workerPool.enabled = false;
         }
@@ -122,8 +129,7 @@ class IntegrationTester {
             force: true // Force run even if "up to date" in test table
         });
 
-        // UPDATE: Removed SQL output fetching logic. 
-        // Always fetch from the standard State Repository (JSON result store).
+        // Always fetch from the standard State Repository (JSON result store)
         const result = await this.orchestrator.stateRepository.getResult(dateStr, targetName);
 
         return {