bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/framework/execution/Orchestrator.js

@@ -5,12 +5,7 @@
  * 2. Data Provisioning (Fetching Data, Loading Dependencies, Reference Data)
  * 3. Execution Strategy (Streaming vs. In-Memory)
  * 4. Delegation (Hands off actual 'work' to TaskRunner + Middleware)
- * * * UPDATE: Added Execution Summary logging to debug Skipped/Blocked/Impossible tasks.
- * * * UPDATE: Includes Global vs Batch Data Split to fix "Identity Crisis".
- * * * UPDATE: Implemented FORCE logic to bypass "up-to-date" checks for testing.
- * * * UPDATE: Aggregates performance reporting to prevent log spam.
- * * * FIX: Resolved N+1 Dependency Fetching (Strict Mode in Streaming).
- * * * FIX: Added missing 'skipped' property to return types for type safety.
+ * * * UPDATE: Added triggerDeploymentBackfill for automatic history correction.
  */
 
 const crypto = require('crypto');
@@ -45,14 +40,14 @@ class Orchestrator {
     constructor(config, logger = null) {
         this.config = config;
         this.logger = logger || console;
-        
+
         // 1. Initialize Base Services
         this.schemaRegistry = new SchemaRegistry(config.bigquery, this.logger);
         this.queryBuilder = new QueryBuilder(config.bigquery, this.schemaRegistry, this.logger);
         this.dataFetcher = new DataFetcher({ ...config.bigquery, tables: config.tables }, this.queryBuilder, this.logger);
         this.storageManager = new StorageManager(config, this.logger);
         this.stateRepository = new StateRepository(config, this.logger);
-        
+
         // 2. Initialize Logic & Rules
         this.manifestBuilder = new ManifestBuilder(config, this.logger);
         const rulesRegistry = new RulesRegistry(config, this.logger);
@@ -62,7 +57,7 @@ class Orchestrator {
         // Keep reference to profiler middleware to access stats later
         this.profilerMiddleware = new ProfilerMiddleware(config);
         this.profilerMiddleware.setStorage(this.storageManager);
-        
+
         this.lineageMiddleware = new LineageMiddleware(config);
         const costTracker = new CostTrackerMiddleware(config);
 
@@ -72,13 +67,13 @@ class Orchestrator {
             this.lineageMiddleware,
             this.profilerMiddleware
         ]);
-        
+
         // 4. Initialize Remote Task Runner (Worker Pool)
         // Only create if worker pool is enabled in config
-        this.remoteRunner = config.workerPool?.enabled 
+        this.remoteRunner = config.workerPool?.enabled
             ? new RemoteTaskRunner(config, this.logger)
             : null;
-        
+
         // State
         this.manifest = null;
         this.runAnalyzer = null;
@@ -99,6 +94,44 @@ class Orchestrator {
         this._log('INFO', `Initialized with ${this.manifest.length} computations`);
     }
 
+    // =========================================================================
+    // NEW: AUTOMATIC DEPLOYMENT BACKFILL
+    // =========================================================================
+
+    /**
+     * Trigger a backfill for all historical dates of a computation.
+     * Called when code changes are detected.
+     */
+    async triggerDeploymentBackfill(computationName) {
+        if (!this.manifest) await this.initialize();
+
+        const normName = computationName.toLowerCase().replace(/[^a-z0-9]/g, '');
+        const entry = this.manifest.find(e => e.name === normName);
+
+        if (!entry) throw new Error(`Computation ${computationName} not found in manifest`);
+
+        // 1. Fetch all dates this computation has ever run
+        // Relies on StateRepository.getRunDates() which must be implemented
+        const dates = await this.stateRepository.getRunDates(computationName);
+        this._log('INFO', `[Deploy] Found ${dates.length} historical dates for ${computationName}`);
+
+        if (dates.length === 0) return { scheduled: 0 };
+
+        // 2. Schedule tasks for all of them
+        // We do NOT use 'force: true'. We rely on RunAnalyzer to see the hash mismatch.
+        let scheduledCount = 0;
+        for (const dateStr of dates) {
+            await this._scheduleCloudTask(entry.originalName, dateStr, 'deployment_backfill');
+            scheduledCount++;
+        }
+
+        return { scheduled: scheduledCount };
+    }
+
+    // =========================================================================
+    // EXECUTION LOGIC
+    // =========================================================================
+
     async analyze(options) {
         const { date } = options;
         if (!this.manifest) await this.initialize();
@@ -112,7 +145,7 @@ class Orchestrator {
     async execute(options) {
         const { date, pass = null, computation = null, dryRun = false, entities = null } = options;
         if (!this.manifest) await this.initialize();
-        
+
         this._log('INFO', `Starting execution for ${date}...`);
 
         let toRun = this.manifest;
@@ -123,7 +156,7 @@ class Orchestrator {
         }
 
         const passes = this.manifestBuilder.groupByPass(toRun);
-        const passNumbers = Object.keys(passes).map(Number).sort((a,b) => a-b);
+        const passNumbers = Object.keys(passes).map(Number).sort((a, b) => a - b);
         const passesToRun = pass ? [parseInt(pass, 10)] : passNumbers;
 
         const summary = {
@@ -135,12 +168,12 @@ class Orchestrator {
         for (const passNum of passesToRun) {
             const passComputations = passes[passNum] || [];
             this._log('INFO', `Executing Pass ${passNum}: ${passComputations.length} computations`);
-            
+
             // Computations in the same pass run in parallel here
             await Promise.all(passComputations.map(async (entry) => {
                 try {
                     const res = await this._executeComputation(entry, date, { ...options, dryRun, entities });
-                    
+
                     if (summary[res.status]) {
                         summary[res.status].push(res);
                         summary.summary[res.status]++;
@@ -160,17 +193,17 @@ class Orchestrator {
                 }
             }));
         }
-        
+
         this._printExecutionSummary(summary);
         return summary;
     }
 
     async runSingle(entry, dateStr, options = {}) {
         if (!this.manifest) await this.initialize();
-        return this._executeComputation(entry, dateStr, { 
+        return this._executeComputation(entry, dateStr, {
             ...options,
-            dryRun: options.dryRun || false, 
-            entities: options.entityIds 
+            dryRun: options.dryRun || false,
+            entities: options.entityIds
         });
     }
 
@@ -182,6 +215,7 @@ class Orchestrator {
         const { name } = entry;
         const forceEntities = options.entities;
 
+        // 1. Analyze Status (Skip if done/cached, unless forced)
         if (!forceEntities) {
             const decision = await this._analyzeEntry(entry, dateStr);
             const isSkippedOrCached = decision.type === 'skipped' || decision.type === 'cached';
@@ -198,15 +232,20 @@ class Orchestrator {
         this._log('INFO', `Running ${name} (Type: ${entry.type})...`);
         const startTime = Date.now();
 
+        // 2. Load Dependencies (Crucial for Lineage, Locking, and Upstream Checks)
         const { depResults, depResultHashes } = await this._loadDependencies(entry, dateStr);
-        
+
+        // =====================================================================
+        // STANDARD JS COMPUTATION (ETL) ONLY
+        // =====================================================================
+
         let previousResult = null;
         if (entry.isHistorical) {
             previousResult = await this.stateRepository.getResult(this._subtractDay(dateStr), name);
         }
 
         let stats = { count: 0, hash: null, skipped: false };
-        
+
         try {
             if (entry.type === 'per-entity' && !forceEntities) {
                 stats = await this._executeStreaming(entry, dateStr, depResults, previousResult, options);
@@ -218,7 +257,7 @@ class Orchestrator {
             if (!options.dryRun) {
                 const wallClockDuration = Date.now() - startTime;
                 const profilerStats = this.profilerMiddleware.profiler.getAndClearStats(entry.name);
-                
+
                 // If we have granular stats, include them. If not (e.g. skipped batches), use defaults.
                 const report = {
                     runId: this.config.testMode?.runId || 'production',
@@ -237,7 +276,7 @@ class Orchestrator {
 
                 // Async save (don't block pipeline)
                 this.storageManager.savePerformanceReport(report).catch(e => {
-                     this._log('WARN', `Failed to save perf report for ${name}: ${e.message}`);
+                    this._log('WARN', `Failed to save perf report for ${name}: ${e.message}`);
                 });
             }
 
@@ -256,8 +295,6 @@ class Orchestrator {
                 await this.lineageMiddleware.flush();
 
                 // Trigger dependency-driven cascading for downstream computations.
-                // This will enqueue Cloud Tasks for any dependents whose full
-                // dependency set has completed for the given date.
                 try {
                     await this._scheduleDependents(entry, dateStr);
                 } catch (cascadeError) {
@@ -278,8 +315,8 @@ class Orchestrator {
                 entityCount: stats.count || 0,
                 status: 'failed',
                 metrics: { error: e.message }
-            }).catch(() => {});
-            
+            }).catch(() => { });
+
             throw e;
         }
     }
@@ -287,11 +324,11 @@ class Orchestrator {
     async _executeStreaming(entry, dateStr, depResults, previousResult, options) {
         const checkpointer = new Checkpointer(this.config, this.storageManager);
         let cp = null;
-        
+
         try {
             if (!options.dryRun) {
                 const forceLock = options.force || this.config.bypassLocks || process.env.NODE_ENV === 'test';
-                cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0, entry.hash, forceLock); 
+                cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0, entry.hash, forceLock);
 
                 if (cp && cp.isLocked) throw new Error(`⚠️ Computation ${entry.name} is currently LOCKED.`);
                 if (cp && cp.skipped) {
@@ -304,7 +341,7 @@ class Orchestrator {
 
             // DECISION: Use remote workers or local execution?
             const useRemote = this._shouldUseRemoteWorkers(entry, options);
-            
+
             if (useRemote) {
                 this._log('INFO', `Using REMOTE worker pool for ${entry.name}`);
                 return await this._executeStreamingRemote(entry, dateStr, depResults, previousResult, options, checkpointer, cp);
@@ -312,6 +349,7 @@ class Orchestrator {
 
             // LOCAL EXECUTION PATH (Original Logic)
             const driverTable = this._getDriverTable(entry.requires);
+            this._log('INFO', `[DRIVER PROOF] Computation '${entry.name}' is being driven by table: '${driverTable}'`);
             const driverEntityField = this.config.tables[driverTable]?.entityField;
             const { batchRequires, globalRequires } = this._splitRequirements(entry.requires, driverTable);
 
@@ -322,7 +360,7 @@ class Orchestrator {
 
             const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
             const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
-            
+
             const rollingHash = crypto.createHash('sha256');
             let totalCount = 0;
             let batchIndex = 0;
@@ -336,41 +374,41 @@ class Orchestrator {
 
                 const { data: batchLocalData, entityIds } = batch;
                 const combinedData = { ...batchLocalData, ...globalData };
-                
+
                 // STRICT FIX: Prefetch dependencies for the batch.
                 const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
-                
-                const { rules } = this.ruleInjector.createContext(); 
+
+                const { rules } = this.ruleInjector.createContext();
                 const batchResults = {};
-                
+
                 await Promise.all(entityIds.map(entityId => limit(async () => {
                     const instance = new entry.class();
                     const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
-                    
+
                     const context = {
                         computation: entry, date: dateStr, entityId, data: entityData,
-                        
+
                         // STRICT FIX: No fallback to _lazyLoadDependency.
                         getDependency: (depName, targetId) => {
                             const id = targetId || entityId;
-                            
+
                             // 1. Look in Batch-Prefetched Dependencies (Priority)
                             if (batchDeps[depName] && batchDeps[depName].has(id)) {
                                 return batchDeps[depName].get(id);
                             }
-                            
+
                             // 2. Look in Global/Preloaded Dependencies
                             if (depResults[depName]) {
                                 if (depResults[depName][id] !== undefined) return depResults[depName][id];
                             }
-                            
+
                             // 3. STRICT MODE: Throw Error
                             throw new Error(
                                 `[Strict Dependency] Dependency '${depName}' (ID: ${id}) not found in batch context. ` +
                                 `Ensure '${depName}' is listed in ${entry.name}.getConfig().dependencies.`
                             );
                         },
-                        
+
                         previousResult, rules, references: this.referenceDataCache,
                         config: this.config, dataFetcher: this.dataFetcher
                     };
@@ -383,7 +421,7 @@ class Orchestrator {
                 })));
 
                 if (!options.dryRun) {
-                    await this.storageManager.commitResults(dateStr, entry, batchResults, {}); 
+                    await this.storageManager.commitResults(dateStr, entry, batchResults, {});
                     const lastId = entityIds[entityIds.length - 1];
                     await checkpointer.markBatchComplete(dateStr, entry.name, cp?.id, batchIndex, batchSize, lastId);
                 }
@@ -399,8 +437,15 @@ class Orchestrator {
 
             // FIX: Return valid object shape including skipped: false
             return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16), skipped: false };
-            
+
         } catch (error) {
+            console.error('________________________________________________________________');
+            console.error('🛑 CRITICAL COMPUTATION CRASH DETECTED');
+            console.error(`📍 Computation: ${entry.name}`);
+            console.error(`💥 Error Message: ${error.message}`);
+            console.error(`📚 Stack Trace:\n${error.stack}`);
+            console.error('________________________________________________________________');
+
             if (cp && cp.id) {
                 this._log('ERROR', `Streaming failed, marking checkpoint ${cp.id} as failed.`);
                 await this.storageManager.failCheckpoint(cp.id, error.message);
@@ -427,22 +472,22 @@ class Orchestrator {
         if (options.useWorkerPool === false) {
             return false;
         }
-        
+
         if (!this.remoteRunner) return false;
         if (options.forceLocal) return false;
-        
+
         const poolConfig = this.config.workerPool || {};
-        
-        if (poolConfig.excludeComputations?.includes(entry.name) || 
+
+        if (poolConfig.excludeComputations?.includes(entry.name) ||
             poolConfig.excludeComputations?.includes(entry.originalName)) {
             return false;
         }
-        
-        if (poolConfig.forceOffloadComputations?.includes(entry.name) || 
+
+        if (poolConfig.forceOffloadComputations?.includes(entry.name) ||
             poolConfig.forceOffloadComputations?.includes(entry.originalName)) {
             return true;
         }
-        
+
         if (entry.type !== 'per-entity') return false;
         return true;
     }
@@ -474,7 +519,7 @@ class Orchestrator {
 
         const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
         const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
-        
+
         const rollingHash = crypto.createHash('sha256');
         let totalCount = 0;
         let totalErrors = 0;
@@ -489,10 +534,10 @@ class Orchestrator {
 
             const { data: batchLocalData, entityIds } = batch;
             const combinedData = { ...batchLocalData, ...globalData };
-            
+
             // Prefetch dependencies for this batch
             const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
-            
+
             // Convert Map to Object for serialization
             const serializedDeps = {};
             for (const [depName, depMap] of Object.entries(batchDeps)) {
@@ -502,7 +547,7 @@ class Orchestrator {
                     serializedDeps[depName] = depMap;
                 }
             }
-            
+
             // Merge with preloaded deps
             const mergedDeps = { ...depResults, ...serializedDeps };
 
@@ -516,18 +561,18 @@ class Orchestrator {
             // INVOKE REMOTE WORKERS
             this._log('INFO', `[Remote] Processing batch ${batchIndex}: ${entityIds.length} entities`);
             const { results: batchResults, errors } = await this.remoteRunner.runBatch(
-                entry, 
-                dateStr, 
-                baseContext, 
-                entityIds, 
-                entityDataMap, 
+                entry,
+                dateStr,
+                baseContext,
+                entityIds,
+                entityDataMap,
                 mergedDeps
             );
 
             if (errors.length > 0) {
                 this._log('WARN', `[Remote] Batch ${batchIndex}: ${errors.length} entities failed`);
                 totalErrors += errors.length;
-                
+
                 errors.slice(0, 3).forEach(e => {
                     this._log('DEBUG', `  - ${e.entityId}: ${e.error}`);
                 });
@@ -569,7 +614,7 @@ class Orchestrator {
         const driverTable = entry.type === 'per-entity' ? this._getDriverTable(entry.requires) : null;
         const driverEntityField = driverTable ? this.config.tables[driverTable]?.entityField : null;
         const instance = new entry.class();
-        
+
         const context = {
             computation: entry, date: dateStr, data,
             getDependency: (dep, ent) => this._lazyLoadDependency(dateStr, dep, ent, depResults),
@@ -578,7 +623,7 @@ class Orchestrator {
         };
 
         let results = {};
-        
+
         if (entry.type === 'per-entity') {
             const ids = forceEntities || this._extractEntityIds(data);
             const limit = pLimit(DEFAULT_CONCURRENCY);
@@ -592,17 +637,19 @@ class Orchestrator {
         }
 
         const finalHash = this._hashResults(results);
-        
-        if (!options.dryRun && !forceEntities) {
+
+        if (!options.dryRun && (!forceEntities || options.allowPartialCommit)) {
             const currentStatus = await this.stateRepository.getDailyStatus(dateStr);
             const status = currentStatus.get(entry.name.toLowerCase());
-            
+
             if (!options.force && status && status.resultHash === finalHash) {
                 return { count: Object.keys(results || {}).length, hash: finalHash, skipped: true };
             }
-            
+
             await this.storageManager.commitResults(dateStr, entry, results, {});
-            await this.storageManager.finalizeResults(dateStr, entry);
+            if (!forceEntities) {
+                await this.storageManager.finalizeResults(dateStr, entry);
+            }
         }
 
         // FIX: Return valid object shape including skipped: false
@@ -617,7 +664,7 @@ class Orchestrator {
         console.log(`✅ Completed: ${summary.summary.completed}`);
         console.log(`❌ Errors:    ${summary.summary.errors}`);
         console.log(`⏭️  Skipped:   ${summary.summary.skipped + summary.summary.blocked + summary.summary.impossible}`);
-        
+
         const skippedItems = [...summary.skipped, ...summary.blocked, ...summary.impossible];
         if (skippedItems.length > 0) {
             console.log('\n--- Details (Why did it skip?) ---');
@@ -636,11 +683,27 @@ class Orchestrator {
     }
 
     _getDriverTable(requires) {
+        // PASS 1: Prioritize tables with a DATE field (Time-Partitioned)
         for (const name of Object.keys(requires)) {
+            const spec = requires[name];
+            // Metrics cannot be drivers
+            if (spec.type === 'metric') continue;
+
+            const conf = this.config.tables[name];
+            // If it has BOTH, it's a perfect driver (e.g. portfolio_snapshots)
+            if (conf && conf.entityField && conf.dateField) return name;
+        }
+
+        // PASS 2: Fallback to any entity table (e.g. Static Master List)
+        for (const name of Object.keys(requires)) {
+            const spec = requires[name];
+            if (spec.type === 'metric') continue;
+
             const conf = this.config.tables[name];
             if (conf && conf.entityField) return name;
         }
-        return null; 
+
+        return null;
     }
 
     _splitRequirements(requires, driverTable) {
@@ -673,7 +736,7 @@ class Orchestrator {
         for (const dep of entry.dependencies) {
             const stat = dailyStatus.get(dep);
             if (stat?.resultHash) depResultHashes[dep] = stat.resultHash;
-            if (stat?.entityCount > 50000) depResults[dep] = null; 
+            if (stat?.entityCount > 50000) depResults[dep] = null;
             else depResults[dep] = await this.stateRepository.getResult(dateStr, dep);
         }
         if (entry.conditionalDependencies) {
@@ -702,10 +765,6 @@ class Orchestrator {
         return prefetched;
     }
 
-    /**
-     * Build a reverse dependency index so that when a computation completes
-     * we can quickly find all computations that depend on it.
-     */
     _buildDependentsIndex() {
         this.dependentsByName = new Map();
         if (!this.manifest) return;
@@ -720,42 +779,91 @@ class Orchestrator {
         }
     }
 
+    // =========================================================================
+    // TASK SCHEDULING & CASCADING
+    // =========================================================================
+
     /**
-     * Schedule dependent computations via Cloud Tasks after a computation
-     * has successfully completed for a given date.
-     *
-     * The scheduler is responsible only for root / pass-1 computations.
-     * All downstream work is triggered here with a configurable time gap
-     * once ALL dependencies of a computation have completed.
+     * Schedules a task on Cloud Tasks (Wraps API call)
      */
-    async _scheduleDependents(entry, dateStr) {
-        const dependents = this.dependentsByName.get(entry.name);
-        if (!dependents || dependents.length === 0) return;
-
+    async _scheduleCloudTask(computationName, dateStr, source, delayMinutes = 0) {
         const cloudTasksConfig = this.config.cloudTasks;
-        if (!cloudTasksConfig) {
-            this._log('WARN', 'cloudTasks config missing; skipping dependent scheduling');
+
+        // Mock fallback for local/test environments without cloud config
+        if (!cloudTasksConfig || !cloudTasksConfig.queueName) {
+            if (process.env.NODE_ENV !== 'production') {
+                this._log('INFO', `[Mock Task] Scheduled ${computationName} for ${dateStr} (Source: ${source})`);
+                return;
+            }
             return;
         }
 
         const { projectId, location, queueName, dispatcherUrl, serviceAccountEmail } = cloudTasksConfig;
+
         if (!projectId || !location || !queueName || !dispatcherUrl || !serviceAccountEmail) {
-            this._log('WARN', 'cloudTasks configuration incomplete; skipping dependent scheduling');
+            this._log('WARN', 'cloudTasks configuration incomplete; skipping scheduling');
             return;
         }
 
-        const dependencyGapMinutes = this.config.scheduling?.dependencyGapMinutes ?? 5;
         const queuePath = this.cloudTasksClient.queuePath(projectId, location, queueName);
+        const scheduleTime = new Date(Date.now() + delayMinutes * 60 * 1000);
+
+        // Unique ID prevents duplicate tasks for the same logical event
+        // Note: For backfills, we might want to allow overwrite, but unique IDs help tracing
+        const taskName = `${queuePath}/tasks/${computationName}-${dateStr}-${Date.now()}`;
+
+        const payload = {
+            computationName: computationName,
+            targetDate: dateStr,
+            source: source,
+            scheduledAt: scheduleTime.toISOString()
+        };
+
+        const task = {
+            httpRequest: {
+                httpMethod: 'POST',
+                url: dispatcherUrl,
+                headers: { 'Content-Type': 'application/json' },
+                body: Buffer.from(JSON.stringify(payload)).toString('base64'),
+                oidcToken: { serviceAccountEmail, audience: dispatcherUrl }
+            },
+            scheduleTime: {
+                seconds: Math.floor(scheduleTime.getTime() / 1000),
+                nanos: 0
+            },
+            name: taskName
+        };
+
+        try {
+            await this.cloudTasksClient.createTask({ parent: queuePath, task });
+            if (source === 'dependency') {
+                this._log('INFO', `Triggered dependent ${computationName} for ${dateStr}`);
+            }
+        } catch (e) {
+            if (e.code === 6) { // ALREADY_EXISTS
+                this._log('INFO', `Task for ${computationName} (${dateStr}) already exists.`);
+            } else {
+                this._log('WARN', `Failed to schedule task ${computationName}: ${e.message}`);
+            }
+        }
+    }
 
-        // Use the latest in-memory status for this date so we can see the
-        // just-updated computation plus any earlier ones.
+    /**
+     * Checks all dependents of the just-completed computation.
+     * If they are ready (all other dependencies met), schedules them.
+     */
+    async _scheduleDependents(entry, dateStr) {
+        const dependents = this.dependentsByName.get(entry.name);
+        if (!dependents || dependents.length === 0) return;
+
+        const dependencyGapMinutes = this.config.scheduling?.dependencyGapMinutes ?? 5;
         const dailyStatus = await this.stateRepository.getDailyStatus(dateStr);
 
         for (const depEntry of dependents) {
-            // Compute the latest completion time across all of this computation's dependencies.
             let latestDependencyTime = null;
             let missingDependency = false;
 
+            // Check if *all* dependencies of this dependent are ready
             for (const depName of depEntry.dependencies || []) {
                 const depStatus = dailyStatus.get(depName);
                 if (!depStatus || !depStatus.updatedAt) {
@@ -771,78 +879,43 @@ class Orchestrator {
                 }
             }
 
-            // If any dependency hasn't completed yet, we will schedule this
-            // dependent when that dependency finishes instead.
-            if (missingDependency || !latestDependencyTime) {
+            if (missingDependency) {
+                // Dependent is not ready yet (waiting on other inputs)
                 continue;
             }
 
-            const scheduleTime = new Date(latestDependencyTime.getTime() + dependencyGapMinutes * 60 * 1000);
-
-            const payload = {
-                computationName: depEntry.originalName,
-                targetDate: dateStr,
-                source: 'dependency',
-                scheduledAt: scheduleTime.toISOString()
-            };
-
-            const taskName = `${queuePath}/tasks/${depEntry.name}-${dateStr}`;
-
-            const task = {
-                httpRequest: {
-                    httpMethod: 'POST',
-                    url: dispatcherUrl,
-                    headers: { 'Content-Type': 'application/json' },
-                    body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: {
-                        serviceAccountEmail,
-                        audience: dispatcherUrl
-                    }
-                },
-                scheduleTime: {
-                    seconds: Math.floor(scheduleTime.getTime() / 1000),
-                    nanos: 0
-                },
-                name: taskName
-            };
-
-            try {
-                await this.cloudTasksClient.createTask({
-                    parent: queuePath,
-                    task
-                });
-
-                this._log('INFO', `Scheduled dependent ${depEntry.name} for ${dateStr} at ${scheduleTime.toISOString()}`);
-            } catch (e) {
-                // Code 6: ALREADY_EXISTS – task already scheduled, this is fine (idempotent)
-                if (e.code === 6) {
-                    this._log('INFO', `Dependent ${depEntry.name} for ${dateStr} already scheduled (duplicate task ignored)`);
-                    continue;
-                }
-
-                this._log('WARN', `Failed to schedule dependent ${depEntry.name}: ${e.message}`);
-            }
+            // Schedule via the unified helper
+            await this._scheduleCloudTask(depEntry.originalName, dateStr, 'dependency', dependencyGapMinutes);
         }
     }
 
     async _lazyLoadDependency(dateStr, depName, entityId, preloaded) {
         if (preloaded[depName] && !entityId) return preloaded[depName];
         if (preloaded[depName] && entityId) return preloaded[depName][entityId];
-        
-        // WARN: This is the slow path that we removed from Streaming
         this._log('WARN', `LAZY LOAD: Fetching single entity '${entityId}' for '${depName}'. This is slow.`);
-        
         if (entityId) return this.stateRepository.getEntityResult(dateStr, depName, entityId);
         return this.stateRepository.getResult(dateStr, depName);
     }
 
     async _loadReferenceData() {
         if (!this.config.referenceData) return;
-        await Promise.all(this.config.referenceData.map(async (table) => {
+
+        await Promise.all(this.config.referenceData.map(async (refEntry) => {
             try {
-                const data = await this.dataFetcher.fetch({ table, targetDate: new Date().toISOString().slice(0, 10), mandatory: false });
+                // FIX: Support both string (legacy) and object (with fields) formats
+                const table = typeof refEntry === 'string' ? refEntry : refEntry.table;
+                const fields = typeof refEntry === 'string' ? null : refEntry.fields;
+
+                const data = await this.dataFetcher.fetch({
+                    table,
+                    fields, // Pass specific fields to prevent "LAZY SELECT BLOCKED"
+                    targetDate: new Date().toISOString().slice(0, 10),
+                    mandatory: false
+                });
                 this.referenceDataCache[table] = data || {};
-            } catch (e) { this._log('WARN', `Failed to load Ref Data ${table}: ${e.message}`); }
+            } catch (e) {
+                this._log('WARN', `Failed to load Ref Data ${refEntry.table || refEntry}: ${e.message}`);
+            }
         }));
     }
 
@@ -860,9 +933,9 @@ class Orchestrator {
         Object.entries(data).forEach(([tbl, d]) => {
             const conf = this.config.tables[tbl] || {};
             if (conf.entityField === driverEntityField && d && !Array.isArray(d)) {
-                 out[tbl] = d[id] || null;
+                out[tbl] = d[id] || null;
             } else {
-                 out[tbl] = d; 
+                out[tbl] = d;
             }
         });
         return out;
@@ -878,12 +951,12 @@ class Orchestrator {
         d.setUTCDate(d.getUTCDate() - 1);
         return d.toISOString().slice(0, 10);
     }
-    _getAllTables() { 
-        const s = new Set(); 
-        if (this.manifest) this.manifest.forEach(e => Object.keys(e.requires).forEach(t => s.add(t))); 
-        return Array.from(s); 
+    _getAllTables() {
+        const s = new Set();
+        if (this.manifest) this.manifest.forEach(e => Object.keys(e.requires).forEach(t => s.add(t)));
+        return Array.from(s);
     }
     _log(l, m) { this.logger.log(l, `[Orchestrator] ${m}`); }
 }
 
-module.exports = { Orchestrator };
+module.exports = { Orchestrator };