bulltrackers-module 1.0.735 → 1.0.736

package/functions/computation-system-v2/framework/execution/Orchestrator.js

@@ -5,6 +5,10 @@
  * 2. Data Provisioning (Fetching Data, Loading Dependencies, Reference Data)
  * 3. Execution Strategy (Streaming vs. In-Memory)
  * 4. Delegation (Hands off actual 'work' to TaskRunner + Middleware)
+ * * * UPDATE: Added Execution Summary logging to debug Skipped/Blocked/Impossible tasks.
+ * * * UPDATE: Includes Global vs Batch Data Split to fix "Identity Crisis".
+ * * * UPDATE: Implemented FORCE logic to bypass "up-to-date" checks for testing.
+ * * * UPDATE: Aggregates performance reporting to prevent log spam.
  */
 
 const crypto = require('crypto');
@@ -26,6 +30,7 @@ const { Checkpointer } = require('../resilience/Checkpointer');
 
 // Execution Components
 const { TaskRunner } = require('./TaskRunner');
+const { RemoteTaskRunner } = require('./RemoteTaskRunner');
 const { ProfilerMiddleware } = require('./middleware/ProfilerMiddleware');
 const { CostTrackerMiddleware } = require('./middleware/CostTrackerMiddleware');
 const { LineageMiddleware } = require('./middleware/LineageMiddleware');
@@ -51,8 +56,9 @@ class Orchestrator {
         this.ruleInjector = new RuleInjector(rulesRegistry);
 
         // 3. Initialize Execution Stack (Middleware)
-        const profiler = new ProfilerMiddleware(config);
-        profiler.setStorage(this.storageManager);
+        // Keep reference to profiler middleware to access stats later
+        this.profilerMiddleware = new ProfilerMiddleware(config);
+        this.profilerMiddleware.setStorage(this.storageManager);
         
         this.lineageMiddleware = new LineageMiddleware(config);
         const costTracker = new CostTrackerMiddleware(config);
@@ -61,9 +67,15 @@ class Orchestrator {
         this.runner = new TaskRunner([
             costTracker,
             this.lineageMiddleware,
-            profiler
+            this.profilerMiddleware
         ]);
         
+        // 4. Initialize Remote Task Runner (Worker Pool)
+        // Only create if worker pool is enabled in config
+        this.remoteRunner = config.workerPool?.enabled 
+            ? new RemoteTaskRunner(config, this.logger)
+            : null;
+        
         // State
         this.manifest = null;
         this.runAnalyzer = null;
@@ -72,49 +84,29 @@ class Orchestrator {
 
     async initialize() {
         this._log('INFO', 'Initializing Orchestrator...');
-        
-        // Build Manifest
         this.manifest = this.manifestBuilder.build(this.config.computations || []);
-        
-        // Initialize Analyzer
         this.runAnalyzer = new RunAnalyzer(this.manifest, this.dataFetcher, this.logger);
-        
-        // Warm Schema Cache
         await this.schemaRegistry.warmCache(this._getAllTables());
-        
-        // Load Reference Data (e.g. sectors, holidays)
         await this._loadReferenceData();
-        
         this._log('INFO', `Initialized with ${this.manifest.length} computations`);
     }
 
-    /**
-     * Analyze what needs to run for a given date.
-     */
     async analyze(options) {
         const { date } = options;
         if (!this.manifest) await this.initialize();
-        
         const dailyStatus = await this.stateRepository.getDailyStatus(date);
         const prevStatus = await this.stateRepository.getDailyStatus(this._subtractDay(date));
-        
         const report = await this.runAnalyzer.analyze(date, dailyStatus, prevStatus);
-        
-        // Compatibility: Merge reRuns into runnable
         report.runnable = [...report.runnable, ...report.reRuns];
         return report;
     }
 
-    /**
-     * Main Execution Loop
-     */
     async execute(options) {
         const { date, pass = null, computation = null, dryRun = false, entities = null } = options;
         if (!this.manifest) await this.initialize();
         
         this._log('INFO', `Starting execution for ${date}...`);
 
-        // 1. Filter Manifest
         let toRun = this.manifest;
         if (computation) {
             const norm = computation.toLowerCase().replace(/[^a-z0-9]/g, '');
@@ -122,7 +114,6 @@ class Orchestrator {
             if (!toRun.length) throw new Error(`Computation not found: ${computation}`);
         }
 
-        // 2. Group by Pass
         const passes = this.manifestBuilder.groupByPass(toRun);
         const passNumbers = Object.keys(passes).map(Number).sort((a,b) => a-b);
         const passesToRun = pass ? [parseInt(pass, 10)] : passNumbers;
@@ -133,20 +124,27 @@ class Orchestrator {
             completed: [], skipped: [], blocked: [], impossible: [], errors: []
         };
 
-        // 3. Execute Passes
         for (const passNum of passesToRun) {
             const passComputations = passes[passNum] || [];
             this._log('INFO', `Executing Pass ${passNum}: ${passComputations.length} computations`);
             
-            // Note: In a strict DAG, items in the same pass are parallelizable.
-            // We use Promise.all to run them concurrently.
+            // Computations in the same pass run in parallel here
             await Promise.all(passComputations.map(async (entry) => {
                 try {
-                    // Pass specific options like "entities" (force entities) down
-                    const res = await this._executeComputation(entry, date, { dryRun, entities });
+                    const res = await this._executeComputation(entry, date, { ...options, dryRun, entities });
                     
-                    summary[res.status].push(res);
-                    summary.summary[res.status]++;
+                    if (summary[res.status]) {
+                        summary[res.status].push(res);
+                        summary.summary[res.status]++;
+                    } else {
+                        summary.errors.push({ name: entry.name, error: `Unknown status: ${res.status}` });
+                        summary.summary.errors++;
+                    }
+
+                    if (res.status === 'skipped' || res.status === 'blocked' || res.status === 'impossible') {
+                        this._log('WARN', `Skipped ${entry.name}: [${res.status.toUpperCase()}] ${res.reason}`);
+                    }
+
                 } catch (e) {
                     summary.errors.push({ name: entry.name, error: e.message });
                     summary.summary.errors++;
@@ -155,12 +153,14 @@ class Orchestrator {
             }));
         }
         
+        this._printExecutionSummary(summary);
         return summary;
     }
 
     async runSingle(entry, dateStr, options = {}) {
         if (!this.manifest) await this.initialize();
         return this._executeComputation(entry, dateStr, { 
+            ...options,
             dryRun: options.dryRun || false, 
             entities: options.entityIds 
         });
@@ -174,18 +174,22 @@ class Orchestrator {
         const { name } = entry;
         const forceEntities = options.entities;
 
-        // 1. Logic Check (Skip if unnecessary)
         if (!forceEntities) {
             const decision = await this._analyzeEntry(entry, dateStr);
-            if (decision.type !== 'runnable' && decision.type !== 'reRuns') {
+            const isSkippedOrCached = decision.type === 'skipped' || decision.type === 'cached';
+            const shouldForce = options.force && isSkippedOrCached;
+
+            if (!shouldForce && decision.type !== 'runnable' && decision.type !== 'reRuns') {
                 return { name, status: decision.type, reason: decision.payload.reason };
             }
+            if (shouldForce) {
+                this._log('INFO', `Force Run Enabled: Ignoring '${decision.type}' status for ${name}`);
+            }
         }
 
         this._log('INFO', `Running ${name} (Type: ${entry.type})...`);
         const startTime = Date.now();
 
-        // 2. Load Dependencies & Previous Results
         const { depResults, depResultHashes } = await this._loadDependencies(entry, dateStr);
         
         let previousResult = null;
@@ -193,113 +197,151 @@ class Orchestrator {
             previousResult = await this.stateRepository.getResult(this._subtractDay(dateStr), name);
         }
 
-        // 3. Select Execution Strategy
         let stats = { count: 0, hash: null, skipped: false };
         
-        if (entry.type === 'per-entity' && !forceEntities) {
-            // STRATEGY A: Streaming (Low Memory, Checkpointing)
-            stats = await this._executeStreaming(entry, dateStr, depResults, previousResult, options);
-        } else {
-            // STRATEGY B: In-Memory (Global, Aggregates, or Forced Entities)
-            stats = await this._executeGlobal(entry, dateStr, depResults, previousResult, options, forceEntities);
-        }
+        try {
+            if (entry.type === 'per-entity' && !forceEntities) {
+                stats = await this._executeStreaming(entry, dateStr, depResults, previousResult, options);
+            } else {
+                stats = await this._executeGlobal(entry, dateStr, depResults, previousResult, options, forceEntities);
+            }
 
-        if (stats.skipped) {
-            return { name, status: 'skipped', reason: 'Results unchanged', duration: Date.now() - startTime };
-        }
+            // REPORTING: Save Aggregated Performance Report
+            if (!options.dryRun) {
+                const wallClockDuration = Date.now() - startTime;
+                const profilerStats = this.profilerMiddleware.profiler.getAndClearStats(entry.name);
+                
+                // If we have granular stats, include them. If not (e.g. skipped batches), use defaults.
+                const report = {
+                    runId: this.config.testMode?.runId || 'production',
+                    computationName: entry.name,
+                    date: dateStr,
+                    durationMs: wallClockDuration,
+                    entityCount: stats.count,
+                    status: stats.skipped ? 'skipped' : 'completed',
+                    metrics: profilerStats ? {
+                        avgEntityDuration: profilerStats.avgDuration,
+                        p95EntityDuration: profilerStats.p95Duration,
+                        avgMemoryDelta: profilerStats.avgMemoryDelta,
+                        throughput: (stats.count / (wallClockDuration / 1000)).toFixed(2) + ' ent/sec'
+                    } : {}
+                };
 
-        // 4. Update State (If real run)
-        if (!options.dryRun) {
-            await this.stateRepository.updateStatusCache(dateStr, name, {
-                hash: entry.hash,
-                resultHash: stats.hash,
-                dependencyResultHashes: depResultHashes,
-                entityCount: stats.count
-            });
+                // Async save (don't block pipeline)
+                this.storageManager.savePerformanceReport(report).catch(e => {
+                     this._log('WARN', `Failed to save perf report for ${name}: ${e.message}`);
+                });
+            }
+
+            if (stats.skipped) {
+                return { name, status: 'skipped', reason: 'Results unchanged or Dead Letter', duration: Date.now() - startTime };
+            }
+
+            if (!options.dryRun) {
+                await this.stateRepository.updateStatusCache(dateStr, name, {
+                    hash: entry.hash,
+                    resultHash: stats.hash,
+                    dependencyResultHashes: depResultHashes,
+                    entityCount: stats.count
+                });
+                await this.lineageMiddleware.flush();
+            }
+
+            return { name, status: 'completed', duration: Date.now() - startTime, resultCount: stats.count };
+
+        } catch (e) {
+            // Also try to save failure report
+            const wallClockDuration = Date.now() - startTime;
+            this.storageManager.savePerformanceReport({
+                runId: this.config.testMode?.runId || 'production',
+                computationName: entry.name,
+                date: dateStr,
+                durationMs: wallClockDuration,
+                entityCount: stats.count || 0,
+                status: 'failed',
+                metrics: { error: e.message }
+            }).catch(() => {});
             
-            // Flush any buffered lineage logs
-            await this.lineageMiddleware.flush();
+            throw e;
         }
-
-        return { name, status: 'completed', duration: Date.now() - startTime, resultCount: stats.count };
     }
 
-    // --- STRATEGY A: STREAMING ---
     async _executeStreaming(entry, dateStr, depResults, previousResult, options) {
-        // 1. Setup Checkpoint
         const checkpointer = new Checkpointer(this.config, this.storageManager);
         let cp = null;
         if (!options.dryRun) {
-            cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0); // 0 = unknown total
+            const forceLock = options.force || this.config.bypassLocks || process.env.NODE_ENV === 'test';
+            cp = await checkpointer.initCheckpoint(dateStr, entry.name, 0, entry.hash, forceLock); 
+
+            if (cp && cp.isLocked) throw new Error(`⚠️ Computation ${entry.name} is currently LOCKED.`);
+            if (cp && cp.skipped) {
+                this._log('INFO', `⏭️ Skipping ${entry.name}: ${cp.reason}`);
+                return { count: 0, hash: 'skipped_dead_letter', skipped: true };
+            }
             if (cp?.isCompleted) return { count: 0, hash: 'cached', skipped: true };
             if (cp?.isResumed) this._log('INFO', `Resuming ${entry.name} from checkpoint...`);
         }
 
-        // 2. Initialize Stream
+        // DECISION: Use remote workers or local execution?
+        const useRemote = this._shouldUseRemoteWorkers(entry, options);
+        
+        if (useRemote) {
+            this._log('INFO', `Using REMOTE worker pool for ${entry.name}`);
+            return this._executeStreamingRemote(entry, dateStr, depResults, previousResult, options, checkpointer, cp);
+        }
+
+        // LOCAL EXECUTION PATH (Original Logic)
+        const driverTable = this._getDriverTable(entry.requires);
+        const driverEntityField = this.config.tables[driverTable]?.entityField;
+        const { batchRequires, globalRequires } = this._splitRequirements(entry.requires, driverTable);
+
+        let globalData = {};
+        if (Object.keys(globalRequires).length > 0) {
+            globalData = await this.dataFetcher.fetchForComputation(globalRequires, dateStr);
+        }
+
         const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
-        const batchStream = this.dataFetcher.fetchComputationBatched(entry.requires, dateStr, batchSize);
+        const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
         
         const rollingHash = crypto.createHash('sha256');
         let totalCount = 0;
         let batchIndex = 0;
-        
         const concurrency = this.config.execution?.entityConcurrency || DEFAULT_CONCURRENCY;
         const limit = pLimit(concurrency);
 
-        // 3. Iterate Batches
         for await (const batch of batchStream) {
-            // Resume Logic: Skip completed batches
-            if (cp && cp.completedBatches.has(batchIndex)) {
-                batchIndex++;
-                continue;
+            if (cp && cp.completedBatches && cp.completedBatches.has(batchIndex)) {
+                batchIndex++; continue;
             }
 
-            const { data: batchData, entityIds } = batch;
-
-            // 4. PREFETCH DEPENDENCIES
+            const { data: batchLocalData, entityIds } = batch;
+            const combinedData = { ...batchLocalData, ...globalData };
             const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
-
-            // 5. Dynamic Context Injection
-            const { rules } = this.ruleInjector.createContext(); // Used is implicit via Proxy
-
-            // 6. Execute Batch Concurrently
+            const { rules } = this.ruleInjector.createContext(); 
             const batchResults = {};
             
             await Promise.all(entityIds.map(entityId => limit(async () => {
                 const instance = new entry.class();
-                const entityData = this._filterDataForEntity(batchData, entityId);
-                
+                const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
                 const context = {
-                    computation: entry,
-                    date: dateStr,
-                    entityId,
-                    data: entityData,
-                    
-                    // Dependency Injector
+                    computation: entry, date: dateStr, entityId, data: entityData,
                     getDependency: (depName, targetId) => {
                         if (batchDeps[depName] && batchDeps[depName].has(targetId || entityId)) {
                             return batchDeps[depName].get(targetId || entityId);
                         }
                         return this._lazyLoadDependency(dateStr, depName, targetId || entityId, depResults);
                     },
-                    
-                    previousResult,
-                    rules,          
-                    references: this.referenceDataCache,
-                    config: this.config,
-                    dataFetcher: this.dataFetcher // <--- ADDED: Required by CostTrackerMiddleware
+                    previousResult, rules, references: this.referenceDataCache,
+                    config: this.config, dataFetcher: this.dataFetcher
                 };
 
-                // DELEGATE TO RUNNER
                 const result = await this.runner.run(instance, context);
-                
                 if (result !== undefined) {
                     batchResults[entityId] = result;
                     this._updateRollingHash(rollingHash, result);
                 }
             })));
 
-            // 7. Commit Batch
             if (!options.dryRun) {
                 await this.storageManager.commitResults(dateStr, entry, batchResults, {}); 
                 const lastId = entityIds[entityIds.length - 1];
@@ -310,44 +352,182 @@ class Orchestrator {
             batchIndex++;
         }
 
-        if (!options.dryRun && cp) await checkpointer.complete(dateStr, entry.name, cp.id);
+        if (!options.dryRun) {
+            await this.storageManager.finalizeResults(dateStr, entry);
+            if (cp) await checkpointer.complete(dateStr, entry.name, cp.id);
+        }
+
+        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16) };
+    }
+
+    /**
+     * Determine if a computation should use remote workers
+     */
+    _shouldUseRemoteWorkers(entry, options) {
+        // No remote runner configured
+        if (!this.remoteRunner) return false;
+        
+        // Force local execution via options
+        if (options.forceLocal) return false;
+        
+        const poolConfig = this.config.workerPool || {};
+        
+        // Exclusion list
+        if (poolConfig.excludeComputations?.includes(entry.name) || 
+            poolConfig.excludeComputations?.includes(entry.originalName)) {
+            return false;
+        }
+        
+        // Force list (override threshold)
+        if (poolConfig.forceOffloadComputations?.includes(entry.name) || 
+            poolConfig.forceOffloadComputations?.includes(entry.originalName)) {
+            return true;
+        }
+        
+        // Only per-entity computations can be offloaded
+        if (entry.type !== 'per-entity') return false;
+        
+        // Default: use remote if worker pool is enabled
+        return true;
+    }
+
+    /**
+     * Execute using remote worker pool
+     * Workers handle individual entities, Orchestrator handles batching and storage
+     */
+    async _executeStreamingRemote(entry, dateStr, depResults, previousResult, options, checkpointer, cp) {
+        const driverTable = this._getDriverTable(entry.requires);
+        const driverEntityField = this.config.tables[driverTable]?.entityField;
+        const { batchRequires, globalRequires } = this._splitRequirements(entry.requires, driverTable);
+
+        // Load global data once (shared across all entities)
+        let globalData = {};
+        if (Object.keys(globalRequires).length > 0) {
+            globalData = await this.dataFetcher.fetchForComputation(globalRequires, dateStr);
+        }
+
+        // Prepare base context (shared across all entities)
+        const baseContext = {
+            references: this.referenceDataCache,
+            config: {
+                // Only pass serializable config to workers
+                project: this.config.project,
+                tables: this.config.tables
+            }
+        };
+
+        const batchSize = this.config.execution?.batchSize || BATCH_SIZE;
+        const batchStream = this.dataFetcher.fetchComputationBatched(batchRequires, dateStr, batchSize);
+        
+        const rollingHash = crypto.createHash('sha256');
+        let totalCount = 0;
+        let totalErrors = 0;
+        let batchIndex = 0;
+
+        for await (const batch of batchStream) {
+            // Skip completed batches (checkpoint resume)
+            if (cp && cp.completedBatches && cp.completedBatches.has(batchIndex)) {
+                batchIndex++;
+                continue;
+            }
+
+            const { data: batchLocalData, entityIds } = batch;
+            const combinedData = { ...batchLocalData, ...globalData };
+            
+            // Prefetch dependencies for this batch
+            const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
+            
+            // Convert Map to Object for serialization
+            const serializedDeps = {};
+            for (const [depName, depMap] of Object.entries(batchDeps)) {
+                if (depMap instanceof Map) {
+                    serializedDeps[depName] = Object.fromEntries(depMap);
+                } else {
+                    serializedDeps[depName] = depMap;
+                }
+            }
+            
+            // Merge with preloaded deps
+            const mergedDeps = { ...depResults, ...serializedDeps };
+
+            // Build entity data map
+            const entityDataMap = new Map();
+            for (const entityId of entityIds) {
+                const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
+                entityDataMap.set(entityId, entityData);
+            }
+
+            // INVOKE REMOTE WORKERS
+            this._log('INFO', `[Remote] Processing batch ${batchIndex}: ${entityIds.length} entities`);
+            const { results: batchResults, errors } = await this.remoteRunner.runBatch(
+                entry, 
+                dateStr, 
+                baseContext, 
+                entityIds, 
+                entityDataMap, 
+                mergedDeps
+            );
+
+            if (errors.length > 0) {
+                this._log('WARN', `[Remote] Batch ${batchIndex}: ${errors.length} entities failed`);
+                totalErrors += errors.length;
+                
+                // Log first few errors for debugging
+                errors.slice(0, 3).forEach(e => {
+                    this._log('DEBUG', `  - ${e.entityId}: ${e.error}`);
+                });
+            }
+
+            // Update rolling hash with results
+            for (const result of Object.values(batchResults)) {
+                this._updateRollingHash(rollingHash, result);
+            }
+
+            // Commit results to storage
+            if (!options.dryRun && Object.keys(batchResults).length > 0) {
+                await this.storageManager.commitResults(dateStr, entry, batchResults, {});
+                const lastId = entityIds[entityIds.length - 1];
+                await checkpointer.markBatchComplete(dateStr, entry.name, cp?.id, batchIndex, batchSize, lastId);
+            }
+
+            totalCount += Object.keys(batchResults).length;
+            batchIndex++;
+        }
+
+        // Finalize
+        if (!options.dryRun) {
+            await this.storageManager.finalizeResults(dateStr, entry);
+            if (cp) await checkpointer.complete(dateStr, entry.name, cp.id);
+        }
+
+        if (totalErrors > 0) {
+            this._log('WARN', `[Remote] Completed with ${totalErrors} total errors out of ${totalCount + totalErrors} entities`);
+        }
 
         return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16) };
     }
 
-    // --- STRATEGY B: GLOBAL / IN-MEMORY ---
     async _executeGlobal(entry, dateStr, depResults, previousResult, options, forceEntities) {
-        // 1. Fetch Full Data
         const data = await this.dataFetcher.fetchForComputation(entry.requires, dateStr, forceEntities);
         const { rules } = this.ruleInjector.createContext();
-        
+        const driverTable = entry.type === 'per-entity' ? this._getDriverTable(entry.requires) : null;
+        const driverEntityField = driverTable ? this.config.tables[driverTable]?.entityField : null;
         const instance = new entry.class();
+        
         const context = {
-            computation: entry,
-            date: dateStr,
-            data,
+            computation: entry, date: dateStr, data,
             getDependency: (dep, ent) => this._lazyLoadDependency(dateStr, dep, ent, depResults),
-            previousResult,
-            rules,
-            references: this.referenceDataCache,
-            config: this.config,
-            entityId: forceEntities ? null : '_global',
-            dataFetcher: this.dataFetcher // <--- ADDED: Required by CostTrackerMiddleware
+            previousResult, rules, references: this.referenceDataCache,
+            config: this.config, entityId: forceEntities ? null : '_global', dataFetcher: this.dataFetcher
         };
 
-        // 2. Delegate to Runner
         let results = {};
         
         if (entry.type === 'per-entity') {
             const ids = forceEntities || this._extractEntityIds(data);
             const limit = pLimit(DEFAULT_CONCURRENCY);
-            
             await Promise.all(ids.map(id => limit(async () => {
-                const subCtx = { 
-                    ...context, 
-                    entityId: id, 
-                    data: this._filterDataForEntity(data, id) 
-                };
+                const subCtx = { ...context, entityId: id, data: this._filterDataForEntity(data, id, driverEntityField) };
                 const res = await this.runner.run(instance, subCtx);
                 if (res) results[id] = res;
             })));
@@ -355,26 +535,73 @@ class Orchestrator {
             results = await this.runner.run(instance, context);
         }
 
-        // 3. Smart Invalidation Check
         const finalHash = this._hashResults(results);
         
         if (!options.dryRun && !forceEntities) {
             const currentStatus = await this.stateRepository.getDailyStatus(dateStr);
             const status = currentStatus.get(entry.name.toLowerCase());
             
-            if (status && status.resultHash === finalHash) {
+            if (!options.force && status && status.resultHash === finalHash) {
                 return { count: Object.keys(results || {}).length, hash: finalHash, skipped: true };
             }
             
             await this.storageManager.commitResults(dateStr, entry, results, {});
+            await this.storageManager.finalizeResults(dateStr, entry);
         }
 
         return { count: Object.keys(results || {}).length, hash: finalHash };
     }
 
-    // =========================================================================
-    // HELPER METHODS
-    // =========================================================================
+    _printExecutionSummary(summary) {
+        console.log('\n┌────────────────────────────────────────────────────────┐');
+        console.log('│                   EXECUTION SUMMARY                    │');
+        console.log('└────────────────────────────────────────────────────────┘');
+        console.log(`\n📅 Date: ${summary.date}`);
+        console.log(`✅ Completed: ${summary.summary.completed}`);
+        console.log(`❌ Errors:    ${summary.summary.errors}`);
+        console.log(`⏭️  Skipped:   ${summary.summary.skipped + summary.summary.blocked + summary.summary.impossible}`);
+        
+        const skippedItems = [...summary.skipped, ...summary.blocked, ...summary.impossible];
+        if (skippedItems.length > 0) {
+            console.log('\n--- Details (Why did it skip?) ---');
+            skippedItems.forEach(item => {
+                let icon = '⏭️';
+                if (item.status === 'blocked') icon = '⛔';
+                if (item.status === 'impossible') icon = '🚫';
+                console.log(`${icon} ${item.name}: ${item.status.toUpperCase()} -> ${item.reason}`);
+            });
+        }
+        if (summary.errors.length > 0) {
+            console.log('\n--- Errors ---');
+            summary.errors.forEach(e => console.log(`❌ ${e.name}: ${e.error}`));
+        }
+        console.log('');
+    }
+
+    _getDriverTable(requires) {
+        for (const name of Object.keys(requires)) {
+            const conf = this.config.tables[name];
+            if (conf && conf.entityField) return name;
+        }
+        return null; 
+    }
+
+    _splitRequirements(requires, driverTable) {
+        const batchRequires = {};
+        const globalRequires = {};
+        if (!driverTable) return { batchRequires: {}, globalRequires: requires };
+        const driverConfig = this.config.tables[driverTable];
+        const driverEntityField = driverConfig ? driverConfig.entityField : null;
+        for (const [name, spec] of Object.entries(requires)) {
+            const conf = this.config.tables[name];
+            if (conf && conf.entityField === driverEntityField) {
+                batchRequires[name] = spec;
+            } else {
+                globalRequires[name] = spec;
+            }
+        }
+        return { batchRequires, globalRequires };
+    }
 
     async _analyzeEntry(entry, dateStr) {
         const d = await this.stateRepository.getDailyStatus(dateStr);
@@ -386,18 +613,12 @@ class Orchestrator {
         const depResults = {};
         const depResultHashes = {};
         const dailyStatus = await this.stateRepository.getDailyStatus(dateStr);
-        
         for (const dep of entry.dependencies) {
             const stat = dailyStatus.get(dep);
             if (stat?.resultHash) depResultHashes[dep] = stat.resultHash;
-            
-            if (stat?.entityCount > 50000) {
-                depResults[dep] = null; 
-            } else {
-                depResults[dep] = await this.stateRepository.getResult(dateStr, dep);
-            }
+            if (stat?.entityCount > 50000) depResults[dep] = null; 
+            else depResults[dep] = await this.stateRepository.getResult(dateStr, dep);
         }
-
         if (entry.conditionalDependencies) {
             for (const condDep of entry.conditionalDependencies) {
                 const shouldLoad = condDep.condition({ date: dateStr, config: this.config });
@@ -410,13 +631,11 @@ class Orchestrator {
                 }
             }
         }
-        
         return { depResults, depResultHashes };
     }
 
     async _prefetchBatchDependencies(entry, dateStr, loadedDeps, batchEntityIds) {
         const prefetched = {};
-        
         for (const depName of entry.dependencies) {
             if (loadedDeps[depName] === null) {
                 const batchRes = await this.stateRepository.getBatchEntityResults(dateStr, depName, batchEntityIds);
@@ -429,10 +648,7 @@ class Orchestrator {
     async _lazyLoadDependency(dateStr, depName, entityId, preloaded) {
         if (preloaded[depName] && !entityId) return preloaded[depName];
         if (preloaded[depName] && entityId) return preloaded[depName][entityId];
-        
-        if (entityId) {
-            return this.stateRepository.getEntityResult(dateStr, depName, entityId);
-        }
+        if (entityId) return this.stateRepository.getEntityResult(dateStr, depName, entityId);
         return this.stateRepository.getResult(dateStr, depName);
     }
 
@@ -440,15 +656,9 @@ class Orchestrator {
         if (!this.config.referenceData) return;
         await Promise.all(this.config.referenceData.map(async (table) => {
             try {
-                const data = await this.dataFetcher.fetch({
-                    table, 
-                    targetDate: new Date().toISOString().slice(0, 10), 
-                    mandatory: false
-                });
+                const data = await this.dataFetcher.fetch({ table, targetDate: new Date().toISOString().slice(0, 10), mandatory: false });
                 this.referenceDataCache[table] = data || {};
-            } catch (e) {
-                this._log('WARN', `Failed to load Ref Data ${table}: ${e.message}`);
-            }
+            } catch (e) { this._log('WARN', `Failed to load Ref Data ${table}: ${e.message}`); }
         }));
     }
 
@@ -461,37 +671,34 @@ class Orchestrator {
         return Array.from(ids);
     }
 
-    _filterDataForEntity(data, id) {
+    _filterDataForEntity(data, id, driverEntityField) {
         const out = {};
         Object.entries(data).forEach(([tbl, d]) => {
             const conf = this.config.tables[tbl] || {};
-            if (conf.entityField && d && !Array.isArray(d)) out[tbl] = d[id] || null;
-            else out[tbl] = d;
+            if (conf.entityField === driverEntityField && d && !Array.isArray(d)) {
+                 out[tbl] = d[id] || null;
+            } else {
+                 out[tbl] = d; 
+            }
         });
         return out;
     }
 
-    _updateRollingHash(hasher, result) {
-        if (result) hasher.update(JSON.stringify(result));
-    }
-    
+    _updateRollingHash(rollingHash, result) { if (result) rollingHash.update(JSON.stringify(result)); }
     _hashResults(results) {
         const canonical = JSON.stringify(results, Object.keys(results || {}).sort());
         return crypto.createHash('sha256').update(canonical).digest('hex').substring(0, 16);
     }
-
     _subtractDay(dateStr) {
         const d = new Date(dateStr + 'T00:00:00Z');
         d.setUTCDate(d.getUTCDate() - 1);
         return d.toISOString().slice(0, 10);
     }
-
     _getAllTables() { 
         const s = new Set(); 
         if (this.manifest) this.manifest.forEach(e => Object.keys(e.requires).forEach(t => s.add(t))); 
         return Array.from(s); 
     }
-
     _log(l, m) { this.logger.log(l, `[Orchestrator] ${m}`); }
 }