bulltrackers-module 1.0.738 → 1.0.739

package/functions/computation-system-v2/docs/plans.md

@@ -0,0 +1,588 @@
+# Comprehensive Review & Improvement Suggestions
+
+## 🚨 **CRITICAL ISSUES**
+
+### 1. **Infinite Loop Risk in Conditional Dependencies**
+```javascript
+// framework/core/Manifest.js - Line ~50
+if (config.conditionalDependencies) {
+    for (const condDep of config.conditionalDependencies) {
+        const shouldLoad = condDep.condition({ date: dateStr, config: this.config });
+        if (shouldLoad) {
+            // NO CYCLE DETECTION for conditional deps!
+            const depStatus = dailyStatus.get(condDep.computation.toLowerCase());
+            if (depStatus) {
+                depResults[condDep.computation] = await this.stateRepository.getResult(dateStr, condDep.computation);
+            }
+        }
+    }
+}
+```
+
+**Problem**: Conditional dependencies bypass cycle detection in `Graph.detectCycle()`. If Comp A conditionally depends on Comp B, and Comp B conditionally depends on Comp A, you get an infinite loop.
+
+**Fix**:
+```javascript
+// In ManifestBuilder._processComputation()
+_processComputation(ComputationClass) {
+    const config = ComputationClass.getConfig();
+    
+    // Merge conditional deps into regular deps for graph analysis
+    const allDeps = [...(config.dependencies || [])];
+    if (config.conditionalDependencies) {
+        allDeps.push(...config.conditionalDependencies.map(cd => cd.computation));
+    }
+    
+    return {
+        // ...
+        dependencies: allDeps.map(d => this._normalize(d)),
+        conditionalDependencies: config.conditionalDependencies || [],
+        // ...
+    };
+}
+```
+
+### 2. **Cost Spiral in Worker Pool Retries**
+```javascript
+// framework/execution/RemoteTaskRunner.js - Line ~200
+async _invokeWorkerWithRetry(payload, attempt = 1) {
+    try {
+        return await this._invokeWorker(payload);
+    } catch (e) {
+        if (isRetryable && attempt < this.retries) {
+            // NO COST TRACKING ON RETRIES!
+            return this._invokeWorkerWithRetry(payload, attempt + 1);
+        }
+        throw e;
+    }
+}
+```
+
+**Problem**: Failed workers retry without tracking costs. If 1000 entities fail and each retries 2x, you pay 3x the expected cost with no warning.
+
+**Fix**: Add cost multiplier tracking:
+```javascript
+class RemoteTaskRunner {
+    constructor(config, logger) {
+        this.costMultiplier = 1.0;
+        this.totalRetries = 0;
+    }
+    
+    async _invokeWorkerWithRetry(payload, attempt = 1) {
+        try {
+            return await this._invokeWorker(payload);
+        } catch (e) {
+            if (isRetryable && attempt < this.retries) {
+                this.totalRetries++;
+                this.costMultiplier = 1 + (this.totalRetries / this.stats.totalInvocations);
+                
+                if (this.costMultiplier > 2.0) {
+                    throw new Error(`🚨 RETRY COST SPIRAL: ${this.totalRetries} retries (${this.costMultiplier.toFixed(1)}x cost). Aborting.`);
+                }
+                
+                await new Promise(r => setTimeout(r, delay));
+                return this._invokeWorkerWithRetry(payload, attempt + 1);
+            }
+            throw e;
+        }
+    }
+}
+```
+
+### 3. **Runaway Query Cost in Historical Computations**
+```javascript
+// framework/data/DataFetcher.js - Line ~80
+async fetchForComputation(requires, targetDate, entities = null) {
+    await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+        const data = await this.fetch({
+            lookback: spec.lookback || 0,  // No upper limit!
+            // ...
+        });
+    }));
+}
+```
+
+**Problem**: A computation with `lookback: 365` on a 10TB table scans 365 days. If you have 100 computations with high lookbacks, costs explode.
+
+**Fix**: Add lookback budget:
+```javascript
+// In config
+execution: {
+    maxLookbackDays: 90,  // Hard limit
+    maxLookbackBudgetGB: 100,  // Cumulative budget per run
+}
+
+// In DataFetcher
+async fetch(options) {
+    if (options.lookback > this.config.execution.maxLookbackDays) {
+        throw new Error(`Lookback ${options.lookback} exceeds limit of ${this.config.execution.maxLookbackDays} days`);
+    }
+    
+    // Estimate scan size before executing
+    const estimatedGB = await this._estimateScanSize(options);
+    this.cumulativeScanGB += estimatedGB;
+    
+    if (this.cumulativeScanGB > this.config.execution.maxLookbackBudgetGB) {
+        throw new Error(`Query budget exceeded: ${this.cumulativeScanGB}GB / ${this.config.execution.maxLookbackBudgetGB}GB`);
+    }
+    
+    // Execute...
+}
+```
+
+---
+
+## ⚡ **PERFORMANCE BOTTLENECKS**
+
+### 4. **N+1 Dependency Fetches**
+```javascript
+// framework/execution/Orchestrator.js - Line ~450
+async _lazyLoadDependency(dateStr, depName, entityId, preloaded) {
+    if (preloaded[depName] && entityId) return preloaded[depName][entityId];
+    // FETCHES ONE ENTITY AT A TIME!
+    return this.stateRepository.getEntityResult(dateStr, depName, entityId);
+}
+```
+
+**Problem**: If a computation depends on `PopularInvestorProfileMetrics` and processes 10,000 users, this issues 10,000 individual BigQuery queries.
+
+**Fix**: Already partially addressed with `getBatchEntityResults()`, but enforce it:
+```javascript
+async _executeStreaming(entry, dateStr, depResults, previousResult, options) {
+    // ...
+    for await (const batch of batchStream) {
+        const { entityIds } = batch;
+        
+        // FORCE batch prefetch
+        const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
+        
+        // Remove lazy load fallback to catch bugs
+        await Promise.all(entityIds.map(entityId => limit(async () => {
+            const context = {
+                getDependency: (depName, targetId) => {
+                    const id = targetId || entityId;
+                    if (!batchDeps[depName]?.has(id)) {
+                        throw new Error(`LAZY LOAD DETECTED: ${depName}:${id}. This is a bug - dependencies must be prefetched.`);
+                    }
+                    return batchDeps[depName].get(id);
+                },
+                // ...
+            };
+        })));
+    }
+}
+```
+
+### 5. **Redundant Schema Fetches**
+```javascript
+// framework/data/SchemaRegistry.js - Line ~70
+async getSchema(tableName) {
+    const cached = this.cache.get(tableName);
+    if (cached && !this._isExpired(cached)) {
+        return cached;  // Good!
+    }
+    // But if 100 concurrent requests arrive during cold start...
+    return await this._fetchAndCacheSchema(tableName);  // 100 concurrent fetches!
+}
+```
+
+**Fix**: Coalesce concurrent requests:
+```javascript
+constructor() {
+    this.cache = new Map();
+    this.pendingFetches = new Map();  // NEW
+}
+
+async getSchema(tableName) {
+    const cached = this.cache.get(tableName);
+    if (cached && !this._isExpired(cached)) return cached;
+    
+    // Coalesce concurrent requests
+    if (this.pendingFetches.has(tableName)) {
+        return this.pendingFetches.get(tableName);
+    }
+    
+    const fetchPromise = this._fetchAndCacheSchema(tableName);
+    this.pendingFetches.set(tableName, fetchPromise);
+    
+    try {
+        return await fetchPromise;
+    } finally {
+        this.pendingFetches.delete(tableName);
+    }
+}
+```
+
+### 6. **Inefficient Batch Splitting**
+```javascript
+// framework/data/DataFetcher.js - Line ~200
+async *fetchBatched(options, batchSize = 1000) {
+    let batch = [];
+    let currentEntity = null;
+
+    for await (const row of rowStream) {
+        if (entityField) {
+            const rowEntity = String(row[entityField]);
+            
+            // Splits batch even if at 999 rows (wastes 1 row)
+            if (batch.length >= batchSize && rowEntity !== currentEntity && currentEntity !== null) {
+                yield this._transform(batch, { lookback, dateField, entityField, dataField });
+                batch = [];
+            }
+            currentEntity = rowEntity;
+        }
+        batch.push(row);
+    }
+}
+```
+
+**Problem**: If an entity has 5,000 rows and `batchSize=1000`, you get 5 batches for one entity. If processing is slow, this serializes work.
+
+**Fix**: Add max batch size override:
+```javascript
+async *fetchBatched(options, batchSize = 1000) {
+    const maxBatchSize = batchSize * 10;  // Allow up to 10x for large entities
+    
+    for await (const row of rowStream) {
+        if (entityField) {
+            const rowEntity = String(row[entityField]);
+            
+            // Force split if exceeding max
+            if (batch.length >= maxBatchSize) {
+                yield this._transform(batch, ...);
+                batch = [];
+                currentEntity = null;
+            }
+            
+            // Normal split logic
+            if (batch.length >= batchSize && rowEntity !== currentEntity && currentEntity !== null) {
+                yield this._transform(batch, ...);
+                batch = [];
+            }
+            currentEntity = rowEntity;
+        }
+        batch.push(row);
+    }
+}
+```
+
+---
+
+## 🔒 **RELIABILITY ISSUES**
+
+### 7. **Zombie Detection Race Condition**
+```javascript
+// handlers/scheduler.js - Line ~80
+let zombies = [];
+try {
+    zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
+    if (zombies.length > 0) {
+        console.log(`DETECTED ${zombies.length} ZOMBIES`);
+    }
+} catch (e) {
+    console.error(`Zombie check failed: ${e.message}`);
+}
+
+// Later: dispatch zombies
+const zombieEntries = zombies.map(z => {...});
+```
+
+**Problem**: If `findZombies()` throws, scheduler continues but `zombieEntries` is empty. Zombies never recover.
+
+**Fix**: Make zombie detection non-blocking:
+```javascript
+async function schedulerHandler(req, res) {
+    // 1. Find due computations (CRITICAL PATH)
+    const dueComputations = findDueComputations(now);
+    
+    // 2. Zombie detection (BEST EFFORT)
+    let zombies = [];
+    try {
+        zombies = await Promise.race([
+            storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 5000))
+        ]);
+    } catch (e) {
+        // Log but don't block scheduler
+        console.warn(`⚠️ Zombie detection failed (non-fatal): ${e.message}`);
+    }
+    
+    // Continue with due + zombies...
+}
+```
+
+### 8. **Silent Firestore Write Failures**
+```javascript
+// framework/storage/StorageManager.js - Line ~650
+async _writeToFirestore(dateStr, entry, results, firestoreConfig) {
+    try {
+        // ... batching logic ...
+        return Promise.all(batches.map(b => limit(() => b.commit()))).then(() => ({ docCount }));
+    } catch (error) {
+        if (error.message.includes('Could not load the default credentials')) {
+            console.log('  Skipped: No GCP credentials');
+            return 'skipped';
+        }
+        console.log(`  Error: ${error.message}`);
+        return false;  // Returns false but doesn't throw!
+    }
+}
+```
+
+**Problem**: If Firestore writes fail, `commitResults()` returns `{ firestore: false }` but execution continues. Users think data is saved.
+
+**Fix**:
+```javascript
+async commitResults(dateStr, entry, results, depResultHashes = {}) {
+    const storageConfig = this._resolveStorageConfig(entry);
+    
+    const firestoreTask = async () => {
+        if (!storageConfig.firestore?.enabled) return null;
+        const result = await this._writeToFirestore(...);
+        
+        // ENFORCE: If enabled, must succeed
+        if (result === false || result === 'skipped') {
+            throw new Error(`Firestore write failed for ${entry.name} but was configured as enabled`);
+        }
+        return result;
+    };
+    
+    const [bigqueryResult, firestoreResult] = await Promise.all([gcsTask(), firestoreTask()]);
+    return { bigquery: bigqueryResult, firestore: firestoreResult };
+}
+```
+
+---
+
+## 🎯 **NEW FEATURE SUGGESTIONS**
+
+### 9. **Adaptive Batch Sizing**
+```javascript
+// NEW: framework/execution/AdaptiveBatcher.js
+class AdaptiveBatcher {
+    constructor(initialSize = 1000) {
+        this.currentSize = initialSize;
+        this.performanceWindow = [];
+    }
+    
+    recordBatch(entityCount, durationMs) {
+        const throughput = entityCount / (durationMs / 1000);  // entities/sec
+        this.performanceWindow.push({ size: entityCount, throughput });
+        
+        if (this.performanceWindow.length > 10) {
+            this.performanceWindow.shift();
+        }
+        
+        // Adjust batch size based on throughput trend
+        const avgThroughput = this.performanceWindow.reduce((sum, p) => sum + p.throughput, 0) / this.performanceWindow.length;
+        
+        if (avgThroughput < 10) {
+            // Slow processing - reduce batch size for parallelism
+            this.currentSize = Math.max(100, this.currentSize * 0.8);
+        } else if (avgThroughput > 100) {
+            // Fast processing - increase batch size
+            this.currentSize = Math.min(10000, this.currentSize * 1.2);
+        }
+    }
+    
+    getSize() {
+        return Math.round(this.currentSize);
+    }
+}
+
+// In Orchestrator
+this.adaptiveBatcher = new AdaptiveBatcher(config.execution.batchSize);
+
+for await (const batch of batchStream) {
+    const startTime = Date.now();
+    // ... process batch ...
+    this.adaptiveBatcher.recordBatch(batch.entityIds.length, Date.now() - startTime);
+    
+    // Use adaptive size for next batch
+    batchSize = this.adaptiveBatcher.getSize();
+}
+```
+
+### 10. **Query Plan Caching**
+```javascript
+// NEW: framework/data/QueryPlanCache.js
+class QueryPlanCache {
+    constructor(bigquery) {
+        this.cache = new Map();
+        this.bigquery = bigquery;
+    }
+    
+    async getEstimate(query) {
+        const queryHash = crypto.createHash('md5').update(query.sql).digest('hex');
+        
+        if (this.cache.has(queryHash)) {
+            return this.cache.get(queryHash);
+        }
+        
+        // Use BigQuery dry run for cost estimation
+        const [job] = await this.bigquery.createQueryJob({
+            query: query.sql,
+            params: query.params,
+            dryRun: true
+        });
+        
+        const [metadata] = await job.getMetadata();
+        const estimate = {
+            bytesProcessed: parseInt(metadata.statistics.totalBytesProcessed || 0, 10),
+            cacheHit: metadata.statistics.query.cacheHit,
+            estimatedCost: (parseInt(metadata.statistics.totalBytesProcessed || 0, 10) / 1099511627776) * 5
+        };
+        
+        this.cache.set(queryHash, estimate);
+        return estimate;
+    }
+}
+
+// In DataFetcher
+async _execute(query) {
+    const estimate = await this.queryPlanCache.getEstimate(query);
+    
+    if (estimate.estimatedCost > 1.0) {  // $1+ queries
+        this._log('WARN', `Expensive query detected: $${estimate.estimatedCost.toFixed(2)} - ${query.sql.substring(0, 100)}`);
+    }
+    
+    // Execute as normal...
+}
+```
+
+### 11. **Circuit Breaker for Failed Computations**
+```javascript
+// NEW: framework/resilience/CircuitBreaker.js
+class CircuitBreaker {
+    constructor(failureThreshold = 5, resetTimeout = 60000) {
+        this.failures = new Map();
+        this.failureThreshold = failureThreshold;
+        this.resetTimeout = resetTimeout;
+    }
+    
+    recordFailure(computationName) {
+        const state = this.failures.get(computationName) || { count: 0, lastFailure: null };
+        state.count++;
+        state.lastFailure = Date.now();
+        this.failures.set(computationName, state);
+        
+        if (state.count >= this.failureThreshold) {
+            console.warn(`🔴 Circuit breaker OPEN for ${computationName} (${state.count} failures)`);
+        }
+    }
+    
+    recordSuccess(computationName) {
+        this.failures.delete(computationName);
+    }
+    
+    shouldSkip(computationName) {
+        const state = this.failures.get(computationName);
+        if (!state || state.count < this.failureThreshold) return false;
+        
+        // Auto-reset after timeout
+        if (Date.now() - state.lastFailure > this.resetTimeout) {
+            console.log(`🟡 Circuit breaker HALF-OPEN for ${computationName} (attempting reset)`);
+            this.failures.delete(computationName);
+            return false;
+        }
+        
+        return true;
+    }
+}
+
+// In Orchestrator._executeComputation
+if (this.circuitBreaker.shouldSkip(entry.name)) {
+    return { name: entry.name, status: 'circuit_open', reason: 'Too many consecutive failures' };
+}
+
+try {
+    const result = await this._executeGlobal(...);
+    this.circuitBreaker.recordSuccess(entry.name);
+    return result;
+} catch (e) {
+    this.circuitBreaker.recordFailure(entry.name);
+    throw e;
+}
+```
+
+---
+
+## 📊 **MONITORING ENHANCEMENTS**
+
+### 12. **Real-Time Cost Dashboard**
+```javascript
+// NEW: Create a separate Cloud Function that queries cost tables
+exports.getCostDashboard = async (req, res) => {
+    const { startDate, endDate } = req.query;
+    
+    const query = `
+        SELECT 
+            computation_name,
+            SUM(estimated_cost_usd) as total_cost,
+            COUNT(*) as runs,
+            AVG(estimated_cost_usd) as avg_cost,
+            MAX(estimated_cost_usd) as max_cost
+        FROM \`${PROJECT}.${DATASET}.computation_costs\`
+        WHERE date BETWEEN @start AND @end
+        GROUP BY computation_name
+        ORDER BY total_cost DESC
+    `;
+    
+    const [rows] = await bigquery.query({ query, params: { start: startDate, end: endDate } });
+    
+    res.json({
+        period: { start: startDate, end: endDate },
+        totalCost: rows.reduce((sum, r) => sum + r.total_cost, 0),
+        computations: rows,
+        alert: rows.some(r => r.total_cost > 10) ? 'HIGH_COST_DETECTED' : null
+    });
+};
+```
+
+---
+
+## 🎛️ **CONFIGURATION IMPROVEMENTS**
+
+### 13. **Cost-Aware Scheduling**
+```json
+// In config/bulltrackers.config.js
+{
+  "computations": [
+    {
+      "class": "PopularInvestorRiskMetrics",
+      "schedule": {
+        "frequency": "daily",
+        "time": "02:00",
+        "costBudget": 5.0,  // NEW: Skip if daily cost exceeded
+        "priority": 1  // NEW: Lower priority runs only if budget allows
+      }
+    }
+  ],
+  "execution": {
+    "dailyCostBudget": 50,  // Global budget
+    "costCheckInterval": 3600000  // Re-check every hour
+  }
+}
+```
+
+---
+
+## Summary of Critical Fixes Needed
+
+| Issue | Severity | Fix Complexity | Impact |
+|-------|----------|----------------|--------|
+| Conditional dependency cycles | 🔴 CRITICAL | Low | Infinite loops |
+| Worker pool retry cost spiral | 🔴 CRITICAL | Medium | Cost overruns |
+| Unbounded lookback queries | 🔴 CRITICAL | Low | Query cost explosion |
+| N+1 dependency fetches | 🟡 HIGH | Medium | 100x slower execution |
+| Schema fetch stampede | 🟡 HIGH | Low | Cold start failures |
+| Silent Firestore failures | 🟡 HIGH | Low | Data loss |
+| Zombie recovery failures | 🟡 HIGH | Low | Stuck computations |
+
+**Immediate Actions:**
+1. Add conditional dependency cycle detection (1 hour fix)
+2. Add retry cost tracking in RemoteTaskRunner (2 hours)
+3. Add lookback budget limits (1 hour)
+4. Enforce batch dependency prefetching (already partially done, needs enforcement - 2 hours)
+5. Add schema fetch coalescing (1 hour)
+

package/functions/computation-system-v2/framework/core/Manifest.js

@@ -38,7 +38,18 @@ class ManifestBuilder {
             const entry = this._processComputation(ComputationClass);
             if (entry) {
                 manifestMap.set(entry.name, entry);
-                adjacency.set(entry.name, entry.dependencies);
+                
+                // CRITICAL FIX: Include conditional dependencies in the DAG for cycle detection and topological sort.
+                // Even if the dependency is conditional at runtime, the execution order (Pass) must respect it.
+                const graphDeps = [...entry.dependencies];
+                if (entry.conditionalDependencies) {
+                    entry.conditionalDependencies.forEach(cd => {
+                        // Ensure we use the normalized name for the graph
+                        graphDeps.push(cd.computation);
+                    });
+                }
+
+                adjacency.set(entry.name, graphDeps);
             }
         }
         
@@ -104,6 +115,13 @@ class ManifestBuilder {
             compositeHash += `|RULE:${mod}:${h}`;
         }
         
+        // Normalize conditional dependencies if they exist
+        // This ensures the Orchestrator can look them up by normalized name later
+        const conditionalDependencies = (config.conditionalDependencies || []).map(cd => ({
+            ...cd,
+            computation: this._normalize(cd.computation)
+        }));
+
         return {
             name,
             originalName: config.name,
@@ -112,6 +130,7 @@ class ManifestBuilder {
             type: config.type || 'global',
             requires: config.requires || {},
             dependencies: (config.dependencies || []).map(d => this._normalize(d)),
+            conditionalDependencies, // FIX: Pass this through to the manifest entry
             isHistorical: config.isHistorical || false,
             isTest: config.isTest || false,
             schedule: this.scheduleValidator.parseSchedule(config.schedule),
@@ -133,6 +152,7 @@ class ManifestBuilder {
     _computeFinalHashes(sorted, manifestMap) {
         for (const entry of sorted) {
             let hashInput = entry.hash;
+            // Includes strict dependencies in the hash chain
             if (entry.dependencies.length > 0) {
                 const depHashes = entry.dependencies.sort().map(d => {
                     const h = manifestMap.get(d)?.hash;
@@ -141,6 +161,10 @@ class ManifestBuilder {
                 });
                 hashInput += `|DEPS:${depHashes.join('|')}`;
             }
+            // Note: Conditional dependencies are currently excluded from the hash chain 
+            // because they might not be loaded. If strict versioning is required for them,
+            // they should be added here too.
+            
             entry.hash = this._hashCode(hashInput);
         }
     }

package/functions/computation-system-v2/framework/data/DataFetcher.js

@@ -6,10 +6,18 @@
  * * V2.3 FIX: "Insufficient History" bug.
  * - fetchBatched now orders by Entity ID to keep historical rows together.
  * - Implemented "Entity-Atomic Batching" to prevent splitting a user's history across batches.
+ * * V2.4 FIX: Runaway Query Cost Prevention [Fix #3].
+ * * V2.5 UPDATE: Super-Entity Monitoring [Safety Valve for Fix #6].
+ * - Warns if a single entity exceeds reasonable batch limits (Memory Risk).
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
 
+// FIX #3: Hard limit to prevent cost spirals
+const MAX_LOOKBACK_DAYS = 30;
+// FIX #6 (Alternative): Warn if an entity is massive (e.g. > 5x batch size)
+const BATCH_GROWTH_WARNING_THRESHOLD = 5; 
+
 class DataFetcher {
     constructor(config, queryBuilder, logger = null) {
         this.projectId = config.projectId;
@@ -151,6 +159,12 @@ class DataFetcher {
     
     async fetch(options) {
         const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
+        
+        // FIX #3: Prevent Runaway Costs
+        if (lookback > MAX_LOOKBACK_DAYS) {
+            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit of ${MAX_LOOKBACK_DAYS}. Table: ${table}`);
+        }
+
         const tableConfig = this.tables[table] || {};
         const { dateField, entityField, dataField } = tableConfig;
         
@@ -168,6 +182,12 @@ class DataFetcher {
 
     async *fetchBatched(options, batchSize = 1000) {
         const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
+
+        // FIX #3: Prevent Runaway Costs
+        if (lookback > MAX_LOOKBACK_DAYS) {
+            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit of ${MAX_LOOKBACK_DAYS}. Table: ${table}`);
+        }
+
         const tableConfig = this.tables[table] || {};
         const { dateField, entityField, dataField } = tableConfig;
 
@@ -181,6 +201,7 @@ class DataFetcher {
         
         let batch = [];
         let currentEntity = null;
+        let batchHasWarned = false; // Flag to prevent log spam for a single massive batch
 
         for await (const row of rowStream) {
             // FIX #2: Entity-Atomic Batching
@@ -188,12 +209,23 @@ class DataFetcher {
             if (entityField) {
                 const rowEntity = String(row[entityField]);
                 
-                // If batch is full AND we have moved to a new entity, yield the batch
-                // This ensures the current entity (which might have many rows) stays together
+                // Check if we should yield
+                // Condition: Batch is full AND we are on a NEW entity
                 if (batch.length >= batchSize && rowEntity !== currentEntity && currentEntity !== null) {
                     yield this._transform(batch, { lookback, dateField, entityField, dataField });
                     batch = [];
+                    batchHasWarned = false;
                 }
+                
+                // SAFETY VALVE (Fix #6 Alternative):
+                // If batch grows huge (Super Entity) and we CANNOT split (same entity), warn the admin.
+                if (batch.length > batchSize * BATCH_GROWTH_WARNING_THRESHOLD && !batchHasWarned) {
+                    this._log('WARN', `SUPER ENTITY DETECTED: Entity '${currentEntity}' in table '${table}' has >${batch.length} rows. ` + 
+                              `This exceeds batch size ${batchSize} by ${BATCH_GROWTH_WARNING_THRESHOLD}x. ` +
+                              `Risk of OOM or Timeouts. Consider filtering this entity.`);
+                    batchHasWarned = true;
+                }
+                
                 currentEntity = rowEntity;
             } else {
                 // Fallback for non-entity tables (strict count)

package/functions/computation-system-v2/framework/data/SchemaRegistry.js

@@ -1,12 +1,12 @@
 /**
  * @fileoverview Schema Registry - Dynamic schema discovery with caching
- * 
- * Core innovation of v2: No hardcoded schemas. Instead, we:
+ * * Core innovation of v2: No hardcoded schemas. Instead, we:
  * 1. Fetch schemas from BigQuery INFORMATION_SCHEMA on first access
  * 2. Cache them with configurable TTL
  * 3. Validate all queries against cached schemas BEFORE sending to BigQuery
- * 
- * This prevents runtime query failures and eliminates schema maintenance burden.
+ * * This prevents runtime query failures and eliminates schema maintenance burden.
+ * * UPDATE: Implemented Request Coalescing (Fix #5) to prevent "Thundering Herd"
+ * on startup or cache expiry.
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
@@ -45,18 +45,21 @@ class SchemaRegistry {
         
         this.client = new BigQuery({ projectId: this.projectId });
         this.cache = new Map();
+        this.pendingFetches = new Map(); // FIX: Track in-flight requests
         
         // Track schema fetch stats for monitoring
         this.stats = {
             hits: 0,
             misses: 0,
             refreshes: 0,
-            errors: 0
+            errors: 0,
+            coalesced: 0 // New metric
         };
     }
     
     /**
      * Get schema for a table, fetching from BigQuery if not cached.
+     * Implements Request Coalescing to handle concurrent access.
      * @param {string} tableName - Table name (without dataset prefix)
      * @returns {Promise<TableSchema>}
      */
@@ -67,6 +70,13 @@ class SchemaRegistry {
             this.stats.hits++;
             return cached;
         }
+
+        // FIX: Check for pending fetch (Request Coalescing)
+        if (this.pendingFetches.has(tableName)) {
+            this.stats.coalesced++;
+            // this._log('DEBUG', `Coalescing request for ${tableName}`);
+            return this.pendingFetches.get(tableName);
+        }
         
         if (cached) {
             this.stats.refreshes++;
@@ -76,7 +86,16 @@ class SchemaRegistry {
             this._log('DEBUG', `Schema cache miss for ${tableName}, fetching...`);
         }
         
-        return await this._fetchAndCacheSchema(tableName);
+        // Create the promise and store it
+        const fetchPromise = this._fetchAndCacheSchema(tableName);
+        this.pendingFetches.set(tableName, fetchPromise);
+
+        try {
+            return await fetchPromise;
+        } finally {
+            // Always clean up pending map, success or failure
+            this.pendingFetches.delete(tableName);
+        }
     }
     
     /**
@@ -158,6 +177,8 @@ class SchemaRegistry {
     async warmCache(tableNames) {
         const results = { success: [], failed: [] };
         
+        // With request coalescing, we can just map and wait.
+        // Simultaneous calls for the same table will automatically merge.
         await Promise.all(tableNames.map(async (tableName) => {
             try {
                 await this.getSchema(tableName);
@@ -178,9 +199,11 @@ class SchemaRegistry {
     clearCache(tableName = null) {
         if (tableName) {
             this.cache.delete(tableName);
+            this.pendingFetches.delete(tableName); // Also clear pending if forced
             this._log('DEBUG', `Cleared schema cache for ${tableName}`);
         } else {
             this.cache.clear();
+            this.pendingFetches.clear();
             this._log('DEBUG', 'Cleared entire schema cache');
         }
     }
@@ -193,6 +216,7 @@ class SchemaRegistry {
         return {
             ...this.stats,
             cachedTables: this.cache.size,
+            pendingRequests: this.pendingFetches.size,
             cacheContents: Array.from(this.cache.keys())
         };
     }
@@ -284,4 +308,4 @@ class SchemaRegistry {
     }
 }
 
-module.exports = { SchemaRegistry };
+module.exports = { SchemaRegistry };

package/functions/computation-system-v2/framework/execution/Orchestrator.js

@@ -9,6 +9,8 @@
  * * * UPDATE: Includes Global vs Batch Data Split to fix "Identity Crisis".
  * * * UPDATE: Implemented FORCE logic to bypass "up-to-date" checks for testing.
  * * * UPDATE: Aggregates performance reporting to prevent log spam.
+ * * * FIX: Resolved N+1 Dependency Fetching (Strict Mode in Streaming).
+ * * * FIX: Added missing 'skipped' property to return types for type safety.
  */
 
 const crypto = require('crypto');
@@ -316,21 +318,41 @@ class Orchestrator {
 
             const { data: batchLocalData, entityIds } = batch;
             const combinedData = { ...batchLocalData, ...globalData };
+            
+            // STRICT FIX: Prefetch dependencies for the batch.
             const batchDeps = await this._prefetchBatchDependencies(entry, dateStr, depResults, entityIds);
+            
             const { rules } = this.ruleInjector.createContext(); 
             const batchResults = {};
             
             await Promise.all(entityIds.map(entityId => limit(async () => {
                 const instance = new entry.class();
                 const entityData = this._filterDataForEntity(combinedData, entityId, driverEntityField);
+                
                 const context = {
                     computation: entry, date: dateStr, entityId, data: entityData,
+                    
+                    // STRICT FIX: No fallback to _lazyLoadDependency.
                     getDependency: (depName, targetId) => {
-                        if (batchDeps[depName] && batchDeps[depName].has(targetId || entityId)) {
-                            return batchDeps[depName].get(targetId || entityId);
+                        const id = targetId || entityId;
+                        
+                        // 1. Look in Batch-Prefetched Dependencies (Priority)
+                        if (batchDeps[depName] && batchDeps[depName].has(id)) {
+                            return batchDeps[depName].get(id);
+                        }
+                        
+                        // 2. Look in Global/Preloaded Dependencies
+                        if (depResults[depName]) {
+                             if (depResults[depName][id] !== undefined) return depResults[depName][id];
                         }
-                        return this._lazyLoadDependency(dateStr, depName, targetId || entityId, depResults);
+                        
+                        // 3. STRICT MODE: Throw Error
+                        throw new Error(
+                            `[Strict Dependency] Dependency '${depName}' (ID: ${id}) not found in batch context. ` +
+                            `Ensure '${depName}' is listed in ${entry.name}.getConfig().dependencies.`
+                        );
                     },
+                    
                     previousResult, rules, references: this.referenceDataCache,
                     config: this.config, dataFetcher: this.dataFetcher
                 };
@@ -357,22 +379,18 @@ class Orchestrator {
             if (cp) await checkpointer.complete(dateStr, entry.name, cp.id);
         }
 
-        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16) };
+        // FIX: Return valid object shape including skipped: false
+        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16), skipped: false };
     }
 
     /**
      * Determine if a computation should use remote workers
-     * 
-     * @param {Object} entry - Manifest entry
+     * * @param {Object} entry - Manifest entry
      * @param {Object} options - Execution options
      * @param {boolean} [options.useWorkerPool] - Runtime override (true/false/undefined)
      * @param {boolean} [options.forceLocal] - Force local execution
      */
     _shouldUseRemoteWorkers(entry, options) {
-        // Runtime override takes precedence (for admin testing)
-        // useWorkerPool: true  -> force use worker pool
-        // useWorkerPool: false -> force local execution
-        // useWorkerPool: undefined -> use config
         if (options.useWorkerPool === true) {
             if (!this.remoteRunner) {
                 this._log('WARN', 'useWorkerPool=true but remoteRunner not initialized');
@@ -384,30 +402,22 @@ class Orchestrator {
             return false;
         }
         
-        // No remote runner configured
         if (!this.remoteRunner) return false;
-        
-        // Force local execution via options
         if (options.forceLocal) return false;
         
         const poolConfig = this.config.workerPool || {};
         
-        // Exclusion list
         if (poolConfig.excludeComputations?.includes(entry.name) || 
             poolConfig.excludeComputations?.includes(entry.originalName)) {
             return false;
         }
         
-        // Force list (override threshold)
         if (poolConfig.forceOffloadComputations?.includes(entry.name) || 
             poolConfig.forceOffloadComputations?.includes(entry.originalName)) {
             return true;
         }
         
-        // Only per-entity computations can be offloaded
         if (entry.type !== 'per-entity') return false;
-        
-        // Default: use remote if worker pool is enabled
         return true;
     }
 
@@ -492,7 +502,6 @@ class Orchestrator {
                 this._log('WARN', `[Remote] Batch ${batchIndex}: ${errors.length} entities failed`);
                 totalErrors += errors.length;
                 
-                // Log first few errors for debugging
                 errors.slice(0, 3).forEach(e => {
                     this._log('DEBUG', `  - ${e.entityId}: ${e.error}`);
                 });
@@ -524,7 +533,8 @@ class Orchestrator {
             this._log('WARN', `[Remote] Completed with ${totalErrors} total errors out of ${totalCount + totalErrors} entities`);
         }
 
-        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16) };
+        // FIX: Return valid object shape including skipped: false
+        return { count: totalCount, hash: rollingHash.digest('hex').substring(0, 16), skipped: false };
     }
 
     async _executeGlobal(entry, dateStr, depResults, previousResult, options, forceEntities) {
@@ -569,7 +579,8 @@ class Orchestrator {
             await this.storageManager.finalizeResults(dateStr, entry);
         }
 
-        return { count: Object.keys(results || {}).length, hash: finalHash };
+        // FIX: Return valid object shape including skipped: false
+        return { count: Object.keys(results || {}).length, hash: finalHash, skipped: false };
     }
 
     _printExecutionSummary(summary) {
@@ -668,6 +679,10 @@ class Orchestrator {
     async _lazyLoadDependency(dateStr, depName, entityId, preloaded) {
         if (preloaded[depName] && !entityId) return preloaded[depName];
         if (preloaded[depName] && entityId) return preloaded[depName][entityId];
+        
+        // WARN: This is the slow path that we removed from Streaming
+        this._log('WARN', `LAZY LOAD: Fetching single entity '${entityId}' for '${depName}'. This is slow.`);
+        
         if (entityId) return this.stateRepository.getEntityResult(dateStr, depName, entityId);
         return this.stateRepository.getResult(dateStr, depName);
     }

package/functions/computation-system-v2/framework/execution/RemoteTaskRunner.js

@@ -1,20 +1,18 @@
 /**
  * @fileoverview Remote Task Runner (Serverless Worker Pool Client)
- * 
- * RESPONSIBILITIES:
+ * * RESPONSIBILITIES:
  * 1. Package entity data and context into GCS files
  * 2. Invoke remote worker functions in parallel
  * 3. Collect results and errors
  * 4. Handle retries for transient failures
- * 
- * DATA FLOW:
+ * * DATA FLOW:
  * Orchestrator calls runBatch() -> Upload to GCS -> Invoke Workers -> Collect Results
- * 
- * DESIGN PRINCIPLES:
+ * * DESIGN PRINCIPLES:
  * - Workers are stateless - all context is passed via GCS
  * - High parallelism - hundreds of concurrent invocations
  * - Fault isolation - one entity failure doesn't affect others
  * - Cost efficient - workers scale to zero between runs
+ * - RESILIENCE: Implements Circuit Breaker to prevent Retry Cost Spirals [Fix #2]
  */
 
 const { Storage } = require('@google-cloud/storage');
@@ -36,6 +34,13 @@ class RemoteTaskRunner {
         this.timeout = poolConfig.timeout || 60000; // 60s default
         this.retries = poolConfig.retries || 2;
         
+        // Circuit Breaker Config [Fix #2]
+        this.cbConfig = {
+            minInvocations: 20,       // Minimum calls before checking rate
+            failureThreshold: 0.30,   // Trip if failure rate > 30%
+            ...poolConfig.circuitBreaker
+        };
+        
         // Local mode for testing
         this.localMode = poolConfig.localMode || process.env.WORKER_LOCAL_MODE === 'true';
         
@@ -53,8 +58,7 @@ class RemoteTaskRunner {
 
     /**
      * Execute a batch of entities remotely (or locally for testing)
-     * 
-     * @param {Object} entry - Manifest entry for the computation
+     * * @param {Object} entry - Manifest entry for the computation
      * @param {string} dateStr - Target date (YYYY-MM-DD)
      * @param {Object} baseContext - Shared context (references, config)
      * @param {string[]} entityIds - Entity IDs to process
@@ -126,11 +130,21 @@ class RemoteTaskRunner {
         const errors = [];
         const uploadedPaths = [];
 
+        // Circuit Breaker Stats (scoped to this batch)
+        const batchStats = {
+            invocations: 0,
+            failures: 0,
+            tripped: false
+        };
+
         // Phase 1: Upload context packages to GCS
         this._log('INFO', 'Uploading context packages to GCS...');
         const uploadStart = Date.now();
         
         const uploadTasks = entityIds.map(entityId => uploadLimit(async () => {
+            // Check tripped status early to save uploads if massive failure occurring
+            if (batchStats.tripped) return;
+
             const contextPackage = this._buildContextPackage(
                 entry,
                 entityId,
@@ -158,13 +172,19 @@ class RemoteTaskRunner {
 
         const invokeTasks = uploadedPaths.map(({ entityId, path }) => 
             invokeLimit(async () => {
+                // FAIL FAST: If circuit tripped, do not invoke worker
+                if (batchStats.tripped) {
+                    errors.push({ entityId, error: 'Skipped: Circuit Breaker Tripped due to high failure rate' });
+                    return;
+                }
+
                 try {
                     const response = await this._invokeWorkerWithRetry({
                         computationName: entry.originalName || entry.name,
                         entityId,
                         date: dateStr,
                         dataUri: { bucket: this.bucketName, path }
-                    });
+                    }, 1, batchStats); // Pass stats object to retry logic
 
                     if (response.status === 'success' && response.result !== null) {
                         results[entityId] = response.result;
@@ -174,12 +194,18 @@ class RemoteTaskRunner {
                     // status === 'success' with result === null means skipped (filtered out)
                     
                 } catch (e) {
+                    // Circuit Breaker errors are thrown here
                     errors.push({ entityId, error: e.message });
                 }
             })
         );
 
         await Promise.all(invokeTasks);
+        
+        if (batchStats.tripped) {
+            this._log('ERROR', `Batch ABORTED by Circuit Breaker. Stats: ${batchStats.failures} failures / ${batchStats.invocations} invocations.`);
+        }
+        
         this._log('INFO', `Invocations complete in ${Date.now() - invokeStart}ms`);
 
         // Phase 3: Cleanup GCS (fire and forget)
@@ -237,12 +263,21 @@ class RemoteTaskRunner {
     }
 
     /**
-     * Invoke a worker with retry logic
+     * Invoke a worker with retry logic and Circuit Breaker
      */
-    async _invokeWorkerWithRetry(payload, attempt = 1) {
+    async _invokeWorkerWithRetry(payload, attempt = 1, stats = null) {
+        // Track Invocation (Cost)
+        if (stats) stats.invocations++;
+
         try {
             return await this._invokeWorker(payload);
         } catch (e) {
+            // Track Failure
+            if (stats) {
+                stats.failures++;
+                this._checkCircuitBreaker(stats);
+            }
+
             const isRetryable = this._isRetryableError(e);
             
             if (isRetryable && attempt < this.retries) {
@@ -250,14 +285,40 @@ class RemoteTaskRunner {
                 const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000);
                 await new Promise(r => setTimeout(r, delay));
                 
+                // Re-check circuit before retrying (another thread might have tripped it)
+                if (stats) this._checkCircuitBreaker(stats);
+
                 this._log('DEBUG', `Retrying ${payload.entityId} (attempt ${attempt + 1})`);
-                return this._invokeWorkerWithRetry(payload, attempt + 1);
+                return this._invokeWorkerWithRetry(payload, attempt + 1, stats);
             }
             
             throw e;
         }
     }
 
+    /**
+     * Check circuit breaker status and throw if tripped
+     */
+    _checkCircuitBreaker(stats) {
+        if (stats.tripped) {
+            throw new Error('Circuit Breaker: Batch aborted due to high failure rate');
+        }
+
+        // Only check after minimum invocations (warmup)
+        if (stats.invocations >= this.cbConfig.minInvocations) {
+            const failureRate = stats.failures / stats.invocations;
+            
+            if (failureRate > this.cbConfig.failureThreshold) {
+                stats.tripped = true;
+                const msg = `🚨 CIRCUIT BREAKER TRIPPED! Failure rate ${(failureRate * 100).toFixed(1)}% ` +
+                            `(${stats.failures}/${stats.invocations}) exceeds threshold of ${(this.cbConfig.failureThreshold * 100)}%`;
+                
+                this._log('ERROR', msg);
+                throw new Error(msg);
+            }
+        }
+    }
+
     /**
      * Invoke a single worker via HTTP
      */
@@ -324,4 +385,4 @@ class RemoteTaskRunner {
     }
 }
 
-module.exports = { RemoteTaskRunner };
+module.exports = { RemoteTaskRunner };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.738",
+  "version": "1.0.739",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [