bulltrackers-module 1.0.768 → 1.0.769

package/functions/computation-system-v2/framework/data/DataFetcher.js

@@ -1,27 +1,19 @@
 /**
  * @fileoverview Data Fetcher - Executes queries and transforms results
  * * The single point of data access for computations.
- * Uses QueryBuilder for validation, executes against BigQuery, transforms results.
- * * V2.2 FIX: "Identity Crisis" & "Date as ID" bugs. 
- * * V2.3 FIX: "Insufficient History" bug.
- * - fetchBatched now orders by Entity ID to keep historical rows together.
- * - Implemented "Entity-Atomic Batching" to prevent splitting a user's history across batches.
- * * V2.4 FIX: Runaway Query Cost Prevention [Fix #3].
- * * V2.5 UPDATE: Super-Entity Monitoring [Safety Valve for Fix #6].
- * - Warns if a single entity exceeds reasonable batch limits (Memory Risk).
- * * V2.6 UPDATE: Query Result Caching.
- * - Implemented in-memory LRU cache to prevent redundant BigQuery costs for reference data.
- * * V2.7 FIX: Double-Encoded JSON Normalization.
- * - Automatically detects and recursively parses JSON strings (e.g. posts_data) to prevent downstream parsing errors.
+ * * V3.3 FIX: "Driver Priority" & "Identity Crisis" Patch.
+ * * V3.5 GUARD: Cost Protections (Dry Run & Limit Checks).
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
 const crypto = require('crypto');
+const { MaterializedViewManager } = require('./MaterializedViewManager');
 
-// FIX #3: Hard limit to prevent cost spirals
-const MAX_LOOKBACK_DAYS = 30;
-// FIX #6 (Alternative): Warn if an entity is massive (e.g. > 5x batch size)
-const BATCH_GROWTH_WARNING_THRESHOLD = 5; 
+// SAFETY CONFIGURATION
+// You can move these to your main config file if preferred
+const DEFAULT_SAFETY_LIMIT_GB = 10; // Max GB per query
+const MAX_LOOKBACK_DAYS = 60;
+const BATCH_GROWTH_WARNING_THRESHOLD = 5;
 
 class DataFetcher {
     constructor(config, queryBuilder, logger = null) {
@@ -31,19 +23,25 @@ class DataFetcher {
         this.tables = config.tables || {};
         this.queryBuilder = queryBuilder;
         this.logger = logger;
-        
+
+        // Safety Limit from Config or Default
+        this.safetyLimitGb = config.safetyLimitGb || DEFAULT_SAFETY_LIMIT_GB;
+
         this.client = new BigQuery({ projectId: this.projectId });
-        
-        // Cache Configuration (V2.6)
+
+        // Initialize MV Manager
+        this.mvManager = new MaterializedViewManager(this.client, this.logger, config);
+
+        // Cache Configuration
         this.cacheConfig = config.queryCache || {
             enabled: true,
             ttlMs: 300000, // 5 minutes default
             maxSize: 1000  // Max unique queries to cache
         };
-        
-        // Use Map as LRU cache (insertion order preserved)
+
+        // Use Map as LRU cache
         this.cache = new Map();
-        
+
         this.stats = {
             queries: 0,
             rowsFetched: 0,
@@ -54,18 +52,25 @@ class DataFetcher {
             cacheEvictions: 0
         };
     }
-    
+
     /**
      * Fetch data for a computation's requirements.
      */
     async fetchForComputation(requires, targetDate, entities = null) {
         const results = {};
         const errors = [];
-        
-        await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+
+        await Promise.all(Object.entries(requires).map(async ([key, spec]) => {
             try {
+                // INTERCEPT METRICS
+                if (spec.type === 'metric') {
+                    results[key] = await this._fetchMetric(spec, targetDate, entities);
+                    return;
+                }
+
+                // Standard Table Fetch
                 const data = await this.fetch({
-                    table: tableName,
+                    table: key,
                     targetDate,
                     lookback: spec.lookback || 0,
                     mandatory: spec.mandatory || false,
@@ -73,38 +78,82 @@ class DataFetcher {
                     fields: spec.fields || null,
                     entities: entities
                 });
-                
-                results[tableName] = data;
-                
+
+                results[key] = data;
+
                 if (spec.mandatory && this._isEmpty(data)) {
-                    errors.push({ table: tableName, reason: 'MANDATORY_MISSING' });
+                    errors.push({ table: key, reason: 'MANDATORY_MISSING' });
                 }
-                
+
             } catch (e) {
                 if (spec.mandatory) {
-                    errors.push({ table: tableName, reason: e.message });
+                    errors.push({ table: key, reason: e.message });
                 } else {
-                    this._log('WARN', `Optional table ${tableName} failed: ${e.message}`);
+                    this._log('WARN', `Optional requirement ${key} failed: ${e.message}`);
                 }
-                results[tableName] = null;
+                results[key] = null;
             }
         }));
-        
+
         if (errors.length > 0) {
             const msg = errors.map(e => `${e.table}: ${e.reason}`).join(', ');
             throw new Error(`[DataFetcher] Missing mandatory data: ${msg}`);
         }
-        
+
         return results;
     }
 
+    /**
+     * Fetch Metric with optional Time Series
+     */
+    async _fetchMetric(spec, targetDate, entities) {
+        const mvName = await this.mvManager.ensureMetricView(spec.source, spec);
+
+        const selectClause = spec.series ? 'entity_id, date, value' : 'entity_id, SUM(value) as value';
+        const groupByClause = spec.series ? '' : 'GROUP BY entity_id';
+
+        let sql = `
+            SELECT ${selectClause}
+            FROM \`${this.projectId}.${this.dataset}.${mvName}\`
+            WHERE date BETWEEN DATE_SUB(@targetDate, INTERVAL @lookback DAY) AND @targetDate
+        `;
+
+        const params = { targetDate, lookback: spec.lookback || 0 };
+
+        if (entities && entities.length > 0) {
+            sql += ` AND entity_id IN UNNEST(@entities)`;
+            params.entities = entities.map(String);
+        }
+
+        sql += ` ${groupByClause}`;
+
+        const rows = await this._execute({ sql, params, table: mvName });
+        const result = {};
+
+        if (spec.series) {
+            rows.forEach(r => {
+                const eid = r.entity_id;
+                const d = r.date.value || r.date;
+                if (!result[eid]) result[eid] = {};
+                result[eid][d] = r.value;
+            });
+        } else {
+            rows.forEach(r => { result[r.entity_id] = r.value; });
+        }
+
+        return result;
+    }
+
+    /**
+     * STREAMING: Fetch data for computation in batches.
+     */
     /**
      * STREAMING: Fetch data for computation in batches.
      */
     async *fetchComputationBatched(requires, targetDate, batchSize = 1000) {
         const driverTableName = this._getDriverTable(requires);
         const driverConfig = this.tables[driverTableName] || {};
-        
+
         if (!driverTableName) {
             this._log('WARN', 'No entity-keyed table found for batching. Falling back to full fetch.');
             const fullData = await this.fetchForComputation(requires, targetDate);
@@ -113,9 +162,12 @@ class DataFetcher {
             return;
         }
 
-        this._log('INFO', `Starting batched fetch driven by table: ${driverTableName} (${driverConfig.entityField})`);
         const driverSpec = requires[driverTableName];
 
+        // [DEBUG] Explicitly log the filter being applied to the driver
+        this._log('INFO', `Starting batched fetch driven by table: ${driverTableName} (${driverConfig.entityField})`);
+        this._log('INFO', `Driver Filter: ${JSON.stringify(driverSpec.filter || {})}`);
+
         const driverStream = this.fetchBatched({
             table: driverTableName,
             targetDate,
@@ -126,145 +178,154 @@ class DataFetcher {
         }, batchSize);
 
         for await (const batch of driverStream) {
-            // FIX: Robust ID Extraction
             const entityIds = this._extractEntityIdsFromBatch(batch, driverTableName);
-            
+
             if (entityIds.length === 0) {
-                this._log('WARN', `Driver batch from ${driverTableName} yielded 0 entity IDs. Check config.entityField matches BigQuery column.`);
+                this._log('WARN', `Driver batch from ${driverTableName} yielded 0 entity IDs.`);
                 continue;
             }
 
             const batchResults = { [driverTableName]: batch };
             const errors = [];
 
-            await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
-                if (tableName === driverTableName) return; 
-
-                // FIX: Identity Crisis Check
-                const depConfig = this.tables[tableName] || {};
-                const shouldFilterById = depConfig.entityField === driverConfig.entityField;
+            await Promise.all(Object.entries(requires).map(async ([key, spec]) => {
+                if (key === driverTableName) return;
 
                 try {
+                    if (spec.type === 'metric') {
+                        batchResults[key] = await this._fetchMetric(spec, targetDate, entityIds);
+                        return;
+                    }
+
+                    const depConfig = this.tables[key] || {};
+                    const hasEntityField = !!depConfig.entityField;
+
                     const data = await this.fetch({
-                        table: tableName,
+                        table: key,
                         targetDate,
                         lookback: spec.lookback || 0,
                         mandatory: spec.mandatory || false,
                         filter: spec.filter || {},
                         fields: spec.fields || null,
-                        entities: shouldFilterById ? entityIds : null
+                        entities: hasEntityField ? entityIds : null
                     });
-                    
-                    batchResults[tableName] = data;
+
+                    batchResults[key] = data;
 
                     if (spec.mandatory && this._isEmpty(data)) {
-                        this._log('WARN', `Batch warning: Mandatory table ${tableName} returned 0 rows. (Filtered by ID: ${shouldFilterById})`);
+                        this._log('WARN', `Batch warning: Mandatory table ${key} returned 0 rows.`);
                     }
                 } catch (e) {
-                    if (spec.mandatory) errors.push({ table: tableName, reason: e.message });
-                    batchResults[tableName] = null;
+                    if (spec.mandatory) errors.push({ table: key, reason: e.message });
+                    batchResults[key] = null;
                 }
             }));
 
             if (errors.length > 0) {
-                this._log('WARN', `Batch missing mandatory data due to errors: ${errors.map(e => e.table).join(', ')}. Skipping batch.`);
-                continue; 
+                this._log('WARN', `Batch missing mandatory data: ${errors.map(e => e.table).join(', ')}. Skipping.`);
+                continue;
             }
 
             yield { data: batchResults, entityIds };
         }
     }
-    
+
     async fetch(options) {
         const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
-        
-        // FIX #3: Prevent Runaway Costs
+
         if (lookback > MAX_LOOKBACK_DAYS) {
-            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit of ${MAX_LOOKBACK_DAYS}. Table: ${table}`);
+            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit.`);
         }
 
         const tableConfig = this.tables[table] || {};
         const { dateField, entityField, dataField } = tableConfig;
-        
+        const physicalTable = tableConfig.tableName || table;
+
         const query = await this.queryBuilder.build({
-            table, select: fields, where: filter, dateField, targetDate, lookback, entityField, entities,
+            table: physicalTable,
+            select: fields,
+            where: filter,
+            dateField,
+            targetDate,
+            lookback,
+            entityField,
+            entities,
             orderBy: dateField || entityField
         });
-        
+
         const rows = await this._execute(query);
-        
+
         if (!rows || rows.length === 0) return null;
-        
+
         return this._transform(rows, { lookback, dateField, entityField, dataField });
     }
 
     async *fetchBatched(options, batchSize = 1000) {
         const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
 
-        // FIX #3: Prevent Runaway Costs
         if (lookback > MAX_LOOKBACK_DAYS) {
-            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit of ${MAX_LOOKBACK_DAYS}. Table: ${table}`);
+            throw new Error(`[DataFetcher] COST GUARD: Lookback of ${lookback} days exceeds limit.`);
         }
 
         const tableConfig = this.tables[table] || {};
         const { dateField, entityField, dataField } = tableConfig;
+        const physicalTable = tableConfig.tableName || table;
 
-        // FIX #1: Prioritize ordering by Entity to keep historical rows together
         const query = await this.queryBuilder.build({
-            table, select: fields, where: filter, dateField, targetDate, lookback, entityField, entities,
-            orderBy: entityField || dateField 
+            table: physicalTable,
+            select: fields,
+            where: filter,
+            dateField,
+            targetDate,
+            lookback,
+            entityField,
+            entities,
+            orderBy: entityField || dateField
         });
 
         const rowStream = this._executeStream(query);
-        
+
         let batch = [];
         let currentEntity = null;
-        let batchHasWarned = false; // Flag to prevent log spam for a single massive batch
+        let batchHasWarned = false;
 
         for await (const row of rowStream) {
-            // FIX #2: Entity-Atomic Batching
-            // If we have an entity field, verify we don't split an entity across batches
             if (entityField) {
+
                 const rowEntity = String(row[entityField]);
-                
-                // Check if we should yield
-                // Condition: Batch is full AND we are on a NEW entity
+
                 if (batch.length >= batchSize && rowEntity !== currentEntity && currentEntity !== null) {
                     yield this._transform(batch, { lookback, dateField, entityField, dataField });
                     batch = [];
                     batchHasWarned = false;
                 }
-                
-                // SAFETY VALVE (Fix #6 Alternative):
-                // If batch grows huge (Super Entity) and we CANNOT split (same entity), warn the admin.
+
                 if (batch.length > batchSize * BATCH_GROWTH_WARNING_THRESHOLD && !batchHasWarned) {
-                    this._log('WARN', `SUPER ENTITY DETECTED: Entity '${currentEntity}' in table '${table}' has >${batch.length} rows. ` + 
-                              `This exceeds batch size ${batchSize} by ${BATCH_GROWTH_WARNING_THRESHOLD}x. ` +
-                              `Risk of OOM or Timeouts. Consider filtering this entity.`);
+                    this._log('WARN', `SUPER ENTITY DETECTED: Entity '${currentEntity}' in table '${table}' has >${batch.length} rows.`);
                     batchHasWarned = true;
                 }
-                
+
                 currentEntity = rowEntity;
             } else {
-                // Fallback for non-entity tables (strict count)
                 if (batch.length >= batchSize) {
                     yield this._transform(batch, { lookback, dateField, entityField, dataField });
                     batch = [];
                 }
             }
-            
             batch.push(row);
         }
-        
+
         if (batch.length > 0) {
             yield this._transform(batch, { lookback, dateField, entityField, dataField });
         }
     }
-    
+
     async hasData(table, targetDate) {
         const tableConfig = this.tables[table] || {};
         const { dateField } = tableConfig;
-        const query = await this.queryBuilder.buildExistsQuery(table, dateField, targetDate);
+        const physicalTable = tableConfig.tableName || table;
+
+        const query = await this.queryBuilder.buildExistsQuery(physicalTable, dateField, targetDate);
         try {
             const rows = await this._execute(query);
             return rows && rows.length > 0;
@@ -273,38 +334,34 @@ class DataFetcher {
             return false;
         }
     }
-    
+
+    // ... checkAvailability, getStats, resetStats, clearCache (unchanged) ...
     async checkAvailability(requires, targetDate) {
         const available = [];
         const missing = [];
-        
-        await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+
+        await Promise.all(Object.entries(requires).map(async ([key, spec]) => {
+            const tableName = spec.type === 'metric' ? spec.source : key;
             const hasData = await this.hasData(tableName, targetDate);
             if (hasData) {
-                available.push(tableName);
+                available.push(key);
             } else if (spec.mandatory) {
-                missing.push(tableName);
+                missing.push(key);
             } else {
-                available.push(tableName);
+                available.push(key);
             }
         }));
-        
+
         return { canRun: missing.length === 0, available, missing };
     }
 
-    
     getStats() { return { ...this.stats }; }
-    
-    resetStats() { 
-        this.stats = { 
-            queries: 0, 
-            rowsFetched: 0, 
-            errors: 0, 
-            bytesProcessed: 0,
-            cacheHits: 0,
-            cacheMisses: 0,
-            cacheEvictions: 0
-        }; 
+
+    resetStats() {
+        this.stats = {
+            queries: 0, rowsFetched: 0, errors: 0, bytesProcessed: 0,
+            cacheHits: 0, cacheMisses: 0, cacheEvictions: 0
+        };
         this.cache.clear();
     }
 
@@ -312,63 +369,80 @@ class DataFetcher {
         this.cache.clear();
         this._log('DEBUG', 'Query cache cleared');
     }
-    
+
     // =========================================================================
     // PRIVATE METHODS
     // =========================================================================
-    
+
+    /**
+     * Executes a query with Cost Guard protection.
+     */
     async _execute(query) {
-        // V2.6: Query Caching
+        // Caching Logic
         if (this.cacheConfig.enabled) {
             const cacheKey = this._generateCacheKey(query);
             const cached = this.cache.get(cacheKey);
-            
             if (cached) {
                 if (Date.now() - cached.timestamp < this.cacheConfig.ttlMs) {
                     this.stats.cacheHits++;
-                    // Refresh LRU position (delete and re-set moves to end)
                     this.cache.delete(cacheKey);
                     this.cache.set(cacheKey, cached);
-                    // Return cached rows immediately - no BigQuery cost
                     return cached.rows;
                 } else {
-                    this.cache.delete(cacheKey); // Expired
+                    this.cache.delete(cacheKey);
                 }
             }
             this.stats.cacheMisses++;
         }
 
+        // --- COST GUARD: DRY RUN ---
+        await this._enforceCostSafety(query);
+        // ---------------------------
+
+        // =========================================================
+        // 🛑 DEBUGGING: PRINT EXACT SQL
+        // =========================================================
+        console.log('\n--- 🔍 EXECUTING SQL -----------------------------------');
+        console.log(query.sql);
+        console.log('PARAMS:', JSON.stringify(query.params));
+        console.log('--------------------------------------------------------\n');
+        // =========================================================
+
         this.stats.queries++;
-        
+
         try {
             const [job] = await this.client.createQueryJob({
                 query: query.sql, params: query.params, location: this.location
             });
             const [rows] = await job.getQueryResults();
             const [metadata] = await job.getMetadata();
-            
+
             this.stats.rowsFetched += rows.length;
             this.stats.bytesProcessed += parseInt(metadata.statistics?.totalBytesProcessed || 0, 10);
-            
-            // FIX V2.7: Normalize Rows (Recursive JSON Parse) BEFORE caching
+
             const normalizedRows = rows.map(r => this._normalizeRow(r));
-            
-            // Store in cache if enabled
+
             if (this.cacheConfig.enabled) {
                 this._addToCache(query, normalizedRows);
             }
-            
+
             return normalizedRows;
         } catch (e) {
             this.stats.errors++;
-            this._log('ERROR', `Query failed: ${e.message}`);
+            this._log('ERROR', `Query failed: ${e.message}\nSQL: ${query.sql}`);
             throw e;
         }
     }
 
     async *_executeStream(query) {
-        // NOTE: We do NOT cache streams. They are typically massive datasets (batch processing)
-        // and caching them in memory would cause OOM.
+        await this._enforceCostSafety(query);
+
+        // [CRITICAL FIX] Added SQL Logging to Stream Execution
+        console.log('\n--- 🔍 EXECUTING DRIVER STREAM SQL ---------------------');
+        console.log(query.sql);
+        console.log('PARAMS:', JSON.stringify(query.params));
+        console.log('--------------------------------------------------------\n');
+
         this.stats.queries++;
         try {
             const [job] = await this.client.createQueryJob({
@@ -377,25 +451,52 @@ class DataFetcher {
             const stream = job.getQueryResultsStream();
             for await (const row of stream) {
                 this.stats.rowsFetched++;
-                // FIX V2.7: Normalize Rows (Recursive JSON Parse)
                 yield this._normalizeRow(row);
             }
         } catch (e) {
             this.stats.errors++;
-            this._log('ERROR', `Stream Query failed: ${e.message}`);
+            this._log('ERROR', `Stream Query failed: ${e.message}\nSQL: ${query.sql}`);
             throw e;
         }
     }
 
     /**
-     * V2.8 FIX: JSON Detection Logic
+     * NEW: Runs a Dry Run to estimate cost and blocks expensive queries.
      */
+    async _enforceCostSafety(query) {
+        try {
+            const [dryRunJob] = await this.client.createQueryJob({
+                query: query.sql,
+                params: query.params,
+                location: this.location,
+                dryRun: true // Costs $0, returns stats only
+            });
+
+            const bytes = parseInt(dryRunJob.metadata.statistics.totalBytesProcessed, 10);
+            const gb = bytes / (1024 * 1024 * 1024);
+
+            if (gb > this.safetyLimitGb) {
+                const errorMsg = `[DataFetcher] 🛑 COST VIOLATION: Query blocked! Estimated size: ${gb.toFixed(2)} GB (Limit: ${this.safetyLimitGb} GB). ` +
+                    `Check your partition filters or clustering. Table: ${query.table}`;
+
+                this._log('ERROR', errorMsg);
+                // We log the offending SQL for debugging
+                this._log('ERROR', `BLOCKED SQL START:\n${query.sql.substring(0, 500)}...\nEND SQL`);
+
+                throw new Error(errorMsg);
+            }
+        } catch (e) {
+            // If the dry run fails (e.g. invalid SQL), we let the real execution fail it naturally, 
+            // unless it was our cost violation error.
+            if (e.message.includes('COST VIOLATION')) throw e;
+        }
+    }
+
     _normalizeRow(row) {
         const normalized = { ...row };
         for (const [key, value] of Object.entries(normalized)) {
             if (typeof value === 'string') {
                 const trimmed = value.trim();
-                // FIX: Check for " (Double Encoded JSON) in addition to { and [
                 if (trimmed.startsWith('{') || trimmed.startsWith('[') || trimmed.startsWith('"')) {
                     normalized[key] = this._safeRecursiveParse(value);
                 }
@@ -404,75 +505,45 @@ class DataFetcher {
         return normalized;
     }
 
-    /**
-     * V2.7 FIX: Helper to safely recursively parse JSON.
-     * Handles: Double-Encoded JSON Strings (parsed recursively)
-     */
-   _safeRecursiveParse(input) {
+    _safeRecursiveParse(input) {
         if (!input) return null;
         if (typeof input === 'object') return input;
         try {
             const parsed = JSON.parse(input);
-            // Recursion for double-encoded strings
             if (typeof parsed === 'string') return this._safeRecursiveParse(parsed);
             return parsed;
         } catch (e) {
-            return input; // Not JSON, return original
+            return input;
         }
     }
 
-    /**
-     * V2.6: Generate a unique cache key for a query
-     */
     _generateCacheKey(query) {
-        // Hash the SQL + Params to ensure uniqueness
         const str = query.sql + JSON.stringify(query.params || {});
         return crypto.createHash('md5').update(str).digest('hex');
     }
 
-    /**
-     * V2.6: Add to cache with LRU eviction
-     */
     _addToCache(query, rows) {
-        // Generate key
         const key = this._generateCacheKey(query);
-        
-        // Eviction Logic
         if (this.cache.size >= this.cacheConfig.maxSize) {
-            // Map iterator yields in insertion order. First item is oldest.
             const oldestKey = this.cache.keys().next().value;
             this.cache.delete(oldestKey);
             this.stats.cacheEvictions++;
         }
-        
-        this.cache.set(key, {
-            rows: rows,
-            timestamp: Date.now()
-        });
+        this.cache.set(key, { rows: rows, timestamp: Date.now() });
     }
-    
-    /**
-     * Transforms raw rows into a structured object.
-     * FIX: PRIORITIZE ENTITY FIELD.
-     * If entityField exists, we MUST return { [id]: [rows] } so IDs can be extracted.
-     */
+
     _transform(rows, config) {
         const { lookback, dateField, entityField, dataField } = config;
         const rowArray = Array.isArray(rows) ? rows : [rows];
 
-        // FIX: Primary Grouping = Entity
         if (entityField) {
             const byEntity = {};
             for (const row of rowArray) {
                 const entityKey = String(row[entityField]);
                 if (!byEntity[entityKey]) {
-                    // If simple fetch (no history), value is single object
-                    // If history fetch (lookback), value is Array of rows
                     byEntity[entityKey] = lookback > 0 ? [] : null;
                 }
-                
                 const value = dataField ? row[dataField] : row;
-                
                 if (Array.isArray(byEntity[entityKey])) {
                     byEntity[entityKey].push(value);
                 } else {
@@ -482,7 +553,6 @@ class DataFetcher {
             return byEntity;
         }
 
-        // Fallback: Date Grouping (Only if no Entity ID)
         if (lookback > 0 && dateField) {
             const byDate = {};
             for (const row of rowArray) {
@@ -493,12 +563,19 @@ class DataFetcher {
             }
             return byDate;
         }
-        
         return rowArray;
     }
 
     _getDriverTable(requires) {
+        // PASS 1: Prioritize tables with a DATE field
+        for (const [name, spec] of Object.entries(requires)) {
+            if (spec.type === 'metric') continue;
+            const config = this.tables[name];
+            if (config && config.entityField && config.dateField) return name;
+        }
+        // PASS 2: Fallback to any entity table
         for (const [name, spec] of Object.entries(requires)) {
+            if (spec.type === 'metric') continue;
             const config = this.tables[name];
             if (config && config.entityField) return name;
         }
@@ -509,25 +586,14 @@ class DataFetcher {
         const config = this.tables[tableName] || {};
         const field = config.entityField;
 
-        // Case 1: Transformed Object { "id1": data, "id2": data }
         if (field && batchData && !Array.isArray(batchData)) {
             return Object.keys(batchData);
         }
-        
-        // Case 2: Array of Rows (Only if _transform didn't group by entity)
         if (Array.isArray(batchData) && field) {
             const ids = [];
-            let undefinedCount = 0;
             for (const r of batchData) {
                 const val = r[field];
-                if (val === undefined) {
-                    undefinedCount++;
-                } else {
-                    ids.push(String(val));
-                }
-            }
-            if (undefinedCount > 0) {
-                this._log('ERROR', `CRITICAL CONFIG ERROR: Found ${undefinedCount} rows in '${tableName}' where entityField '${field}' was UNDEFINED.`);
+                if (val !== undefined) ids.push(String(val));
             }
             return ids;
         }
@@ -537,28 +603,28 @@ class DataFetcher {
     _extractAllEntityIds(fullData) {
         const ids = new Set();
         Object.values(fullData || {}).forEach(tableData => {
-             if (tableData && typeof tableData === 'object' && !Array.isArray(tableData)) {
-                 Object.keys(tableData).forEach(k => ids.add(k));
-             }
+            if (tableData && typeof tableData === 'object' && !Array.isArray(tableData)) {
+                Object.keys(tableData).forEach(k => ids.add(k));
+            }
         });
         return Array.from(ids);
     }
-    
+
     _formatDate(dateValue) {
         if (!dateValue) return null;
         if (typeof dateValue === 'string') return dateValue;
-        if (dateValue.value) return dateValue.value; 
+        if (dateValue.value) return dateValue.value;
         if (dateValue instanceof Date) return dateValue.toISOString().slice(0, 10);
         return String(dateValue);
     }
-    
+
     _isEmpty(data) {
         if (data == null) return true;
         if (Array.isArray(data)) return data.length === 0;
         if (typeof data === 'object') return Object.keys(data).length === 0;
         return false;
     }
-    
+
     _log(level, message) {
         if (this.logger && typeof this.logger.log === 'function') {
             this.logger.log(level, `[DataFetcher] ${message}`);