bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/data/DataFetcher.js

@@ -0,0 +1,399 @@
+/**
+ * @fileoverview Data Fetcher - Executes queries and transforms results
+ * * The single point of data access for computations.
+ * Uses QueryBuilder for validation, executes against BigQuery, transforms results.
+ * * V2.1 UPDATE: Added Streaming/Batching support to fix "Memory God" issues.
+ * Now supports O(1) memory usage for large datasets via fetchBatched.
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+
+/**
+ * @typedef {Object} FetchOptions
+ * @property {string} table - Table name
+ * @property {string} targetDate - Target date (YYYY-MM-DD)
+ * @property {number} [lookback=0] - Days to look back
+ * @property {boolean} [mandatory=false] - If true, throws on no data
+ * @property {Object} [filter] - Additional WHERE conditions
+ * @property {string[]} [fields] - Specific fields to select
+ * @property {string[]} [entities] - Specific entity IDs to fetch
+ */
+
+class DataFetcher {
+    /**
+     * @param {Object} config - Configuration object
+     * @param {string} config.projectId - GCP project ID
+     * @param {string} config.dataset - BigQuery dataset name
+     * @param {string} [config.location='US'] - BigQuery location
+     * @param {Object} config.tables - Table configuration map
+     * @param {QueryBuilder} queryBuilder - Query builder instance
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, queryBuilder, logger = null) {
+        this.projectId = config.projectId;
+        this.dataset = config.dataset;
+        this.location = config.location || 'US';
+        this.tables = config.tables || {};
+        this.queryBuilder = queryBuilder;
+        this.logger = logger;
+        
+        this.client = new BigQuery({ projectId: this.projectId });
+        
+        // Query stats
+        this.stats = {
+            queries: 0,
+            rowsFetched: 0,
+            errors: 0,
+            bytesProcessed: 0
+        };
+    }
+    
+    /**
+     * Fetch data for a computation's requirements.
+     * @param {Object} requires - Computation's requires declaration
+     * @param {string} targetDate - Target date
+     * @param {string[]} [entities] - Optional entity IDs to filter
+     * @returns {Promise<Object>} Data keyed by table name
+     */
+    async fetchForComputation(requires, targetDate, entities = null) {
+        const results = {};
+        const errors = [];
+        
+        // Fetch all tables in parallel
+        await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+            try {
+                const data = await this.fetch({
+                    table: tableName,
+                    targetDate,
+                    lookback: spec.lookback || 0,
+                    mandatory: spec.mandatory || false,
+                    filter: spec.filter || {},
+                    fields: spec.fields || null,
+                    entities: entities
+                });
+                
+                results[tableName] = data;
+                
+                if (spec.mandatory && this._isEmpty(data)) {
+                    errors.push({ table: tableName, reason: 'MANDATORY_MISSING' });
+                }
+                
+            } catch (e) {
+                if (spec.mandatory) {
+                    errors.push({ table: tableName, reason: e.message });
+                } else {
+                    this._log('WARN', `Optional table ${tableName} failed: ${e.message}`);
+                }
+                results[tableName] = null;
+            }
+        }));
+        
+        // Throw if mandatory data is missing
+        if (errors.length > 0) {
+            const msg = errors.map(e => `${e.table}: ${e.reason}`).join(', ');
+            throw new Error(`[DataFetcher] Missing mandatory data: ${msg}`);
+        }
+        
+        return results;
+    }
+
+    /**
+     * STREAMING: Fetch data for computation in batches.
+     * Uses a "Driver Table" (one with entityField) to stream entities, 
+     * then fetches dependencies for just that batch.
+     * * @param {Object} requires - Computation requires
+     * @param {string} targetDate - Target Date
+     * @param {number} batchSize - Rows per batch
+     * @returns {AsyncGenerator<{data: Object, entityIds: string[]}>}
+     */
+    async *fetchComputationBatched(requires, targetDate, batchSize = 1000) {
+        // 1. Identify Driver Table (The one we iterate over)
+        const driverTableName = this._getDriverTable(requires);
+        
+        if (!driverTableName) {
+            this._log('WARN', 'No entity-keyed table found for batching. Falling back to full fetch.');
+            const fullData = await this.fetchForComputation(requires, targetDate);
+            yield { data: fullData, entityIds: null };
+            return;
+        }
+
+        this._log('INFO', `Starting batched fetch driven by table: ${driverTableName}`);
+        const driverSpec = requires[driverTableName];
+
+        // 2. Stream Driver Table
+        const driverStream = this.fetchBatched({
+            table: driverTableName,
+            targetDate,
+            lookback: driverSpec.lookback || 0,
+            filter: driverSpec.filter || {},
+            fields: driverSpec.fields,
+            mandatory: driverSpec.mandatory
+        }, batchSize);
+
+        // 3. Process Streams
+        for await (const batch of driverStream) {
+            // Extract Entity IDs from this batch
+            // The batch is already transformed (e.g. Object by EntityID or Array)
+            const entityIds = this._extractEntityIdsFromBatch(batch, driverTableName);
+            
+            if (entityIds.length === 0) continue;
+
+            const batchResults = { [driverTableName]: batch };
+            const errors = [];
+
+            // 4. Fetch Dependencies for this SPECIFIC batch of entities
+            await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+                if (tableName === driverTableName) return; // Already have this
+
+                try {
+                    const data = await this.fetch({
+                        table: tableName,
+                        targetDate,
+                        lookback: spec.lookback || 0,
+                        mandatory: spec.mandatory || false,
+                        filter: spec.filter || {},
+                        fields: spec.fields || null,
+                        entities: entityIds // <--- CRITICAL: Filter by current batch
+                    });
+                    
+                    batchResults[tableName] = data;
+
+                    if (spec.mandatory && this._isEmpty(data)) {
+                        errors.push({ table: tableName, reason: 'MANDATORY_MISSING_IN_BATCH' });
+                    }
+                } catch (e) {
+                    if (spec.mandatory) errors.push({ table: tableName, reason: e.message });
+                    batchResults[tableName] = null;
+                }
+            }));
+
+            if (errors.length > 0) {
+                this._log('WARN', `Batch missing mandatory data: ${errors.map(e => e.table).join(', ')}. Skipping batch.`);
+                continue; 
+            }
+
+            yield { data: batchResults, entityIds };
+        }
+    }
+    
+    /**
+     * Fetch data from a single table (Full Load).
+     * @param {FetchOptions} options - Fetch options
+     * @returns {Promise<Object|Array|null>} Transformed data
+     */
+    async fetch(options) {
+        const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
+        
+        const tableConfig = this.tables[table] || {};
+        const { dateField, entityField, dataField } = tableConfig;
+        
+        const query = await this.queryBuilder.build({
+            table, select: fields, where: filter, dateField, targetDate, lookback, entityField, entities,
+            orderBy: dateField || entityField
+        });
+        
+        const rows = await this._execute(query);
+        
+        if (!rows || rows.length === 0) return null;
+        
+        return this._transform(rows, { lookback, dateField, entityField, dataField });
+    }
+
+    /**
+     * STREAMING: Fetch data from a single table in batches.
+     * @param {FetchOptions} options 
+     * @param {number} batchSize 
+     * @returns {AsyncGenerator<Object|Array>}
+     */
+    async *fetchBatched(options, batchSize = 1000) {
+        const { table, targetDate, lookback = 0, filter = {}, fields = null, entities = null } = options;
+        const tableConfig = this.tables[table] || {};
+        const { dateField, entityField, dataField } = tableConfig;
+
+        const query = await this.queryBuilder.build({
+            table, select: fields, where: filter, dateField, targetDate, lookback, entityField, entities,
+            orderBy: dateField || entityField
+        });
+
+        // Use Stream Executor
+        const rowStream = this._executeStream(query);
+        
+        let batch = [];
+        for await (const row of rowStream) {
+            batch.push(row);
+            if (batch.length >= batchSize) {
+                yield this._transform(batch, { lookback, dateField, entityField, dataField });
+                batch = [];
+            }
+        }
+        
+        if (batch.length > 0) {
+            yield this._transform(batch, { lookback, dateField, entityField, dataField });
+        }
+    }
+    
+    /**
+     * Check if data exists for a table on a given date.
+     */
+    async hasData(table, targetDate) {
+        const tableConfig = this.tables[table] || {};
+        const { dateField } = tableConfig;
+        const query = await this.queryBuilder.buildExistsQuery(table, dateField, targetDate);
+        try {
+            const rows = await this._execute(query);
+            return rows && rows.length > 0;
+        } catch (e) {
+            this._log('WARN', `hasData check failed for ${table}: ${e.message}`);
+            return false;
+        }
+    }
+    
+    /**
+     * Check availability for multiple tables.
+     */
+    async checkAvailability(requires, targetDate) {
+        const available = [];
+        const missing = [];
+        
+        await Promise.all(Object.entries(requires).map(async ([tableName, spec]) => {
+            const hasData = await this.hasData(tableName, targetDate);
+            if (hasData) {
+                available.push(tableName);
+            } else if (spec.mandatory) {
+                missing.push(tableName);
+            } else {
+                available.push(tableName);
+            }
+        }));
+        
+        return { canRun: missing.length === 0, available, missing };
+    }
+    
+    getStats() { return { ...this.stats }; }
+    resetStats() { this.stats = { queries: 0, rowsFetched: 0, errors: 0, bytesProcessed: 0 }; }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+    
+    async _execute(query) {
+        this.stats.queries++;
+        try {
+            this._log('DEBUG', `Executing (Full): ${query.sql.substring(0, 100)}...`);
+            const [job] = await this.client.createQueryJob({
+                query: query.sql, params: query.params, location: this.location
+            });
+            const [rows] = await job.getQueryResults();
+            
+            const [metadata] = await job.getMetadata();
+            const bytesProcessed = parseInt(metadata.statistics?.totalBytesProcessed || 0, 10);
+            this.stats.rowsFetched += rows.length;
+            this.stats.bytesProcessed += bytesProcessed;
+            
+            return rows;
+        } catch (e) {
+            this.stats.errors++;
+            this._log('ERROR', `Query failed: ${e.message}`);
+            throw e;
+        }
+    }
+
+    // New Stream Executor using BigQuery Streams
+    async *_executeStream(query) {
+        this.stats.queries++;
+        try {
+            this._log('DEBUG', `Executing (Stream): ${query.sql.substring(0, 100)}...`);
+            const [job] = await this.client.createQueryJob({
+                query: query.sql, params: query.params, location: this.location
+            });
+            
+            const stream = job.getQueryResultsStream();
+            
+            for await (const row of stream) {
+                this.stats.rowsFetched++;
+                yield row;
+            }
+        } catch (e) {
+            this.stats.errors++;
+            this._log('ERROR', `Stream Query failed: ${e.message}`);
+            throw e;
+        }
+    }
+    
+    _transform(rows, config) {
+        const { lookback, dateField, entityField, dataField } = config;
+        
+        // Ensure rows is array
+        const rowArray = Array.isArray(rows) ? rows : [rows];
+
+        // For lookback > 0, return date-keyed object
+        if (lookback > 0 && dateField) {
+            const byDate = {};
+            for (const row of rowArray) {
+                const dateKey = this._formatDate(row[dateField]);
+                if (!byDate[dateKey]) byDate[dateKey] = [];
+                const value = dataField ? row[dataField] : row;
+                byDate[dateKey].push(value);
+            }
+            return byDate;
+        }
+        
+        // For entity-keyed tables, return entity-keyed object
+        if (entityField) {
+            const byEntity = {};
+            for (const row of rowArray) {
+                const entityKey = String(row[entityField]);
+                byEntity[entityKey] = dataField ? row[dataField] : row;
+            }
+            return byEntity;
+        }
+        
+        // Default: return array
+        return rowArray;
+    }
+
+    _getDriverTable(requires) {
+        for (const [name, spec] of Object.entries(requires)) {
+            const config = this.tables[name];
+            if (config && config.entityField) return name;
+        }
+        return null;
+    }
+
+    _extractEntityIdsFromBatch(batchData, tableName) {
+        const config = this.tables[tableName] || {};
+        if (config.entityField && batchData && !Array.isArray(batchData)) {
+            return Object.keys(batchData);
+        }
+        // If it's an array, we need to map
+        if (Array.isArray(batchData) && config.entityField) {
+            return batchData.map(r => String(r[config.entityField]));
+        }
+        return [];
+    }
+    
+    _formatDate(dateValue) {
+        if (!dateValue) return null;
+        if (typeof dateValue === 'string') return dateValue;
+        if (dateValue.value) return dateValue.value; 
+        if (dateValue instanceof Date) return dateValue.toISOString().slice(0, 10);
+        return String(dateValue);
+    }
+    
+    _isEmpty(data) {
+        if (data == null) return true;
+        if (Array.isArray(data)) return data.length === 0;
+        if (typeof data === 'object') return Object.keys(data).length === 0;
+        return false;
+    }
+    
+    _log(level, message) {
+        if (this.logger && typeof this.logger.log === 'function') {
+            this.logger.log(level, `[DataFetcher] ${message}`);
+        } else if (level === 'ERROR') {
+            console.error(`[DataFetcher] ${message}`);
+        }
+    }
+}
+
+module.exports = { DataFetcher };

package/functions/computation-system-v2/framework/data/QueryBuilder.js

@@ -0,0 +1,232 @@
+/**
+ * @fileoverview Query Builder - Builds validated SQL queries
+ * 
+ * All queries are validated against the SchemaRegistry BEFORE execution.
+ * This catches errors early and prevents runtime failures.
+ */
+
+/**
+ * @typedef {Object} QuerySpec
+ * @property {string} table - Table name
+ * @property {string[]} [select] - Columns to select (null = *)
+ * @property {Object} [where] - WHERE conditions as { column: value }
+ * @property {string} [dateField] - Date field name for date filtering
+ * @property {string} [targetDate] - Target date (YYYY-MM-DD)
+ * @property {number} [lookback=0] - Days to look back (0 = target date only)
+ * @property {string} [entityField] - Entity field name for entity filtering
+ * @property {string[]} [entities] - Entity IDs to filter
+ * @property {string} [orderBy] - Column to order by
+ * @property {string} [orderDir='DESC'] - Order direction
+ * @property {number} [limit] - Limit number of rows
+ */
+
+/**
+ * @typedef {Object} BuiltQuery
+ * @property {string} sql - The SQL query string
+ * @property {Object} params - Query parameters
+ * @property {string} table - Table that was queried
+ * @property {Object} spec - Original query spec
+ */
+
+class QueryBuilder {
+    /**
+     * @param {Object} config - Configuration object
+     * @param {string} config.projectId - GCP project ID
+     * @param {string} config.dataset - BigQuery dataset name
+     * @param {SchemaRegistry} schemaRegistry - Schema registry instance
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, schemaRegistry, logger = null) {
+        this.projectId = config.projectId;
+        this.dataset = config.dataset;
+        this.schemaRegistry = schemaRegistry;
+        this.logger = logger;
+    }
+    
+    /**
+     * Build a validated query.
+     * @param {QuerySpec} spec - Query specification
+     * @returns {Promise<BuiltQuery>}
+     * @throws {Error} If validation fails
+     */
+    async build(spec) {
+        const { table } = spec;
+        
+        if (!table) {
+            throw new Error('[QueryBuilder] Table name is required');
+        }
+        
+        // Validate table exists and get schema
+        const schema = await this.schemaRegistry.getSchema(table);
+        
+        // Validate SELECT columns
+        if (spec.select && spec.select.length > 0) {
+            await this.schemaRegistry.validateColumns(table, spec.select);
+        }
+        
+        // Validate WHERE columns
+        if (spec.where) {
+            await this.schemaRegistry.validateWhere(table, spec.where);
+        }
+        
+        // Validate date field exists
+        if (spec.dateField) {
+            const hasDateField = await this.schemaRegistry.hasColumn(table, spec.dateField);
+            if (!hasDateField) {
+                throw new Error(`[QueryBuilder] Date field '${spec.dateField}' not found in table '${table}'`);
+            }
+        }
+        
+        // Validate entity field exists
+        if (spec.entityField) {
+            const hasEntityField = await this.schemaRegistry.hasColumn(table, spec.entityField);
+            if (!hasEntityField) {
+                throw new Error(`[QueryBuilder] Entity field '${spec.entityField}' not found in table '${table}'`);
+            }
+        }
+        
+        // Validate orderBy column
+        if (spec.orderBy) {
+            const hasOrderBy = await this.schemaRegistry.hasColumn(table, spec.orderBy);
+            if (!hasOrderBy) {
+                throw new Error(`[QueryBuilder] Order by field '${spec.orderBy}' not found in table '${table}'`);
+            }
+        }
+        
+        // Build the query
+        return this._buildSQL(spec, schema);
+    }
+    
+    /**
+     * Build a simple existence check query.
+     * @param {string} table - Table name
+     * @param {string} dateField - Date field name
+     * @param {string} targetDate - Target date
+     * @returns {Promise<BuiltQuery>}
+     */
+    async buildExistsQuery(table, dateField, targetDate) {
+        // Validate table and date field
+        await this.schemaRegistry.getSchema(table);
+        if (dateField) {
+            await this.schemaRegistry.validateColumns(table, [dateField]);
+        }
+        
+        const tablePath = `\`${this.projectId}.${this.dataset}.${table}\``;
+        
+        let sql, params;
+        if (dateField) {
+            sql = `SELECT 1 FROM ${tablePath} WHERE ${dateField} = @targetDate LIMIT 1`;
+            params = { targetDate };
+        } else {
+            sql = `SELECT 1 FROM ${tablePath} LIMIT 1`;
+            params = {};
+        }
+        
+        return { sql, params, table, spec: { table, dateField, targetDate } };
+    }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+    
+    _buildSQL(spec, schema) {
+        const {
+            table,
+            select = null,
+            where = {},
+            dateField = null,
+            targetDate = null,
+            lookback = 0,
+            entityField = null,
+            entities = null,
+            orderBy = null,
+            orderDir = 'DESC',
+            limit = null
+        } = spec;
+        
+        const tablePath = `\`${this.projectId}.${this.dataset}.${table}\``;
+        const params = {};
+        const conditions = [];
+        
+        // SELECT clause
+        const selectClause = select && select.length > 0 
+            ? select.join(', ')
+            : '*';
+        
+        // Date filtering
+        if (dateField && targetDate) {
+            if (lookback === 0) {
+                conditions.push(`${dateField} = @targetDate`);
+                params.targetDate = targetDate;
+            } else {
+                conditions.push(`${dateField} BETWEEN DATE_SUB(@targetDate, INTERVAL ${lookback} DAY) AND @targetDate`);
+                params.targetDate = targetDate;
+            }
+        }
+        
+        // Entity filtering
+        if (entityField && entities && entities.length > 0) {
+            conditions.push(`${entityField} IN UNNEST(@entities)`);
+            // Detect if entity field is numeric based on schema
+            const entityColumn = schema.columns.find(c => c.name === entityField);
+            if (entityColumn && this._isNumericType(entityColumn.type)) {
+                params.entities = entities.map(e => parseInt(e, 10));
+            } else {
+                params.entities = entities.map(String);
+            }
+        }
+        
+        // Additional WHERE conditions
+        let whereIndex = 0;
+        for (const [column, value] of Object.entries(where)) {
+            const paramName = `where_${whereIndex++}`;
+            
+            if (Array.isArray(value)) {
+                conditions.push(`${column} IN UNNEST(@${paramName})`);
+                params[paramName] = value;
+            } else if (value === null) {
+                conditions.push(`${column} IS NULL`);
+            } else {
+                conditions.push(`${column} = @${paramName}`);
+                params[paramName] = value;
+            }
+        }
+        
+        // Build WHERE clause
+        const whereClause = conditions.length > 0
+            ? `WHERE ${conditions.join(' AND ')}`
+            : '';
+        
+        // ORDER BY clause
+        const orderClause = orderBy
+            ? `ORDER BY ${orderBy} ${orderDir}`
+            : '';
+        
+        // LIMIT clause
+        const limitClause = limit
+            ? `LIMIT ${limit}`
+            : '';
+        
+        const sql = [
+            `SELECT ${selectClause}`,
+            `FROM ${tablePath}`,
+            whereClause,
+            orderClause,
+            limitClause
+        ].filter(Boolean).join('\n');
+        
+        return {
+            sql,
+            params,
+            table,
+            spec
+        };
+    }
+    
+    _isNumericType(type) {
+        const numericTypes = ['INT64', 'FLOAT64', 'NUMERIC', 'BIGNUMERIC', 'INTEGER', 'FLOAT'];
+        return numericTypes.includes(type.toUpperCase());
+    }
+}
+
+module.exports = { QueryBuilder };