bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/data/SchemaRegistry.js

@@ -0,0 +1,287 @@
+/**
+ * @fileoverview Schema Registry - Dynamic schema discovery with caching
+ * 
+ * Core innovation of v2: No hardcoded schemas. Instead, we:
+ * 1. Fetch schemas from BigQuery INFORMATION_SCHEMA on first access
+ * 2. Cache them with configurable TTL
+ * 3. Validate all queries against cached schemas BEFORE sending to BigQuery
+ * 
+ * This prevents runtime query failures and eliminates schema maintenance burden.
+ */
+
+const { BigQuery } = require('@google-cloud/bigquery');
+
+/**
+ * @typedef {Object} ColumnInfo
+ * @property {string} name - Column name
+ * @property {string} type - BigQuery data type (STRING, INT64, DATE, JSON, etc.)
+ * @property {boolean} nullable - Whether column allows NULL
+ * @property {number} position - Ordinal position in table
+ */
+
+/**
+ * @typedef {Object} TableSchema
+ * @property {string} tableName - Table name
+ * @property {ColumnInfo[]} columns - Array of column definitions
+ * @property {Set<string>} columnNames - Set of column names for fast lookup
+ * @property {number} fetchedAt - Timestamp when schema was fetched
+ */
+
+class SchemaRegistry {
+    /**
+     * @param {Object} config - Configuration object
+     * @param {string} config.projectId - GCP project ID
+     * @param {string} config.dataset - BigQuery dataset name
+     * @param {string} [config.location='US'] - BigQuery location
+     * @param {number} [config.cacheTTLMs=3600000] - Schema cache TTL in milliseconds (default: 1 hour)
+     * @param {Object} [logger] - Logger instance
+     */
+    constructor(config, logger = null) {
+        this.projectId = config.projectId;
+        this.dataset = config.dataset;
+        this.location = config.location || 'US';
+        this.cacheTTLMs = config.cacheTTLMs || 3600000; // 1 hour default
+        this.logger = logger;
+        
+        this.client = new BigQuery({ projectId: this.projectId });
+        this.cache = new Map();
+        
+        // Track schema fetch stats for monitoring
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            refreshes: 0,
+            errors: 0
+        };
+    }
+    
+    /**
+     * Get schema for a table, fetching from BigQuery if not cached.
+     * @param {string} tableName - Table name (without dataset prefix)
+     * @returns {Promise<TableSchema>}
+     */
+    async getSchema(tableName) {
+        const cached = this.cache.get(tableName);
+        
+        if (cached && !this._isExpired(cached)) {
+            this.stats.hits++;
+            return cached;
+        }
+        
+        if (cached) {
+            this.stats.refreshes++;
+            this._log('DEBUG', `Schema cache expired for ${tableName}, refreshing...`);
+        } else {
+            this.stats.misses++;
+            this._log('DEBUG', `Schema cache miss for ${tableName}, fetching...`);
+        }
+        
+        return await this._fetchAndCacheSchema(tableName);
+    }
+    
+    /**
+     * Check if a table exists in the dataset.
+     * @param {string} tableName - Table name
+     * @returns {Promise<boolean>}
+     */
+    async tableExists(tableName) {
+        try {
+            await this.getSchema(tableName);
+            return true;
+        } catch (e) {
+            if (e.message.includes('not found') || e.message.includes('Not found')) {
+                return false;
+            }
+            throw e;
+        }
+    }
+    
+    /**
+     * Validate that columns exist in a table's schema.
+     * Throws if any column doesn't exist.
+     * @param {string} tableName - Table name
+     * @param {string[]} columns - Column names to validate
+     * @throws {Error} If any column doesn't exist
+     */
+    async validateColumns(tableName, columns) {
+        const schema = await this.getSchema(tableName);
+        const invalid = columns.filter(col => !schema.columnNames.has(col));
+        
+        if (invalid.length > 0) {
+            throw new Error(
+                `[SchemaRegistry] Invalid columns for table '${tableName}': ${invalid.join(', ')}. ` +
+                `Valid columns: ${Array.from(schema.columnNames).join(', ')}`
+            );
+        }
+    }
+    
+    /**
+     * Validate a WHERE clause object against table schema.
+     * @param {string} tableName - Table name
+     * @param {Object} whereClause - Object of { columnName: value } conditions
+     * @throws {Error} If any column in the where clause doesn't exist
+     */
+    async validateWhere(tableName, whereClause) {
+        if (!whereClause || typeof whereClause !== 'object') return;
+        await this.validateColumns(tableName, Object.keys(whereClause));
+    }
+    
+    /**
+     * Get column type for a specific column.
+     * @param {string} tableName - Table name
+     * @param {string} columnName - Column name
+     * @returns {Promise<string|null>} Column type or null if not found
+     */
+    async getColumnType(tableName, columnName) {
+        const schema = await this.getSchema(tableName);
+        const column = schema.columns.find(c => c.name === columnName);
+        return column ? column.type : null;
+    }
+    
+    /**
+     * Check if a table has a specific column.
+     * @param {string} tableName - Table name
+     * @param {string} columnName - Column name
+     * @returns {Promise<boolean>}
+     */
+    async hasColumn(tableName, columnName) {
+        const schema = await this.getSchema(tableName);
+        return schema.columnNames.has(columnName);
+    }
+    
+    /**
+     * Pre-warm the cache by fetching schemas for specified tables.
+     * Useful at startup to avoid cold-start latency.
+     * @param {string[]} tableNames - Tables to pre-warm
+     * @returns {Promise<Object>} Summary of pre-warm results
+     */
+    async warmCache(tableNames) {
+        const results = { success: [], failed: [] };
+        
+        await Promise.all(tableNames.map(async (tableName) => {
+            try {
+                await this.getSchema(tableName);
+                results.success.push(tableName);
+            } catch (e) {
+                results.failed.push({ table: tableName, error: e.message });
+            }
+        }));
+        
+        this._log('INFO', `Schema cache warmed: ${results.success.length} success, ${results.failed.length} failed`);
+        return results;
+    }
+    
+    /**
+     * Clear the schema cache (useful for testing or after schema changes).
+     * @param {string} [tableName] - Specific table to clear, or all if not specified
+     */
+    clearCache(tableName = null) {
+        if (tableName) {
+            this.cache.delete(tableName);
+            this._log('DEBUG', `Cleared schema cache for ${tableName}`);
+        } else {
+            this.cache.clear();
+            this._log('DEBUG', 'Cleared entire schema cache');
+        }
+    }
+    
+    /**
+     * Get cache statistics.
+     * @returns {Object} Cache statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            cachedTables: this.cache.size,
+            cacheContents: Array.from(this.cache.keys())
+        };
+    }
+    
+    /**
+     * List all tables in the dataset.
+     * @returns {Promise<string[]>} Array of table names
+     */
+    async listTables() {
+        const query = `
+            SELECT table_name
+            FROM \`${this.projectId}.${this.dataset}.INFORMATION_SCHEMA.TABLES\`
+            WHERE table_type = 'BASE TABLE'
+            ORDER BY table_name
+        `;
+        
+        try {
+            const [rows] = await this.client.query({ query, location: this.location });
+            return rows.map(r => r.table_name);
+        } catch (e) {
+            this._log('ERROR', `Failed to list tables: ${e.message}`);
+            throw e;
+        }
+    }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+    
+    async _fetchAndCacheSchema(tableName) {
+        const query = `
+            SELECT 
+                column_name,
+                data_type,
+                is_nullable,
+                ordinal_position
+            FROM \`${this.projectId}.${this.dataset}.INFORMATION_SCHEMA.COLUMNS\`
+            WHERE table_name = @tableName
+            ORDER BY ordinal_position
+        `;
+        
+        try {
+            const [rows] = await this.client.query({
+                query,
+                params: { tableName },
+                location: this.location
+            });
+            
+            if (rows.length === 0) {
+                throw new Error(`Table '${tableName}' not found in dataset '${this.dataset}'`);
+            }
+            
+            const columns = rows.map(r => ({
+                name: r.column_name,
+                type: r.data_type,
+                nullable: r.is_nullable === 'YES',
+                position: r.ordinal_position
+            }));
+            
+            const schema = {
+                tableName,
+                columns,
+                columnNames: new Set(columns.map(c => c.name)),
+                fetchedAt: Date.now()
+            };
+            
+            this.cache.set(tableName, schema);
+            this._log('INFO', `Cached schema for '${tableName}': ${columns.length} columns`);
+            
+            return schema;
+            
+        } catch (e) {
+            this.stats.errors++;
+            this._log('ERROR', `Failed to fetch schema for '${tableName}': ${e.message}`);
+            throw e;
+        }
+    }
+    
+    _isExpired(schema) {
+        return Date.now() - schema.fetchedAt > this.cacheTTLMs;
+    }
+    
+    _log(level, message) {
+        if (this.logger && typeof this.logger.log === 'function') {
+            this.logger.log(level, `[SchemaRegistry] ${message}`);
+        } else if (level === 'ERROR') {
+            console.error(`[SchemaRegistry] ${message}`);
+        }
+    }
+}
+
+module.exports = { SchemaRegistry };