bulltrackers-module 1.0.768 → 1.0.769

package/functions/computation-system-v2/framework/data/MaterializedViewManager.js

@@ -0,0 +1,84 @@
+/**
+ * @fileoverview Materialized View Manager
+ * * V2.1 UPDATE: Added verbose logging for long-running MV creation operations.
+ */
+class MaterializedViewManager {
+    constructor(bigqueryClient, logger, config) {
+        this.bq = bigqueryClient;
+        this.logger = logger || console;
+        this.config = config;
+        this.checkedViews = new Set();
+    }
+
+    async ensureMetricView(baseTable, spec) {
+        const suffix = spec.jsonPath ? `_${this._hash(spec.jsonPath)}` : '';
+        const viewName = `mv_${baseTable}_${spec.func}_${spec.field}${suffix}_by_day`.toLowerCase();
+
+        if (this.checkedViews.has(viewName)) return viewName;
+
+        const dataset = this.bq.dataset(this.config.dataset);
+        const table = dataset.table(viewName);
+
+        try {
+            const [exists] = await table.exists();
+            if (exists) {
+                this.checkedViews.add(viewName);
+                return viewName;
+            }
+        } catch (e) {
+            // Ignore 404s
+        }
+
+        await this._createView(dataset, viewName, baseTable, spec);
+        this.checkedViews.add(viewName);
+        return viewName;
+    }
+
+    async _createView(dataset, viewName, baseTable, spec) {
+        // LOGGING ADDED HERE
+        this.logger.log('INFO', `[MV-Manager] ⏳ Creating Materialized View: ${viewName}...`);
+        this.logger.log('INFO', `[MV-Manager] ⚠️ This triggers a historical backfill and may take 30-60 seconds. Please wait.`);
+
+        const { field, func, entityField, dateField, jsonPath } = spec;
+
+        let valueExpression = field;
+        if (jsonPath) {
+            valueExpression = `CAST(JSON_VALUE(${field}, '${jsonPath}') AS FLOAT64)`;
+        }
+
+        const query = `
+            CREATE MATERIALIZED VIEW \`${dataset.id}.${viewName}\`
+            OPTIONS (enable_refresh = true, refresh_interval_minutes = 60)
+            AS SELECT 
+                ${entityField} as entity_id,
+                ${dateField} as date,
+                ${func}(${valueExpression}) as value,
+                COUNT(*) as count
+            FROM \`${dataset.id}.${baseTable}\`
+            GROUP BY 1, 2
+        `;
+
+        try {
+            await dataset.query({ query });
+            this.logger.log('INFO', `[MV-Manager] ✅ Successfully created ${viewName}. Backfill running in background.`);
+        } catch (e) {
+            if (e.message.includes('Already Exists')) {
+                this.logger.log('INFO', `[MV-Manager] View ${viewName} was created concurrently.`);
+                return;
+            }
+            throw new Error(`Failed to create Materialized View ${viewName}: ${e.message}`);
+        }
+    }
+
+    _hash(str) {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = (hash << 5) - hash + char;
+            hash |= 0;
+        }
+        return Math.abs(hash).toString(16);
+    }
+}
+
+module.exports = { MaterializedViewManager };

package/functions/computation-system-v2/framework/data/QueryBuilder.js

@@ -1,44 +1,20 @@
 /**
  * @fileoverview Query Builder - Builds validated SQL queries
- * 
- * All queries are validated against the SchemaRegistry BEFORE execution.
+ * * All queries are validated against the SchemaRegistry BEFORE execution.
  * This catches errors early and prevents runtime failures.
- */
-
-/**
- * @typedef {Object} QuerySpec
- * @property {string} table - Table name
- * @property {string[]} [select] - Columns to select (null = *)
- * @property {Object} [where] - WHERE conditions as { column: value }
- * @property {string} [dateField] - Date field name for date filtering
- * @property {string} [targetDate] - Target date (YYYY-MM-DD)
- * @property {number} [lookback=0] - Days to look back (0 = target date only)
- * @property {string} [entityField] - Entity field name for entity filtering
- * @property {string[]} [entities] - Entity IDs to filter
- * @property {string} [orderBy] - Column to order by
- * @property {string} [orderDir='DESC'] - Order direction
- * @property {number} [limit] - Limit number of rows
- */
-
-/**
- * @typedef {Object} BuiltQuery
- * @property {string} sql - The SQL query string
- * @property {Object} params - Query parameters
- * @property {string} table - Table that was queried
- * @property {Object} spec - Original query spec
+ * * V2.0 UPDATE: Safety features (Ban SELECT * and enforce partitions).
  */
 
 class QueryBuilder {
     /**
      * @param {Object} config - Configuration object
-     * @param {string} config.projectId - GCP project ID
-     * @param {string} config.dataset - BigQuery dataset name
      * @param {SchemaRegistry} schemaRegistry - Schema registry instance
      * @param {Object} [logger] - Logger instance
      */
     constructor(config, schemaRegistry, logger = null) {
         this.projectId = config.projectId;
         this.dataset = config.dataset;
+        this.tables = config.tables || {}; // Store table config for validation
         this.schemaRegistry = schemaRegistry;
         this.logger = logger;
     }
@@ -47,7 +23,7 @@ class QueryBuilder {
      * Build a validated query.
      * @param {QuerySpec} spec - Query specification
      * @returns {Promise<BuiltQuery>}
-     * @throws {Error} If validation fails
+     * @throws {Error} If validation fails or safety checks trigger
      */
     async build(spec) {
         const { table } = spec;
@@ -56,6 +32,27 @@ class QueryBuilder {
             throw new Error('[QueryBuilder] Table name is required');
         }
         
+        // --- 1. PARTITION SAFETY CHECK ---
+        // If the table is configured with a Date Field (Partition), 
+        // we MUST see a targetDate or a custom where-clause using that field.
+        // Otherwise, it's a guaranteed full-table scan over all history.
+        const tableConfig = this.tables[table];
+        
+        if (tableConfig && tableConfig.dateField) {
+            const hasTargetDate = !!spec.targetDate;
+            // Check if 'where' clause contains the date field key
+            const hasCustomDateFilter = spec.where && Object.keys(spec.where).includes(tableConfig.dateField);
+            
+            if (!hasTargetDate && !hasCustomDateFilter) {
+                throw new Error(
+                    `[QueryBuilder] 🛑 UNSAFE QUERY BLOCKED: Table '${table}' is partitioned by '${tableConfig.dateField}', ` +
+                    `but no date filter was provided. This would trigger a full table scan. ` +
+                    `Please provide a 'lookback' or explicit filter in your computation config.`
+                );
+            }
+        }
+        // ---------------------------------
+
         // Validate table exists and get schema
         const schema = await this.schemaRegistry.getSchema(table);
         
@@ -99,13 +96,8 @@ class QueryBuilder {
     
     /**
      * Build a simple existence check query.
-     * @param {string} table - Table name
-     * @param {string} dateField - Date field name
-     * @param {string} targetDate - Target date
-     * @returns {Promise<BuiltQuery>}
      */
     async buildExistsQuery(table, dateField, targetDate) {
-        // Validate table and date field
         await this.schemaRegistry.getSchema(table);
         if (dateField) {
             await this.schemaRegistry.validateColumns(table, [dateField]);
@@ -148,10 +140,18 @@ class QueryBuilder {
         const params = {};
         const conditions = [];
         
-        // SELECT clause
-        const selectClause = select && select.length > 0 
-            ? select.join(', ')
-            : '*';
+        // --- 2. SELECT * SAFETY CHECK ---
+        // BigQuery charges by column. SELECT * on a 100-col table costs 100x.
+        // We force developers to be specific.
+        if (!select || select.length === 0 || select.includes('*')) {
+            throw new Error(
+                `[QueryBuilder] 🛑 LAZY SELECT BLOCKED: 'SELECT *' or empty select is forbidden to save costs. ` +
+                `Table: ${table}. Please list the specific columns you need in the 'fields' array of your computation config.`
+            );
+        }
+        
+        const selectClause = select.join(', ');
+        // --------------------------------
         
         // Date filtering
         if (dateField && targetDate) {
@@ -229,4 +229,4 @@ class QueryBuilder {
     }
 }
 
-module.exports = { QueryBuilder };
+module.exports = { QueryBuilder };