bulltrackers-module 1.0.765 → 1.0.768

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -84,7 +84,7 @@ module.exports = {
         'portfolio_snapshots': {
             dateField: 'date',
             entityField: 'user_id',
-            dataField: 'portfolio_data',
+            //dataField: 'portfolio_data',
             description: 'Daily portfolio snapshots for all users'
         },
         
@@ -92,7 +92,7 @@ module.exports = {
         'trade_history_snapshots': {
             dateField: 'date',
             entityField: 'user_id',
-            dataField: 'history_data',
+            //dataField: 'history_data',
             description: 'Daily trade history snapshots'
         },
         
@@ -100,7 +100,7 @@ module.exports = {
         'social_post_snapshots': {
             dateField: 'date',
             entityField: 'user_id',
-            dataField: 'posts_data',
+            //dataField: 'posts_data',
             description: 'Daily social post snapshots'
         },
         
@@ -108,7 +108,7 @@ module.exports = {
         'asset_prices': {
             dateField: 'date',
             entityField: 'instrument_id',
-            dataField: null,  // Flat table
+            //dataField: null,  // Flat table
             description: 'Daily asset prices'
         },
         
@@ -116,7 +116,7 @@ module.exports = {
         'pi_rankings': {
             dateField: 'date',
             entityField: 'pi_id',
-            dataField: 'rankings_data',
+            //dataField: 'rankings_data',
             description: 'Daily PI rankings snapshot'
         },
         
@@ -124,7 +124,7 @@ module.exports = {
         'pi_master_list': {
             dateField: null,  // Not date-partitioned
             entityField: 'cid',
-            dataField: null,
+            //dataField: null,
             description: 'Master list of all Popular Investors'
         },
         
@@ -132,7 +132,7 @@ module.exports = {
         'pi_ratings': {
             dateField: 'date',
             entityField: 'pi_id',
-            dataField: null,
+            //dataField: null,
             description: 'Daily PI ratings'
         },
         
@@ -140,7 +140,7 @@ module.exports = {
         'pi_page_views': {
             dateField: 'date',
             entityField: 'pi_id',
-            dataField: null,
+            //dataField: null,
             description: 'Daily PI page view metrics'
         },
         
@@ -148,7 +148,7 @@ module.exports = {
         'watchlist_membership': {
             dateField: 'date',
             entityField: 'pi_id',
-            dataField: null,
+            //dataField: null,
             description: 'Daily watchlist membership counts'
         },
         
@@ -156,7 +156,7 @@ module.exports = {
         'pi_alert_history': {
             dateField: 'date',
             entityField: 'pi_id',
-            dataField: 'metadata',
+            //dataField: 'metadata',
             description: 'Daily alert trigger history'
         },
         
@@ -164,7 +164,7 @@ module.exports = {
         'instrument_insights': {
             dateField: 'date',
             entityField: 'instrument_id',
-            dataField: 'insights_data',
+            //dataField: 'insights_data',
             description: 'Daily instrument insights'
         },
         
@@ -172,7 +172,7 @@ module.exports = {
         'ticker_mappings': {
             dateField: null,
             entityField: 'instrument_id',
-            dataField: null,
+            //dataField: null,
             description: 'Instrument ID to ticker symbol mappings'
         },
         
@@ -180,15 +180,27 @@ module.exports = {
         'computation_results': {
             dateField: 'date',
             entityField: null,  // Keyed by computation_name
-            dataField: 'result_data',
+            //dataField: 'result_data',
             description: 'Stored computation results'
         },
         // NEW: Sector Mappings Table
         'sector_mappings': {
             dateField: null,      // Static data
             entityField: 'symbol', // Key the data by symbol for fast lookup
-            dataField: null,
+            //dataField: null,
             description: 'Ticker to Sector mappings migrated from Firestore'
+        },
+
+        // NEW: Map the abstract requirement 'behavioral_features' to the actual BQ table
+        'behavioral_features': {
+            tableName: 'daily_behavioral_features',
+            dateField: 'date',
+            entityField: 'user_id', // Important for per-entity fetching
+            schema: [
+                { name: 'user_id', type: 'STRING' },
+                { name: 'hhi_score', type: 'FLOAT' },
+                { name: 'martingale_events', type: 'INTEGER' }
+            ]
         }
     },
     

package/functions/computation-system-v2/framework/core/Manifest.js

@@ -39,12 +39,9 @@ class ManifestBuilder {
             if (entry) {
                 manifestMap.set(entry.name, entry);
                 
-                // CRITICAL FIX: Include conditional dependencies in the DAG for cycle detection and topological sort.
-                // Even if the dependency is conditional at runtime, the execution order (Pass) must respect it.
                 const graphDeps = [...entry.dependencies];
                 if (entry.conditionalDependencies) {
                     entry.conditionalDependencies.forEach(cd => {
-                        // Ensure we use the normalized name for the graph
                         graphDeps.push(cd.computation);
                     });
                 }
@@ -60,7 +57,7 @@ class ManifestBuilder {
             throw new Error(`[Manifest] Circular dependency detected: ${cycle}`);
         }
 
-        // 3. Topological Sort (calculates passes)
+        // 3. Topological Sort
         const sortedItems = Graph.topologicalSort(nodes, adjacency);
 
         // 4. Hydrate Sorted List
@@ -115,8 +112,6 @@ class ManifestBuilder {
             compositeHash += `|RULE:${mod}:${h}`;
         }
         
-        // Normalize conditional dependencies if they exist
-        // This ensures the Orchestrator can look them up by normalized name later
         const conditionalDependencies = (config.conditionalDependencies || []).map(cd => ({
             ...cd,
             computation: this._normalize(cd.computation)
@@ -128,16 +123,20 @@ class ManifestBuilder {
             class: ComputationClass,
             category: config.category || 'default',
             type: config.type || 'global',
+            
+            outputTable: config.outputTable || null,
+            // --------------------------------------
+
             requires: config.requires || {},
             dependencies: (config.dependencies || []).map(d => this._normalize(d)),
-            conditionalDependencies, // FIX: Pass this through to the manifest entry
+            conditionalDependencies,
             isHistorical: config.isHistorical || false,
             isTest: config.isTest || false,
             schedule: this.scheduleValidator.parseSchedule(config.schedule),
             storage: this._parseStorageConfig(config.storage),
             ttlDays: config.ttlDays,
-            pass: 0, // Set later by Graph.js
-            hash: this._hashCode(compositeHash), // Intrinsic hash
+            pass: 0, 
+            hash: this._hashCode(compositeHash),
             weight: ComputationClass.getWeight ? ComputationClass.getWeight() : 1.0,
             composition: {
                 epoch: this.epoch,
@@ -152,7 +151,6 @@ class ManifestBuilder {
     _computeFinalHashes(sorted, manifestMap) {
         for (const entry of sorted) {
             let hashInput = entry.hash;
-            // Includes strict dependencies in the hash chain
             if (entry.dependencies.length > 0) {
                 const depHashes = entry.dependencies.sort().map(d => {
                     const h = manifestMap.get(d)?.hash;
@@ -161,10 +159,6 @@ class ManifestBuilder {
                 });
                 hashInput += `|DEPS:${depHashes.join('|')}`;
             }
-            // Note: Conditional dependencies are currently excluded from the hash chain 
-            // because they might not be loaded. If strict versioning is required for them,
-            // they should be added here too.
-            
             entry.hash = this._hashCode(hashInput);
         }
     }
@@ -203,7 +197,7 @@ class ManifestBuilder {
         const used = {};
         for (const [name, exports] of Object.entries(this.sharedLayers)) {
             const found = Object.keys(exports).filter(exp => 
-                code.includes(exp) // Simple include check, similar to original regex
+                code.includes(exp)
             );
             if (found.length) used[name] = found;
         }
@@ -240,7 +234,6 @@ class ManifestBuilder {
     
     _log(l, m) { this.logger ? this.logger.log(l, `[Manifest] ${m}`) : console.log(`[Manifest] ${m}`); }
     
-    // Public alias for groupByPass matching the Interface
     groupByPass(m) { return this._groupByPass(m); }
 }
 

package/functions/computation-system-v2/framework/core/RunAnalyzer.js

@@ -2,6 +2,7 @@
  * @fileoverview Run Analyzer
  * * Pure logic component that determines which computations need to run.
  * Decouples decision-making from execution and storage.
+ * * * UPDATE: Removed SQL bypass. All computations are now checked for data availability.
  */
 
 class RunAnalyzer {
@@ -57,7 +58,7 @@ class RunAnalyzer {
         }
 
         // 2. Data Availability Check
-        // Note: This is the only async IO part (calls DataFetcher)
+        // UPDATE: Removed isSql check. All computations must have their raw data available.
         const availability = await this.dataFetcher.checkAvailability(requires, dateStr);
         if (!availability.canRun) {
             if (!isToday) {

package/functions/computation-system-v2/framework/data/DataFetcher.js

@@ -9,9 +9,14 @@
  * * V2.4 FIX: Runaway Query Cost Prevention [Fix #3].
  * * V2.5 UPDATE: Super-Entity Monitoring [Safety Valve for Fix #6].
  * - Warns if a single entity exceeds reasonable batch limits (Memory Risk).
+ * * V2.6 UPDATE: Query Result Caching.
+ * - Implemented in-memory LRU cache to prevent redundant BigQuery costs for reference data.
+ * * V2.7 FIX: Double-Encoded JSON Normalization.
+ * - Automatically detects and recursively parses JSON strings (e.g. posts_data) to prevent downstream parsing errors.
  */
 
 const { BigQuery } = require('@google-cloud/bigquery');
+const crypto = require('crypto');
 
 // FIX #3: Hard limit to prevent cost spirals
 const MAX_LOOKBACK_DAYS = 30;
@@ -29,11 +34,24 @@ class DataFetcher {
         
         this.client = new BigQuery({ projectId: this.projectId });
         
+        // Cache Configuration (V2.6)
+        this.cacheConfig = config.queryCache || {
+            enabled: true,
+            ttlMs: 300000, // 5 minutes default
+            maxSize: 1000  // Max unique queries to cache
+        };
+        
+        // Use Map as LRU cache (insertion order preserved)
+        this.cache = new Map();
+        
         this.stats = {
             queries: 0,
             rowsFetched: 0,
             errors: 0,
-            bytesProcessed: 0
+            bytesProcessed: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            cacheEvictions: 0
         };
     }
     
@@ -273,21 +291,74 @@ class DataFetcher {
         
         return { canRun: missing.length === 0, available, missing };
     }
+
     
     getStats() { return { ...this.stats }; }
-    resetStats() { this.stats = { queries: 0, rowsFetched: 0, errors: 0, bytesProcessed: 0 }; }
+    
+    resetStats() { 
+        this.stats = { 
+            queries: 0, 
+            rowsFetched: 0, 
+            errors: 0, 
+            bytesProcessed: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            cacheEvictions: 0
+        }; 
+        this.cache.clear();
+    }
+
+    clearCache() {
+        this.cache.clear();
+        this._log('DEBUG', 'Query cache cleared');
+    }
+    
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
     
     async _execute(query) {
+        // V2.6: Query Caching
+        if (this.cacheConfig.enabled) {
+            const cacheKey = this._generateCacheKey(query);
+            const cached = this.cache.get(cacheKey);
+            
+            if (cached) {
+                if (Date.now() - cached.timestamp < this.cacheConfig.ttlMs) {
+                    this.stats.cacheHits++;
+                    // Refresh LRU position (delete and re-set moves to end)
+                    this.cache.delete(cacheKey);
+                    this.cache.set(cacheKey, cached);
+                    // Return cached rows immediately - no BigQuery cost
+                    return cached.rows;
+                } else {
+                    this.cache.delete(cacheKey); // Expired
+                }
+            }
+            this.stats.cacheMisses++;
+        }
+
         this.stats.queries++;
+        
         try {
             const [job] = await this.client.createQueryJob({
                 query: query.sql, params: query.params, location: this.location
             });
             const [rows] = await job.getQueryResults();
             const [metadata] = await job.getMetadata();
+            
             this.stats.rowsFetched += rows.length;
             this.stats.bytesProcessed += parseInt(metadata.statistics?.totalBytesProcessed || 0, 10);
-            return rows;
+            
+            // FIX V2.7: Normalize Rows (Recursive JSON Parse) BEFORE caching
+            const normalizedRows = rows.map(r => this._normalizeRow(r));
+            
+            // Store in cache if enabled
+            if (this.cacheConfig.enabled) {
+                this._addToCache(query, normalizedRows);
+            }
+            
+            return normalizedRows;
         } catch (e) {
             this.stats.errors++;
             this._log('ERROR', `Query failed: ${e.message}`);
@@ -296,6 +367,8 @@ class DataFetcher {
     }
 
     async *_executeStream(query) {
+        // NOTE: We do NOT cache streams. They are typically massive datasets (batch processing)
+        // and caching them in memory would cause OOM.
         this.stats.queries++;
         try {
             const [job] = await this.client.createQueryJob({
@@ -304,7 +377,8 @@ class DataFetcher {
             const stream = job.getQueryResultsStream();
             for await (const row of stream) {
                 this.stats.rowsFetched++;
-                yield row;
+                // FIX V2.7: Normalize Rows (Recursive JSON Parse)
+                yield this._normalizeRow(row);
             }
         } catch (e) {
             this.stats.errors++;
@@ -312,6 +386,70 @@ class DataFetcher {
             throw e;
         }
     }
+
+    /**
+     * V2.8 FIX: JSON Detection Logic
+     */
+    _normalizeRow(row) {
+        const normalized = { ...row };
+        for (const [key, value] of Object.entries(normalized)) {
+            if (typeof value === 'string') {
+                const trimmed = value.trim();
+                // FIX: Check for " (Double Encoded JSON) in addition to { and [
+                if (trimmed.startsWith('{') || trimmed.startsWith('[') || trimmed.startsWith('"')) {
+                    normalized[key] = this._safeRecursiveParse(value);
+                }
+            }
+        }
+        return normalized;
+    }
+
+    /**
+     * V2.7 FIX: Helper to safely recursively parse JSON.
+     * Handles: Double-Encoded JSON Strings (parsed recursively)
+     */
+   _safeRecursiveParse(input) {
+        if (!input) return null;
+        if (typeof input === 'object') return input;
+        try {
+            const parsed = JSON.parse(input);
+            // Recursion for double-encoded strings
+            if (typeof parsed === 'string') return this._safeRecursiveParse(parsed);
+            return parsed;
+        } catch (e) {
+            return input; // Not JSON, return original
+        }
+    }
+
+    /**
+     * V2.6: Generate a unique cache key for a query
+     */
+    _generateCacheKey(query) {
+        // Hash the SQL + Params to ensure uniqueness
+        const str = query.sql + JSON.stringify(query.params || {});
+        return crypto.createHash('md5').update(str).digest('hex');
+    }
+
+    /**
+     * V2.6: Add to cache with LRU eviction
+     */
+    _addToCache(query, rows) {
+        // Generate key
+        const key = this._generateCacheKey(query);
+        
+        // Eviction Logic
+        if (this.cache.size >= this.cacheConfig.maxSize) {
+            // Map iterator yields in insertion order. First item is oldest.
+            const oldestKey = this.cache.keys().next().value;
+            this.cache.delete(oldestKey);
+            this.stats.cacheEvictions++;
+        }
+        
+        this.cache.set(key, {
+            rows: rows,
+            timestamp: Date.now()
+        });
+    }
     
     /**
      * Transforms raw rows into a structured object.