bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/docs/HowToCreateComputations.MD

@@ -0,0 +1,602 @@
+# Developing Computations for the BullTrackers DAG System
+
+## Overview
+
+The computation system is a **directed acyclic graph (DAG)** of data transformations that run on BigQuery data. Each computation is a self-contained class that declares its data requirements, dependencies, and processing logic.
+
+---
+
+## Core Concepts
+
+### 1. **Computation Types**
+
+```javascript
+type: 'per-entity'  // Runs once per entity (user, asset, etc.)
+type: 'global'      // Runs once for all data (aggregations, rankings)
+```
+
+### 2. **Result Keys**
+
+```javascript
+// Per-entity computations
+this.setResult(entityId, { ...data });
+
+// Global computations
+this.setResult('_global', { ...data });
+```
+
+**Critical:** Global computations must use `'_global'` as the key, not `'global'`.
+
+---
+
+## Basic Computation Structure
+
+```javascript
+const { Computation } = require('../framework');
+
+class MyComputation extends Computation {
+    static getConfig() {
+        return {
+            name: 'MyComputation',
+            type: 'per-entity',  // or 'global'
+            category: 'analytics',
+            isHistorical: false,  // true if needs previous day's data
+            
+            requires: {
+                'table_name': {
+                    lookback: 0,      // days to fetch (0 = today only)
+                    mandatory: true,
+                    fields: ['col1', 'col2'],  // REQUIRED for cost control
+                    filter: { user_type: 'POPULAR_INVESTOR' }
+                }
+            },
+            
+            dependencies: [],  // Other computations needed
+            
+            storage: {
+                bigquery: true,
+                firestore: {
+                    enabled: true,
+                    path: 'results/{entityId}',
+                    merge: true
+                }
+            }
+        };
+    }
+
+    async process(context) {
+        const { data, entityId, date, rules, references } = context;
+        
+        // Your logic here
+        const result = { /* ... */ };
+        
+        this.setResult(entityId, result);
+    }
+}
+
+module.exports = MyComputation;
+```
+
+---
+
+## Cost Optimization Rules
+
+### **1. Always Specify Fields**
+
+```javascript
+// ❌ BAD - Will throw LAZY SELECT BLOCKED error
+requires: {
+    'portfolio_snapshots': {
+        fields: []  // or null or '*'
+    }
+}
+
+// ✅ GOOD - Only fetch what you need
+requires: {
+    'portfolio_snapshots': {
+        fields: ['user_id', 'portfolio_data', 'date']
+    }
+}
+```
+
+### **2. Always Use Partitioning**
+
+```javascript
+// ❌ BAD - Full table scan
+requires: {
+    'portfolio_snapshots': {
+        lookback: 0  // Missing! Will throw UNSAFE QUERY error
+    }
+}
+
+// ✅ GOOD - Date partition filter
+requires: {
+    'portfolio_snapshots': {
+        lookback: 0,  // Today only
+        fields: ['user_id', 'portfolio_data', 'date']
+    }
+}
+```
+
+### **3. Limit Lookback**
+
+```javascript
+// Maximum enforced: 60 days
+lookback: 30  // 30 days of history (max enforced: 60)
+```
+
+### **4. Use Filters to Reduce Data**
+
+```javascript
+requires: {
+    'portfolio_snapshots': {
+        lookback: 0,
+        fields: ['user_id', 'portfolio_data', 'date'],
+        filter: { 
+            user_type: 'POPULAR_INVESTOR'  // Reduces rows significantly
+        }
+    }
+}
+```
+
+---
+
+## Materialized Views
+
+### **Overview**
+
+Materialized views enable efficient **metric-based queries** without requiring custom SQL. The system **automatically creates and manages** them.
+
+### **Requesting a Materialized View**
+
+```javascript
+requires: {
+    'aum_metric': {
+        type: 'metric',           // Triggers MV creation
+        source: 'pi_rankings',    // Base table
+        func: 'SUM',              // Aggregation function
+        field: 'rankings_data',   // Column to aggregate
+        jsonPath: '$.AUMValue',   // JSON path (if needed)
+        lookback: 7,
+        series: true              // true = time series, false = single value
+    }
+}
+```
+
+### **What Happens**
+
+1. System generates MV name: `mv_pi_rankings_sum_rankings_data_by_day`
+2. Automatically creates MV with schema:
+   ```sql
+   CREATE MATERIALIZED VIEW mv_...
+   AS SELECT 
+       entity_id,
+       date,
+       SUM(CAST(JSON_VALUE(field, '$.path') AS FLOAT64)) as value
+   FROM base_table
+   GROUP BY entity_id, date
+   ```
+3. Auto-refreshes every 60 minutes
+
+### **Accessing Results**
+
+```javascript
+async process(context) {
+    const { data } = context;
+    
+    // Series mode (series: true)
+    // Returns: { "entityId": { "2024-01-01": 123, "2024-01-02": 456 } }
+    const aumTimeSeries = data['aum_metric'];
+    const todayAum = aumTimeSeries[entityId][date];
+    
+    // Single value mode (series: false)
+    // Returns: { "entityId": 123 }
+    const totalAum = data['aum_metric'][entityId];
+}
+```
+
+### **Supported Functions**
+
+- `SUM`, `AVG`, `MIN`, `MAX`, `COUNT`
+
+---
+
+## Dependencies Between Computations
+
+### **Basic Dependency**
+
+```javascript
+class DownstreamComputation extends Computation {
+    static getConfig() {
+        return {
+            name: 'DownstreamComputation',
+            dependencies: ['UpstreamComputation'],
+            requires: {
+                'computation_results': {
+                    lookback: 0,
+                    filter: { 
+                        computation_name: 'upstreamcomputation'  // lowercase!
+                    }
+                }
+            }
+        };
+    }
+
+    async process(context) {
+        const { getDependency, entityId } = context;
+        
+        // Access dependency result
+        const upstreamResult = getDependency('UpstreamComputation', entityId);
+        
+        // Use it
+        const value = upstreamResult.someField;
+    }
+}
+```
+
+### **Conditional Dependencies**
+
+```javascript
+conditionalDependencies: [
+    {
+        computation: 'SeasonalAdjustment',
+        condition: ({ date }) => {
+            const month = new Date(date).getMonth();
+            return month === 0; // Only in January
+        }
+    }
+]
+```
+
+---
+
+## Example Computations
+
+### **Example 1: Per-Entity Alert**
+
+```javascript
+class RiskScoreIncrease extends Computation {
+    static getConfig() {
+        return {
+            name: 'RiskScoreIncrease',
+            type: 'per-entity',
+            category: 'alerts',
+            isHistorical: true,  // Needs yesterday's data
+            
+            requires: {
+                'pi_rankings': {
+                    lookback: 1,  // Today + yesterday
+                    mandatory: true,
+                    fields: ['pi_id', 'rankings_data', 'date'],
+                    filter: { user_type: 'POPULAR_INVESTOR' }
+                }
+            },
+            
+            storage: {
+                bigquery: true,
+                firestore: {
+                    enabled: true,
+                    path: 'alerts/{date}/RiskScoreIncrease/{entityId}',
+                    merge: true
+                }
+            }
+        };
+    }
+
+    async process(context) {
+        const { data, entityId, date, rules } = context;
+        
+        const history = data['pi_rankings'] || [];
+        const todayRow = history.find(d => d.date === date);
+        const yesterdayRow = history.find(d => d.date === this._yesterday(date));
+        
+        const todayRisk = rules.rankings.getRiskScore(
+            rules.rankings.extractRankingsData(todayRow)
+        );
+        const yesterdayRisk = rules.rankings.getRiskScore(
+            rules.rankings.extractRankingsData(yesterdayRow)
+        );
+        
+        const change = todayRisk - (yesterdayRisk || 0);
+        
+        this.setResult(entityId, {
+            currentRisk: todayRisk,
+            previousRisk: yesterdayRisk,
+            change,
+            triggered: change > 0
+        });
+    }
+    
+    _yesterday(dateStr) {
+        const d = new Date(dateStr);
+        d.setDate(d.getDate() - 1);
+        return d.toISOString().slice(0, 10);
+    }
+}
+```
+
+### **Example 2: Global Aggregation**
+
+```javascript
+class GlobalAumPerAsset extends Computation {
+    static getConfig() {
+        return {
+            name: 'GlobalAumPerAsset',
+            type: 'global',
+            category: 'market_insights',
+            
+            requires: {
+                'computation_results': {
+                    lookback: 0,
+                    mandatory: true,
+                    fields: ['result_data', 'computation_name', 'date'],
+                    filter: { 
+                        computation_name: 'pidailyassetaum'
+                    }
+                }
+            },
+            
+            dependencies: ['PIDailyAssetAUM'],
+            
+            storage: {
+                bigquery: true,
+                firestore: {
+                    enabled: true,
+                    path: 'global_metrics/aum_per_asset_{date}',
+                    merge: true
+                }
+            }
+        };
+    }
+
+    async process(context) {
+        const { data } = context;
+        
+        let rows = data['computation_results'];
+        if (!Array.isArray(rows)) {
+            rows = Object.values(rows);
+        }
+        
+        const globalTotals = new Map();
+        
+        for (const row of rows) {
+            let assetMap = row.result_data;
+            if (typeof assetMap === 'string') {
+                assetMap = JSON.parse(assetMap);
+            }
+            
+            for (const [ticker, value] of Object.entries(assetMap)) {
+                const current = globalTotals.get(ticker) || 0;
+                globalTotals.set(ticker, current + value);
+            }
+        }
+        
+        const result = Array.from(globalTotals.entries())
+            .map(([symbol, amount]) => ({ symbol, amount }))
+            .sort((a, b) => b.amount - a.amount);
+        
+        this.setResult('_global', result);  // Note: _global, not global
+    }
+}
+```
+
+### **Example 3: Using Materialized Views**
+
+```javascript
+class PortfolioVelocity extends Computation {
+    static getConfig() {
+        return {
+            name: 'PortfolioVelocity',
+            type: 'per-entity',
+            category: 'analytics',
+            
+            requires: {
+                'total_exposure_metric': {
+                    type: 'metric',
+                    source: 'portfolio_snapshots',
+                    func: 'SUM',
+                    field: 'portfolio_data',
+                    jsonPath: '$.exposure',
+                    lookback: 7,
+                    series: true  // Get 7 days of data
+                }
+            },
+            
+            storage: { bigquery: true }
+        };
+    }
+
+    async process(context) {
+        const { data, entityId, date } = context;
+        
+        const exposureSeries = data['total_exposure_metric'][entityId] || {};
+        const dates = Object.keys(exposureSeries).sort();
+        
+        if (dates.length < 2) {
+            return this.setResult(entityId, { velocity: 0 });
+        }
+        
+        // Calculate rate of change
+        const values = dates.map(d => exposureSeries[d]);
+        const diffs = [];
+        for (let i = 1; i < values.length; i++) {
+            diffs.push(values[i] - values[i-1]);
+        }
+        
+        const avgVelocity = diffs.reduce((a, b) => a + b, 0) / diffs.length;
+        
+        this.setResult(entityId, {
+            velocity: avgVelocity,
+            current: values[values.length - 1],
+            trend: avgVelocity > 0 ? 'increasing' : 'decreasing'
+        });
+    }
+}
+```
+
+### **Example 4: Using Rules**
+
+```javascript
+class PortfolioSummary extends Computation {
+    static getConfig() {
+        return {
+            name: 'PortfolioSummary',
+            type: 'per-entity',
+            category: 'analytics',
+            
+            requires: {
+                'portfolio_snapshots': {
+                    lookback: 0,
+                    mandatory: true,
+                    fields: ['user_id', 'portfolio_data', 'date']
+                },
+                'ticker_mappings': {
+                    mandatory: false,
+                    fields: ['instrument_id', 'ticker']
+                },
+                'sector_mappings': {
+                    mandatory: false,
+                    fields: ['symbol', 'sector']
+                }
+            },
+            
+            storage: { bigquery: true }
+        };
+    }
+
+    async process(context) {
+        const { data, entityId, rules, references } = context;
+        
+        const portfolioRow = data['portfolio_snapshots'];
+        const portfolioData = rules.portfolio.extractPortfolioData(portfolioRow);
+        const positions = rules.portfolio.extractPositions(portfolioData);
+        
+        // Use business rules for calculations
+        const totalValue = rules.portfolio.calculateTotalValue(positions);
+        const exposure = rules.portfolio.getExposure(portfolioData);
+        const winRatio = rules.portfolio.calculateWinRatio(positions);
+        
+        // Access reference data
+        const tickerMap = this._buildTickerMap(references['ticker_mappings']);
+        const sectorMap = this._buildSectorMap(references['sector_mappings']);
+        
+        // Calculate sector exposure
+        const sectorExposure = rules.portfolio.calculateSectorExposure(
+            positions,
+            this._buildInstrumentToSectorMap(tickerMap, sectorMap)
+        );
+        
+        this.setResult(entityId, {
+            totalValue,
+            exposure,
+            winRatio,
+            sectorExposure,
+            positionCount: positions.length
+        });
+    }
+    
+    _buildTickerMap(rows) {
+        const map = {};
+        Object.values(rows || {}).forEach(r => {
+            map[r.instrument_id] = r.ticker;
+        });
+        return map;
+    }
+    
+    _buildSectorMap(rows) {
+        const map = {};
+        Object.values(rows || {}).forEach(r => {
+            map[r.symbol] = r.sector;
+        });
+        return map;
+    }
+    
+    _buildInstrumentToSectorMap(tickerMap, sectorMap) {
+        const map = {};
+        for (const [instId, ticker] of Object.entries(tickerMap)) {
+            map[instId] = sectorMap[ticker] || 'Unknown';
+        }
+        return map;
+    }
+}
+```
+
+---
+
+## Common Patterns
+
+### **Pattern 1: Date Comparison**
+
+```javascript
+const toDateStr = (d) => {
+    if (d?.value) return d.value;
+    if (d instanceof Date) return d.toISOString().slice(0, 10);
+    return String(d);
+};
+
+const todayRow = history.find(d => toDateStr(d.date) === date);
+```
+
+### **Pattern 2: Safe Entity Row Extraction**
+
+```javascript
+const getEntityRows = (dataset) => {
+    if (!dataset) return [];
+    if (dataset[entityId]) {
+        return Array.isArray(dataset[entityId]) 
+            ? dataset[entityId] 
+            : [dataset[entityId]];
+    }
+    if (Array.isArray(dataset)) {
+        return dataset.filter(r => 
+            String(r.user_id || r.pi_id || r.cid) === String(entityId)
+        );
+    }
+    return [];
+};
+```
+
+### **Pattern 3: JSON Parsing**
+
+```javascript
+let data = row.data_field;
+if (typeof data === 'string') {
+    try {
+        data = JSON.parse(data);
+    } catch (e) {
+        data = null;
+    }
+}
+```
+
+---
+
+## Testing
+
+```javascript
+const { IntegrationTester } = require('../framework/testing/ComputationTester');
+
+async function testComputation() {
+    const tester = new IntegrationTester(config);
+    
+    // Automatically finds runnable date
+    const date = await tester.findLatestRunnableDate('MyComputation');
+    
+    // Runs computation + dependencies
+    const { result } = await tester.run('MyComputation', date);
+    
+    console.log(result);
+}
+```
+
+---
+
+## Key Takeaways
+
+1. **Always specify `fields`** - Prevents cost explosions
+2. **Always use date partitioning** - Required for safety
+3. **Use `_global` not `'global'`** - Critical for global computations
+4. **Leverage materialized views** - Automatic metric aggregation
+5. **Use business rules** - Centralized, tested logic
+6. **Reference data** - Use `references` for mappings
+7. **Test incrementally** - Use IntegrationTester

package/functions/computation-system-v2/framework/core/Manifest.js

@@ -39,12 +39,9 @@ class ManifestBuilder {
             if (entry) {
                 manifestMap.set(entry.name, entry);
                 
-                // CRITICAL FIX: Include conditional dependencies in the DAG for cycle detection and topological sort.
-                // Even if the dependency is conditional at runtime, the execution order (Pass) must respect it.
                 const graphDeps = [...entry.dependencies];
                 if (entry.conditionalDependencies) {
                     entry.conditionalDependencies.forEach(cd => {
-                        // Ensure we use the normalized name for the graph
                         graphDeps.push(cd.computation);
                     });
                 }
@@ -60,7 +57,7 @@ class ManifestBuilder {
             throw new Error(`[Manifest] Circular dependency detected: ${cycle}`);
         }
 
-        // 3. Topological Sort (calculates passes)
+        // 3. Topological Sort
         const sortedItems = Graph.topologicalSort(nodes, adjacency);
 
         // 4. Hydrate Sorted List
@@ -115,8 +112,6 @@ class ManifestBuilder {
             compositeHash += `|RULE:${mod}:${h}`;
         }
         
-        // Normalize conditional dependencies if they exist
-        // This ensures the Orchestrator can look them up by normalized name later
         const conditionalDependencies = (config.conditionalDependencies || []).map(cd => ({
             ...cd,
             computation: this._normalize(cd.computation)
@@ -128,16 +123,20 @@ class ManifestBuilder {
             class: ComputationClass,
             category: config.category || 'default',
             type: config.type || 'global',
+            
+            outputTable: config.outputTable || null,
+            // --------------------------------------
+
             requires: config.requires || {},
             dependencies: (config.dependencies || []).map(d => this._normalize(d)),
-            conditionalDependencies, // FIX: Pass this through to the manifest entry
+            conditionalDependencies,
             isHistorical: config.isHistorical || false,
             isTest: config.isTest || false,
             schedule: this.scheduleValidator.parseSchedule(config.schedule),
             storage: this._parseStorageConfig(config.storage),
             ttlDays: config.ttlDays,
-            pass: 0, // Set later by Graph.js
-            hash: this._hashCode(compositeHash), // Intrinsic hash
+            pass: 0, 
+            hash: this._hashCode(compositeHash),
             weight: ComputationClass.getWeight ? ComputationClass.getWeight() : 1.0,
             composition: {
                 epoch: this.epoch,
@@ -152,7 +151,6 @@ class ManifestBuilder {
     _computeFinalHashes(sorted, manifestMap) {
         for (const entry of sorted) {
             let hashInput = entry.hash;
-            // Includes strict dependencies in the hash chain
             if (entry.dependencies.length > 0) {
                 const depHashes = entry.dependencies.sort().map(d => {
                     const h = manifestMap.get(d)?.hash;
@@ -161,10 +159,6 @@ class ManifestBuilder {
                 });
                 hashInput += `|DEPS:${depHashes.join('|')}`;
             }
-            // Note: Conditional dependencies are currently excluded from the hash chain 
-            // because they might not be loaded. If strict versioning is required for them,
-            // they should be added here too.
-            
             entry.hash = this._hashCode(hashInput);
         }
     }
@@ -203,7 +197,7 @@ class ManifestBuilder {
         const used = {};
         for (const [name, exports] of Object.entries(this.sharedLayers)) {
             const found = Object.keys(exports).filter(exp => 
-                code.includes(exp) // Simple include check, similar to original regex
+                code.includes(exp)
             );
             if (found.length) used[name] = found;
         }
@@ -240,7 +234,6 @@ class ManifestBuilder {
     
     _log(l, m) { this.logger ? this.logger.log(l, `[Manifest] ${m}`) : console.log(`[Manifest] ${m}`); }
     
-    // Public alias for groupByPass matching the Interface
     groupByPass(m) { return this._groupByPass(m); }
 }
 

package/functions/computation-system-v2/framework/core/RunAnalyzer.js

@@ -2,6 +2,7 @@
  * @fileoverview Run Analyzer
  * * Pure logic component that determines which computations need to run.
  * Decouples decision-making from execution and storage.
+ * * * UPDATE: Removed SQL bypass. All computations are now checked for data availability.
  */
 
 class RunAnalyzer {
@@ -57,7 +58,7 @@ class RunAnalyzer {
         }
 
         // 2. Data Availability Check
-        // Note: This is the only async IO part (calls DataFetcher)
+        // UPDATE: Removed isSql check. All computations must have their raw data available.
         const availability = await this.dataFetcher.checkAvailability(requires, dateStr);
         if (!availability.canRun) {
             if (!isToday) {