bulltrackers-module 1.0.765 → 1.0.768

package/functions/computation-system-v2/framework/testing/ComputationTester.js

@@ -1,86 +1,175 @@
 /**
- * @fileoverview Unit Testing Utility for Computations
+ * @fileoverview Integration Tester
+ * Wraps the Orchestrator to run computations in a test environment.
+ * - Redirects output to a test table.
+ * - Resolves and executes dependency chains.
+ * - Finds valid runnable dates based on raw data availability.
+ * * * UPDATE: Removed SQL output fetching support.
  */
 
-class ComputationTester {
-    constructor(ComputationClass, config) {
-        this.ComputationClass = ComputationClass;
-        this.config = config;
-        this.mockData = {};
-        this.mockDependencies = {};
-    }
-    
-    withMockData(tableName, data) {
-        this.mockData[tableName] = data;
-        return this;
+const fs = require('fs');
+const path = require('path');
+const { Orchestrator } = require('../execution/Orchestrator');
+const { ManifestBuilder } = require('../core/Manifest');
+
+class IntegrationTester {
+    constructor(baseConfig, logger = console) {
+        this.baseConfig = baseConfig;
+        this.logger = logger;
+        this.testConfig = this._createTestConfig(baseConfig);
     }
-    
-    withMockDependency(computationName, results) {
-        this.mockDependencies[computationName] = results;
-        return this;
+
+    /**
+     * Create a modified config for testing
+     * - Writes to computation_results_test
+     * - Disables Cloud Tasks
+     * - Forces local execution (no worker pool)
+     */
+    _createTestConfig(config) {
+        const testConfig = JSON.parse(JSON.stringify(config)); // Deep clone
+        
+        // 1. Redirect Storage
+        if (!testConfig.resultStore) testConfig.resultStore = {};
+        testConfig.resultStore.table = 'computation_results_test';
+        
+        // 2. Disable Side Effects
+        testConfig.cloudTasks = null; // Don't trigger downstream
+        
+        // 3. Force Local Execution for simplified debugging
+        if (testConfig.workerPool) {
+            testConfig.workerPool.enabled = false;
+        }
+
+        // Preserve the original computation classes (JSON.stringify lost them)
+        testConfig.computations = config.computations; 
+        // Preserve rules (functions)
+        testConfig.rules = config.rules;
+
+        return testConfig;
     }
-    
-    async run(entityId = 'test_entity') {
-        const instance = new this.ComputationClass();
+
+    /**
+     * Build a standalone Orchestrator instance restricted to the specific dependency chain
+     */
+    async setupForComputation(targetName) {
+        // 1. Identify Dependency Chain
+        const allComputations = this.testConfig.computations;
+        const chain = this._resolveDependencyChain(targetName, allComputations);
         
-        const context = {
-            date: '2026-01-24',
-            entityId,
-            data: this.mockData,
-            dependencies: this.mockDependencies,
-            rules: this.config.rules,
-            config: this.config,
-            logger: { log: () => {} }
-        };
+        this.logger.log('INFO', `[Tester] execution chain: ${chain.map(c => c.getConfig().name).join(' -> ')}`);
+
+        // 2. Configure Orchestrator with ONLY this chain
+        const runConfig = { ...this.testConfig, computations: chain };
+        this.orchestrator = new Orchestrator(runConfig, this.logger);
+        await this.orchestrator.initialize();
         
-        instance._meta = { logger: context.logger, config: this.config };
+        return chain;
+    }
+
+    /**
+     * Find the most recent date where Raw Data requirements are met
+     */
+    async findLatestRunnableDate(targetName, lookbackDays = 7) {
+        if (!this.orchestrator) await this.setupForComputation(targetName);
+
+        const manifest = this.orchestrator.manifest;
+        const chainNames = new Set(manifest.map(c => c.name));
         
-        await instance.process(context);
+        // Collect all Raw Data requirements (exclude requirements that are other computations in the chain)
+        const rawRequirements = {};
         
-        const allResults = await instance.getResult();
-        return allResults[entityId] || allResults;
+        for (const entry of manifest) {
+            for (const [reqName, reqSpec] of Object.entries(entry.requires)) {
+                // If the requirement is NOT a computation in our chain, it's raw data
+                rawRequirements[reqName] = reqSpec;
+            }
+        }
+
+        this.logger.log('INFO', `[Tester] Checking data availability for: ${Object.keys(rawRequirements).join(', ')}`);
+
+        // Iterate backwards from today
+        const today = new Date();
+        for (let i = 1; i <= lookbackDays; i++) {
+            const d = new Date(today);
+            d.setDate(today.getDate() - i);
+            const dateStr = d.toISOString().slice(0, 10);
+            
+            const availability = await this.orchestrator.dataFetcher.checkAvailability(rawRequirements, dateStr);
+            
+            if (availability.canRun) {
+                this.logger.log('INFO', `[Tester] Found runnable date: ${dateStr}`);
+                return dateStr;
+            } else {
+                this.logger.log('DEBUG', `[Tester] Date ${dateStr} missing: ${availability.missing.join(', ')}`);
+            }
+        }
+        
+        throw new Error(`No runnable date found in last ${lookbackDays} days.`);
     }
-    
-    async expect(entityId = 'test_entity') {
-        const result = await this.run(entityId);
+
+    async run(targetName, dateStr) {
+        if (!this.orchestrator) await this.setupForComputation(targetName);
+        
+        // Ensure test table exists
+        await this.orchestrator.storageManager._ensureBigQueryTable('computation_results_test');
         
+        this.logger.log('INFO', `[Tester] Running chain for ${targetName} on ${dateStr}`);
+        
+        // Run the DAG
+        // Orchestrator will handle passes automatically (Deps first, then Target)
+        const summary = await this.orchestrator.execute({ 
+            date: dateStr, 
+            force: true // Force run even if "up to date" in test table
+        });
+
+        // UPDATE: Removed SQL output fetching logic. 
+        // Always fetch from the standard State Repository (JSON result store).
+        const result = await this.orchestrator.stateRepository.getResult(dateStr, targetName);
+
         return {
-            result,
-            
-            toHaveProperty(prop, value) {
-                const actual = this._getNestedProperty(result, prop);
-                if (value !== undefined && actual !== value) {
-                    throw new Error(`Expected ${prop} to be ${value}, got ${actual}`);
-                }
-                if (actual === undefined) {
-                    throw new Error(`Expected ${prop} to exist`);
-                }
-                return this;
-            },
+            date: dateStr,
+            summary,
+            result
+        };
+    }
+
+    /**
+     * Recursively find all dependencies for the target computation
+     */
+    _resolveDependencyChain(targetName, allComputations) {
+        const compMap = new Map(allComputations.map(c => {
+            try {
+                const conf = c.getConfig();
+                return [conf.name.toLowerCase(), c];
+            } catch (e) { return [null, null]; }
+        }));
+
+        const chain = new Set();
+        const visit = (name) => {
+            const key = name.toLowerCase();
+            if (chain.has(key)) return;
             
-            toMatchSchema(schema) {
-                const errors = this._validateSchema(result, schema);
-                if (errors.length > 0) {
-                    throw new Error(`Schema validation failed: ${errors.join(', ')}`);
-                }
-                return this;
-            },
+            const compClass = compMap.get(key);
+            if (!compClass) throw new Error(`Computation '${name}' not found.`);
             
-            _getNestedProperty(obj, path) {
-                return path.split('.').reduce((curr, key) => curr?.[key], obj);
-            },
+            const config = compClass.getConfig();
             
-            _validateSchema(data, schema) {
-                const errors = [];
-                for (const [key, type] of Object.entries(schema)) {
-                    if (typeof data[key] !== type) {
-                        errors.push(`${key}: expected ${type}, got ${typeof data[key]}`);
-                    }
-                }
-                return errors;
+            // Recurse dependencies
+            if (config.dependencies) {
+                config.dependencies.forEach(dep => visit(dep));
             }
+            // Recurse conditional dependencies
+            if (config.conditionalDependencies) {
+                config.conditionalDependencies.forEach(cd => visit(cd.computation));
+            }
+
+            chain.add(key);
         };
+
+        visit(targetName);
+        
+        return Array.from(chain).map(k => compMap.get(k));
     }
 }
 
-module.exports = { ComputationTester };
+module.exports = { IntegrationTester };

package/functions/computation-system-v2/handlers/scheduler.js

@@ -121,17 +121,27 @@ async function runWatchdog(req, res) {
         // 1. Find Zombies
         const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
         
-        if (zombies.length === 0) {
-            return res.status(200).send('No zombies detected.');
+        // Filter out excessive attempts
+        const actionableZombies = [];
+        for (const z of zombies) {
+            if ((z.attempts || 0) >= 3) {
+                console.warn(`[Watchdog] Ignoring zombie ${z.name} (Checkpoint: ${z.checkpointId}) - Max attempts reached (${z.attempts})`);
+                continue;
+            }
+            actionableZombies.push(z);
+        }
+
+        if (actionableZombies.length === 0) {
+            return res.status(200).send('No recoverable zombies.');
         }
 
-        console.log(`[Watchdog] 🧟 Found ${zombies.length} zombies. Initiating recovery...`);
+        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Initiating recovery...`);
 
         // 2. Claim & Recover
         // We claim them first so the next watchdog doesn't grab them while we are dispatching
-        await Promise.all(zombies.map(z => storageManager.claimZombie(z.checkpointId)));
+        await Promise.all(actionableZombies.map(z => storageManager.claimZombie(z.checkpointId)));
 
-        const recoveryTasks = zombies.map(z => {
+        const recoveryTasks = actionableZombies.map(z => {
             const entry = manifest.find(m => m.name === z.name);
             if (!entry) {
                 console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);

package/functions/computation-system-v2/scripts/test-computation-dag.js

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview Test Computation DAG CLI
+ * Usage: node scripts/test-computation-dag.js <ComputationName> [Date]
+ * Example: node scripts/test-computation-dag.js UserPortfolioSummary
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { IntegrationTester } = require('../framework/testing/ComputationTester');
+const config = require('../config/bulltrackers.config');
+
+// Setup logging
+const logger = {
+    log: (level, msg) => console.log(`[${new Date().toISOString()}] ${level}: ${msg}`),
+    error: (msg) => console.error(`[ERROR] ${msg}`)
+};
+
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length < 1) {
+        console.log('Usage: node scripts/test-computation-dag.js <ComputationName> [YYYY-MM-DD]');
+        process.exit(1);
+    }
+
+    const targetName = args[0];
+    let dateStr = args[1]; // Optional
+
+    try {
+        console.log('┌────────────────────────────────────────────────────────┐');
+        console.log('│             COMPUTATION INTEGRATION TEST               │');
+        console.log('└────────────────────────────────────────────────────────┘');
+        console.log(`Target: ${targetName}`);
+        console.log(`Mode:   DAG Integration (Production Read / Test Write)`);
+        
+        const tester = new IntegrationTester(config, logger);
+
+        // 1. Setup & Resolve Chain
+        await tester.setupForComputation(targetName);
+
+        // 2. Find Date if not provided
+        if (!dateStr) {
+            console.log('\n🔍 Auto-detecting latest runnable date...');
+            dateStr = await tester.findLatestRunnableDate(targetName);
+        }
+
+        // 3. Execute
+        console.log(`\n🚀 Executing chain on ${dateStr}...`);
+        const output = await tester.run(targetName, dateStr);
+
+        // 4. Report
+        const { summary, result } = output;
+        
+        console.log('\n┌────────────────────────────────────────────────────────┐');
+        console.log('│                     TEST RESULTS                       │');
+        console.log('└────────────────────────────────────────────────────────┘');
+        
+        if (summary.summary.errors > 0) {
+            console.log('❌ Execution FAILED');
+            summary.errors.forEach(e => console.log(`   - ${e.name}: ${e.error}`));
+        } else {
+            console.log('✅ Execution SUCCESS');
+
+            // SQL computations: result is { _sqlOutput, outputTable, rowCount, rows }
+            if (result && result._sqlOutput) {
+                console.log(`\n📊 SQL output table: \`${config.bigquery.dataset}.${result.outputTable}\``);
+                console.log(`   - Rows for ${dateStr}: ${result.rowCount}`);
+                if (result._error) {
+                    console.log(`   - ⚠️ Read error: ${result._error}`);
+                } else if (result.rowCount > 0) {
+                    const sample = result.rows[0];
+                    console.log(`   - Sample row: ${JSON.stringify(sample).substring(0, 120)}...`);
+                    const filename = `test-results-${targetName}-${dateStr}.json`;
+                    const outputPath = path.join(__dirname, '..', filename);
+                    fs.writeFileSync(outputPath, JSON.stringify(result.rows, null, 2));
+                    console.log(`\n💾 Full rows saved to: ${outputPath}`);
+                } else {
+                    console.log('   - No rows for this date. Check source table has data: portfolio_snapshots for this date.');
+                }
+                console.log(`\nQuery in BQ: SELECT * FROM \`${config.bigquery.projectId}.${config.bigquery.dataset}.${result.outputTable}\` WHERE date = '${dateStr}'`);
+            } else if (result) {
+                // Non-SQL: result from computation_results_test
+                const resultKeys = Object.keys(result);
+                const sampleKey = resultKeys[0];
+                const sampleData = resultKeys.length > 0 ? result[sampleKey] : null;
+
+                console.log(`\n📊 Result Summary for ${targetName}:`);
+                console.log(`   - Count: ${resultKeys.length} entities/records`);
+                console.log(`   - Sample ID: ${sampleKey || 'N/A'}`);
+                if (sampleData) {
+                    console.log(`   - Sample Data Preview: ${JSON.stringify(sampleData).substring(0, 100)}...`);
+                }
+                const filename = `test-results-${targetName}-${dateStr}.json`;
+                const outputPath = path.join(__dirname, '..', filename);
+                fs.writeFileSync(outputPath, JSON.stringify(result, null, 2));
+                console.log(`\n💾 Full results saved to: ${outputPath}`);
+                console.log(`Query results in BQ: SELECT * FROM \`${config.bigquery.dataset}.computation_results_test\` WHERE computation_name = '${targetName}' AND date = '${dateStr}'`);
+            } else {
+                console.log('⚠️  No results produced (Computation returned empty set).');
+            }
+        }
+
+    } catch (e) {
+        console.error('\n❌ FATAL ERROR:');
+        console.error(e);
+        process.exit(1);
+    }
+}
+
+main();

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -43,7 +43,7 @@ async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid,
                 date: date,
                 user_id: Number(cid),
                 user_type: 'SIGNED_IN_USER',
-                portfolio_data: JSON.stringify(portfolioData), // BigQuery JSON type requires a string
+                portfolio_data: portfolioData,
                 fetched_at: new Date().toISOString()
             };
             
@@ -103,7 +103,7 @@ async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, c
                 date: date,
                 user_id: Number(cid),
                 user_type: 'SIGNED_IN_USER',
-                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                history_data: historyData,
                 fetched_at: new Date().toISOString()
             };
             
@@ -172,7 +172,7 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
                 date: date,
                 user_id: Number(cid),
                 user_type: 'SIGNED_IN_USER',
-                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                posts_data: { posts: postsMap, postCount: posts.length },
                 fetched_at: new Date().toISOString()
             };
             
@@ -257,7 +257,7 @@ async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, c
                 date: date,
                 user_id: Number(cid),
                 user_type: 'POPULAR_INVESTOR',
-                portfolio_data: JSON.stringify(portfolioDoc), // BigQuery JSON type requires a string
+                portfolio_data: portfolioDoc,
                 fetched_at: new Date().toISOString()
             };
             
@@ -316,7 +316,7 @@ async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry
                 date: date,
                 user_id: Number(cid),
                 user_type: 'POPULAR_INVESTOR',
-                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                history_data: historyData,
                 fetched_at: new Date().toISOString()
             };
             
@@ -383,7 +383,7 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
                 date: date,
                 user_id: Number(cid),
                 user_type: 'POPULAR_INVESTOR',
-                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                posts_data: { posts: postsMap, postCount: posts.length },
                 fetched_at: new Date().toISOString()
             };
             

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.765",
+  "version": "1.0.768",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system-v2/computations/PopularInvestorRiskAssessment.js

@@ -1,176 +0,0 @@
-/**
- * @fileoverview Popular Investor Risk Assessment - v2
- * 
- * This computation DEPENDS on PopularInvestorProfileMetrics.
- * It uses the profile metrics to calculate additional risk assessments.
- * 
- * This tests:
- * 1. Dependency declaration works
- * 2. Pass assignment is incremented (should be Pass 2)
- * 3. Dependency data is loaded and passed to process()
- */
-
-const { Computation } = require('../framework');
-
-class PopularInvestorRiskAssessment extends Computation {
-    
-    static getConfig() {
-        return {
-            name: 'PopularInvestorRiskAssessment',
-            category: 'popular_investor',
-            
-            // Only needs rankings for additional risk factors
-            requires: {
-                'pi_rankings': {
-                    lookback: 0,
-                    mandatory: true,
-                    filter: null
-                }
-            },
-            
-            // DEPENDS ON PopularInvestorProfileMetrics!
-            dependencies: ['PopularInvestorProfileMetrics'],
-            
-            type: 'per-entity',
-            isHistorical: false,
-            ttlDays: 30
-        };
-    }
-    
-    static getSchema() {
-        return {
-            type: 'object',
-            properties: {
-                riskScore: { type: 'number' },
-                riskLevel: { type: 'string' },
-                diversificationScore: { type: 'number' },
-                volatilityScore: { type: 'number' },
-                concentrationRisk: { type: 'number' },
-                sectorConcentration: { type: 'object' },
-                recommendations: { type: 'array' }
-            }
-        };
-    }
-    
-    static getWeight() {
-        return 3.0;
-    }
-    
-    async process(context) {
-        const { date, entityId: userId, data, dependencies } = context;
-        
-        // Get the profile metrics from our dependency
-        const profileMetrics = dependencies?.['popularinvestorprofilemetrics']?.[userId];
-        
-        // Get rankings data
-        const rankingsData = data['pi_rankings'];
-        const rankEntry = this._findRankEntry(rankingsData, userId);
-        
-        // Initialize result
-        const result = {
-            riskScore: 0,
-            riskLevel: 'Unknown',
-            diversificationScore: 0,
-            volatilityScore: 0,
-            concentrationRisk: 0,
-            sectorConcentration: {},
-            recommendations: []
-        };
-        
-        // If we don't have profile metrics, we can still calculate basic risk from rankings
-        if (rankEntry) {
-            const riskScore = rankEntry.RiskScore || rankEntry.riskScore || 0;
-            result.riskScore = riskScore;
-            result.riskLevel = this._getRiskLevel(riskScore);
-        }
-        
-        // If we have profile metrics, enhance the assessment
-        if (profileMetrics) {
-            // Use sector exposure for diversification analysis
-            const sectorExposure = profileMetrics.sectorExposure?.data || {};
-            const sectors = Object.keys(sectorExposure);
-            
-            if (sectors.length > 0) {
-                // Diversification score (more sectors = better diversification)
-                result.diversificationScore = Math.min(100, sectors.length * 15);
-                
-                // Concentration risk (highest sector weight)
-                const maxExposure = Math.max(...Object.values(sectorExposure), 0);
-                result.concentrationRisk = maxExposure;
-                
-                // Copy sector breakdown
-                result.sectorConcentration = sectorExposure;
-                
-                // Generate recommendations
-                if (maxExposure > 50) {
-                    result.recommendations.push(`High concentration in single sector (${maxExposure.toFixed(1)}%). Consider diversifying.`);
-                }
-                if (sectors.length < 3) {
-                    result.recommendations.push('Portfolio is concentrated in few sectors. Consider adding exposure to other sectors.');
-                }
-            }
-            
-            // Use portfolio summary for additional risk factors
-            const portfolioSummary = profileMetrics.portfolioSummary || {};
-            if (portfolioSummary.profitPercent < -10) {
-                result.recommendations.push('Portfolio showing significant losses. Review position sizing.');
-            }
-            
-            // Use rankings data from profile for win ratio analysis
-            const rankingsFromProfile = profileMetrics.rankingsData || {};
-            if (rankingsFromProfile.winRatio < 0.5) {
-                result.volatilityScore = 80;  // Lower win ratio = higher volatility risk
-                result.recommendations.push('Win ratio below 50%. Trading strategy may need review.');
-            } else {
-                result.volatilityScore = Math.max(0, 100 - (rankingsFromProfile.winRatio * 100));
-            }
-            
-            // Adjust overall risk score based on analysis
-            const adjustedRisk = (
-                (result.riskScore * 0.4) +
-                (result.concentrationRisk * 0.3) +
-                (result.volatilityScore * 0.3)
-            );
-            result.riskScore = Math.round(adjustedRisk);
-            result.riskLevel = this._getRiskLevel(result.riskScore);
-        }
-        
-        this.setResult(userId, result);
-    }
-    
-    _findRankEntry(rankings, userId) {
-        if (!rankings) return null;
-        
-        // Rankings might be date-keyed or an array
-        if (typeof rankings === 'object' && !Array.isArray(rankings)) {
-            const dates = Object.keys(rankings).sort().reverse();
-            for (const dateStr of dates) {
-                const dayRankings = rankings[dateStr];
-                if (Array.isArray(dayRankings)) {
-                    const entry = dayRankings.find(r => 
-                        String(r.CustomerId || r.pi_id || r.userId) === String(userId)
-                    );
-                    if (entry) return entry;
-                }
-            }
-        }
-        
-        if (Array.isArray(rankings)) {
-            return rankings.find(r => 
-                String(r.CustomerId || r.pi_id || r.userId) === String(userId)
-            );
-        }
-        
-        return null;
-    }
-    
-    _getRiskLevel(score) {
-        if (score <= 2) return 'Very Low';
-        if (score <= 4) return 'Low';
-        if (score <= 6) return 'Medium';
-        if (score <= 8) return 'High';
-        return 'Very High';
-    }
-}
-
-module.exports = PopularInvestorRiskAssessment;