bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/framework/testing/ComputationTester.js

@@ -1,86 +1,181 @@
 /**
- * @fileoverview Unit Testing Utility for Computations
+ * @fileoverview Integration Tester
+ * Wraps the Orchestrator to run computations in a test environment.
+ * - Redirects output to a test table.
+ * - Resolves and executes dependency chains.
+ * - Finds valid runnable dates based on raw data availability.
+ * * * UPDATE: Fixed dependency reading to look at test table instead of production.
  */
 
-class ComputationTester {
-    constructor(ComputationClass, config) {
-        this.ComputationClass = ComputationClass;
-        this.config = config;
-        this.mockData = {};
-        this.mockDependencies = {};
-    }
-    
-    withMockData(tableName, data) {
-        this.mockData[tableName] = data;
-        return this;
+const fs = require('fs');
+const path = require('path');
+const { Orchestrator } = require('../execution/Orchestrator');
+const { ManifestBuilder } = require('../core/Manifest');
+
+class IntegrationTester {
+    constructor(baseConfig, logger = console) {
+        this.baseConfig = baseConfig;
+        this.logger = logger;
+        this.testConfig = this._createTestConfig(baseConfig);
     }
-    
-    withMockDependency(computationName, results) {
-        this.mockDependencies[computationName] = results;
-        return this;
+
+    /**
+     * Create a modified config for testing
+     * - Writes to computation_results_test
+     * - Disables Cloud Tasks
+     * - Forces local execution (no worker pool)
+     */
+    _createTestConfig(config) {
+        const testConfig = JSON.parse(JSON.stringify(config)); // Deep clone
+        
+        // 1. Redirect Storage (Writes)
+        if (!testConfig.resultStore) testConfig.resultStore = {};
+        testConfig.resultStore.table = 'computation_results_test';
+
+        // 2. Redirect Reads for Intermediate Results (Crucial for Chained Computations)
+        // We must ensure that when a computation asks for 'computation_results', 
+        // it reads from the TEST table where the upstream dependency just wrote.
+        if (testConfig.tables && testConfig.tables['computation_results']) {
+            testConfig.tables['computation_results'].tableName = 'computation_results_test';
+        }
+        
+        // 3. Disable Side Effects
+        testConfig.cloudTasks = null; // Don't trigger downstream
+        
+        // 4. Force Local Execution for simplified debugging
+        if (testConfig.workerPool) {
+            testConfig.workerPool.enabled = false;
+        }
+
+        // Preserve the original computation classes (JSON.stringify lost them)
+        testConfig.computations = config.computations; 
+        // Preserve rules (functions)
+        testConfig.rules = config.rules;
+
+        return testConfig;
     }
-    
-    async run(entityId = 'test_entity') {
-        const instance = new this.ComputationClass();
+
+    /**
+     * Build a standalone Orchestrator instance restricted to the specific dependency chain
+     */
+    async setupForComputation(targetName) {
+        // 1. Identify Dependency Chain
+        const allComputations = this.testConfig.computations;
+        const chain = this._resolveDependencyChain(targetName, allComputations);
         
-        const context = {
-            date: '2026-01-24',
-            entityId,
-            data: this.mockData,
-            dependencies: this.mockDependencies,
-            rules: this.config.rules,
-            config: this.config,
-            logger: { log: () => {} }
-        };
+        this.logger.log('INFO', `[Tester] execution chain: ${chain.map(c => c.getConfig().name).join(' -> ')}`);
+
+        // 2. Configure Orchestrator with ONLY this chain
+        const runConfig = { ...this.testConfig, computations: chain };
+        this.orchestrator = new Orchestrator(runConfig, this.logger);
+        await this.orchestrator.initialize();
         
-        instance._meta = { logger: context.logger, config: this.config };
+        return chain;
+    }
+
+    /**
+     * Find the most recent date where Raw Data requirements are met
+     */
+    async findLatestRunnableDate(targetName, lookbackDays = 7) {
+        if (!this.orchestrator) await this.setupForComputation(targetName);
+
+        const manifest = this.orchestrator.manifest;
+        const chainNames = new Set(manifest.map(c => c.name));
+        
+        // Collect all Raw Data requirements (exclude requirements that are other computations in the chain)
+        const rawRequirements = {};
         
-        await instance.process(context);
+        for (const entry of manifest) {
+            for (const [reqName, reqSpec] of Object.entries(entry.requires)) {
+                // If the requirement is NOT a computation in our chain, it's raw data
+                rawRequirements[reqName] = reqSpec;
+            }
+        }
+
+        this.logger.log('INFO', `[Tester] Checking data availability for: ${Object.keys(rawRequirements).join(', ')}`);
+
+        // Iterate backwards from today
+        const today = new Date();
+        for (let i = 1; i <= lookbackDays; i++) {
+            const d = new Date(today);
+            d.setDate(today.getDate() - i);
+            const dateStr = d.toISOString().slice(0, 10);
+            
+            const availability = await this.orchestrator.dataFetcher.checkAvailability(rawRequirements, dateStr);
+            
+            if (availability.canRun) {
+                this.logger.log('INFO', `[Tester] Found runnable date: ${dateStr}`);
+                return dateStr;
+            } else {
+                this.logger.log('DEBUG', `[Tester] Date ${dateStr} missing: ${availability.missing.join(', ')}`);
+            }
+        }
         
-        const allResults = await instance.getResult();
-        return allResults[entityId] || allResults;
+        throw new Error(`No runnable date found in last ${lookbackDays} days.`);
     }
-    
-    async expect(entityId = 'test_entity') {
-        const result = await this.run(entityId);
+
+    async run(targetName, dateStr) {
+        if (!this.orchestrator) await this.setupForComputation(targetName);
         
+        // Ensure test table exists
+        await this.orchestrator.storageManager._ensureBigQueryTable('computation_results_test');
+        
+        this.logger.log('INFO', `[Tester] Running chain for ${targetName} on ${dateStr}`);
+        
+        // Run the DAG
+        // Orchestrator will handle passes automatically (Deps first, then Target)
+        const summary = await this.orchestrator.execute({ 
+            date: dateStr, 
+            force: true // Force run even if "up to date" in test table
+        });
+
+        // Always fetch from the standard State Repository (JSON result store)
+        const result = await this.orchestrator.stateRepository.getResult(dateStr, targetName);
+
         return {
-            result,
-            
-            toHaveProperty(prop, value) {
-                const actual = this._getNestedProperty(result, prop);
-                if (value !== undefined && actual !== value) {
-                    throw new Error(`Expected ${prop} to be ${value}, got ${actual}`);
-                }
-                if (actual === undefined) {
-                    throw new Error(`Expected ${prop} to exist`);
-                }
-                return this;
-            },
+            date: dateStr,
+            summary,
+            result
+        };
+    }
+
+    /**
+     * Recursively find all dependencies for the target computation
+     */
+    _resolveDependencyChain(targetName, allComputations) {
+        const compMap = new Map(allComputations.map(c => {
+            try {
+                const conf = c.getConfig();
+                return [conf.name.toLowerCase(), c];
+            } catch (e) { return [null, null]; }
+        }));
+
+        const chain = new Set();
+        const visit = (name) => {
+            const key = name.toLowerCase();
+            if (chain.has(key)) return;
             
-            toMatchSchema(schema) {
-                const errors = this._validateSchema(result, schema);
-                if (errors.length > 0) {
-                    throw new Error(`Schema validation failed: ${errors.join(', ')}`);
-                }
-                return this;
-            },
+            const compClass = compMap.get(key);
+            if (!compClass) throw new Error(`Computation '${name}' not found.`);
             
-            _getNestedProperty(obj, path) {
-                return path.split('.').reduce((curr, key) => curr?.[key], obj);
-            },
+            const config = compClass.getConfig();
             
-            _validateSchema(data, schema) {
-                const errors = [];
-                for (const [key, type] of Object.entries(schema)) {
-                    if (typeof data[key] !== type) {
-                        errors.push(`${key}: expected ${type}, got ${typeof data[key]}`);
-                    }
-                }
-                return errors;
+            // Recurse dependencies
+            if (config.dependencies) {
+                config.dependencies.forEach(dep => visit(dep));
             }
+            // Recurse conditional dependencies
+            if (config.conditionalDependencies) {
+                config.conditionalDependencies.forEach(cd => visit(cd.computation));
+            }
+
+            chain.add(key);
         };
+
+        visit(targetName);
+        
+        return Array.from(chain).map(k => compMap.get(k));
     }
 }
 
-module.exports = { ComputationTester };
+module.exports = { IntegrationTester };

package/functions/computation-system-v2/handlers/dispatcher.js

@@ -1,6 +1,10 @@
 /**
  * @fileoverview Computation Dispatcher
- * ...
+ * Handles incoming HTTP requests to run computations.
+ * Supports:
+ * 1. Standard execution (Scheduled/On-Demand)
+ * 2. Deployment Events (Triggers History Backfill Fan-Out)
+ * 3. Stale Task Protection (Prevents running old code versions)
  */
 
 const crypto = require('crypto');
@@ -9,6 +13,7 @@ exports.dispatcherHandler = async (req, res) => {
     const startTime = Date.now();
     
     try {
+        // Load the system entry point (index.js)
         const system = require('../index');
 
         const {
@@ -30,56 +35,84 @@ exports.dispatcherHandler = async (req, res) => {
             });
         }
 
-        // [FIXED LOGIC HERE] --------------------------------------------------
-        // Stale Task Protection
+        console.log(`[Dispatcher] Received ${source} request: ${computationName}`);
+
+        // Safety check to ensure system is loaded correctly
+        if (!system) {
+            throw new Error('System not fully initialized. Check index.js exports.');
+        }
+
+        // =====================================================================
+        // SPECIAL HANDLING: DEPLOYMENT EVENTS (Fan-Out)
+        // =====================================================================
+        if (source === 'deployment') {
+            if (typeof system.triggerBackfill !== 'function') {
+                throw new Error('System does not support auto-backfill (triggerBackfill missing).');
+            }
+
+            console.log(`[Dispatcher] 🚀 Triggering Deployment Backfill for ${computationName}...`);
+            const stats = await system.triggerBackfill(computationName);
+
+            const duration = Date.now() - startTime;
+            console.log(`[Dispatcher] Deployment processed in ${duration}ms. Scheduled ${stats.scheduled} tasks.`);
+
+            return res.status(200).json({
+                status: 'triggered',
+                computation: computationName,
+                action: 'backfill_fan_out',
+                scheduledTasks: stats.scheduled,
+                duration
+            });
+        }
+        // =====================================================================
+
+        // 2. Stale Task Protection
+        // Prevents execution if the task was scheduled with an older version of the configuration
         if (configHash && !force) {
-            // FIX: Use getManifest() as system.manifest is not exposed directly.
             const manifest = await system.getManifest();
             
-            // Normalize name to match manifest keys (matches logic in core-api.js)
+            // Normalize name to match manifest keys
             const normalizedName = computationName.toLowerCase().replace(/[^a-z0-9]/g, '');
             const entry = manifest.find(c => c.name === normalizedName);
             
             if (entry) {
-                // 1. Re-calculate the hash of the CURRENTLY DEPLOYED code
+                // Re-calculate the hash of the CURRENTLY DEPLOYED code
                 const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
                 const currentHash = crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
                 
-                // 2. Compare
+                // Compare Task Hash vs Current Hash
                 if (configHash !== currentHash) {
                     console.warn(`[Dispatcher] ♻️ Skipped STALE task for ${computationName}. (Task Hash: ${configHash} != Current: ${currentHash})`);
                     
                     return res.status(200).json({
                         status: 'skipped',
                         reason: 'STALE_CONFIG',
-                        message: 'Task configuration (schedule/pass) is obsolete relative to current deployment.',
+                        message: 'Task configuration is obsolete relative to current deployment.',
                         hash: currentHash
                     });
                 }
             }
         }
-        // ---------------------------------------------------------------------
 
+        // 3. Prepare Execution
         const date = targetDate || new Date().toISOString().split('T')[0];
-        console.log(`[Dispatcher] Received ${source} request: ${computationName} for ${date}`);
-
-        // Safety check to ensure system is loaded correctly
-        if (!system || typeof system.runComputation !== 'function') {
-            throw new Error('System not fully initialized (runComputation is missing). Check index.js exports.');
-        }
         
-        // 2. DELEGATE TO ORCHESTRATOR
+        // 4. DELEGATE TO ORCHESTRATOR
+        if (typeof system.runComputation !== 'function') {
+            throw new Error('system.runComputation is missing.');
+        }
+
         const result = await system.runComputation({
             date,
             computation: computationName,
             entityIds,
             dryRun,
-            force // Pass force down to orchestrator if needed
+            force
         });
 
         const duration = Date.now() - startTime;
 
-        // 3. HANDLE SUCCESS (Completed or Skipped/Up-to-date)
+        // 5. HANDLE SUCCESS (Completed or Skipped/Up-to-date)
         if (result.status === 'completed' || result.status === 'skipped') {
             console.log(`[Dispatcher] ${computationName} ${result.status}: ${result.resultCount || 0} entities in ${duration}ms`);
             
@@ -94,7 +127,7 @@ exports.dispatcherHandler = async (req, res) => {
             });
         } 
         
-        // 4. HANDLE NON-RUNNABLE STATES (Blocked / Impossible)
+        // 6. HANDLE NON-RUNNABLE STATES (Blocked / Impossible)
         if (result.status === 'blocked' || result.status === 'impossible') {
             console.log(`[Dispatcher] ${computationName} ${result.status}: ${result.reason}`);
 
@@ -105,24 +138,19 @@ exports.dispatcherHandler = async (req, res) => {
             });
         }
 
-        // 5. Fallback for other statuses
+        // 7. Fallback
         return res.status(200).json(result);
         
     } catch (error) {
         const duration = Date.now() - startTime;
         console.error(`[Dispatcher] Error after ${duration}ms:`, error);
         
-        if (error.message && error.message.includes('Computation not found')) {
-            return res.status(400).json({
-                status: 'error',
-                reason: 'UNKNOWN_COMPUTATION',
-                message: error.message
-            });
-        }
+        const statusCode = (error.message && error.message.includes('Computation not found')) ? 400 : 500;
+        const reason = statusCode === 400 ? 'UNKNOWN_COMPUTATION' : 'EXECUTION_FAILED';
 
-        return res.status(500).json({
+        return res.status(statusCode).json({
             status: 'error',
-            reason: 'EXECUTION_FAILED',
+            reason: reason,
             message: error.message
         });
     }

package/functions/computation-system-v2/legacy/PiAssetRecommender.js.old

@@ -0,0 +1,115 @@
+const { Computation } = require('../framework');
+
+class PiAssetRecommender extends Computation {
+    static getConfig() {
+        return {
+            name: 'PiAssetRecommender',
+            type: 'per-entity', // Recommendation is specific to THIS user
+            category: 'recommendations',
+            
+            // 1. We rely on Global computations that pre-calculate the similarity graph
+            dependencies: ['PiSimilarityMatrix', 'PiTopHoldings'],
+            
+            requires: {
+                'portfolio_snapshots': {
+                    lookback: 0,
+                    mandatory: true,
+                    fields: ['user_id', 'portfolio_data'] // Needed to get Mirrors
+                },
+                'user_watchlists': { // Assuming table existence
+                    lookback: 0,
+                    mandatory: false,
+                    fields: ['user_id', 'watched_pi_ids']
+                }
+            },
+            
+            storage: {
+                bigquery: true,
+                firestore: { enabled: true, path: 'users/{entityId}/recommendations' }
+            }
+        };
+    }
+
+    async process(context) {
+        const { data, entityId, rules, getDependency } = context;
+
+        // 1. Get User's Signals (Mirrors + Watchlist)
+        const seedPIs = new Set();
+        
+        // A. From Mirrors
+        const portfolioRow = data['portfolio_snapshots'];
+        if (portfolioRow) {
+            const pData = rules.portfolio.extractPortfolioData(portfolioRow);
+            const mirrors = rules.portfolio.extractMirrors(pData);
+            mirrors.forEach(m => {
+                if (m.ParentCID) seedPIs.add(String(m.ParentCID));
+            });
+        }
+
+        // B. From Watchlist
+        const watchlistRow = data['user_watchlists'];
+        if (watchlistRow && watchlistRow.watched_pi_ids) {
+            // Assuming array of IDs in column
+            const ids = Array.isArray(watchlistRow.watched_pi_ids) 
+                ? watchlistRow.watched_pi_ids 
+                : JSON.parse(watchlistRow.watched_pi_ids);
+            ids.forEach(id => seedPIs.add(String(id)));
+        }
+
+        if (seedPIs.size === 0) return; // No signals, no recommendations
+
+        // 2. Load Global Context (The Graph) via Dependencies
+        // These are large objects, so typically we fetch specific parts or the whole thing
+        // Assuming getDependency returns a Map of PI_ID -> Data
+        const similarityGraph = getDependency('PiSimilarityMatrix'); 
+        const piHoldings = getDependency('PiTopHoldings');
+
+        if (!similarityGraph || !piHoldings) return;
+
+        // 3. Find Similar PIs (Collaborative Filtering)
+        const candidateAssets = new Map(); // AssetID -> Score
+
+        seedPIs.forEach(seedId => {
+            // Find PIs similar to the ones I already copy/watch
+            const neighbors = similarityGraph[seedId] || []; // List of { id, score }
+            
+            neighbors.forEach(neighbor => {
+                const neighborId = String(neighbor.id);
+                const similarityScore = neighbor.score;
+
+                // What does this neighbor hold?
+                const holdings = piHoldings[neighborId] || []; // List of { assetId, weight }
+                
+                holdings.forEach(holding => {
+                    const currentScore = candidateAssets.get(holding.assetId) || 0;
+                    // Boost score based on PI similarity and holding weight
+                    candidateAssets.set(holding.assetId, currentScore + (similarityScore * holding.weight));
+                });
+            });
+        });
+
+        // 4. Filter Assets User Already Owns
+        // (Re-use portfolio logic)
+        const myAssets = new Set();
+        if (portfolioRow) {
+            const pData = rules.portfolio.extractPortfolioData(portfolioRow);
+            const positions = rules.portfolio.extractPositions(pData);
+            positions.forEach(p => myAssets.add(String(rules.portfolio.getInstrumentId(p))));
+        }
+
+        // 5. Final Ranking
+        const recommendations = Array.from(candidateAssets.entries())
+            .filter(([assetId]) => !myAssets.has(String(assetId))) // Exclude owned
+            .sort((a, b) => b[1] - a[1]) // Sort by score DESC
+            .slice(0, 5) // Top 5
+            .map(([assetId, score]) => ({
+                assetId: Number(assetId),
+                score: Number(score.toFixed(4)),
+                reason: 'Held by investors similar to those you copy'
+            }));
+
+        this.setResult(entityId, { recommendations });
+    }
+}
+
+module.exports = PiAssetRecommender;

package/functions/computation-system-v2/legacy/PiSimilarityMatrix.js

@@ -0,0 +1,104 @@
+/**
+ * @fileoverview Similarity Matrix Meta-Computation.
+ * Calculates cosine similarity between all PI vectors to find nearest neighbors.
+ */
+class PiSimilarityMatrix {
+    constructor() { this.results = {}; }
+
+    static getMetadata() {
+        return {
+            type: 'meta', // Runs once per date
+            category: 'popular_investor_meta',
+            userType: 'POPULAR_INVESTOR'
+        };
+    }
+
+    // DEPENDENCY: Forces this to run in Pass 2
+    static getDependencies() { return ['PiSimilarityVector']; } 
+
+    static getSchema() {
+        return {
+            type: 'object', // Map<UserId, Array<SimilarUser>>
+            description: 'Top 5 similar users for each PI'
+        };
+    }
+
+    async process(context) {
+        const vectorMap = context.computed['PiSimilarityVector'];
+        
+        // [FIX] Ensure we have enough peers to actually compare
+        if (!vectorMap || Object.keys(vectorMap).length < 2) {
+            this.results = {}; // Explicitly set empty
+            return this.results; 
+        }
+
+        const userIds = Object.keys(vectorMap);
+        const matrix = {};
+
+        for (let i = 0; i < userIds.length; i++) {
+            const u1 = userIds[i];
+            const v1 = vectorMap[u1];
+            
+            // Skip empty vectors
+            if (!v1 || Object.keys(v1).length === 0) continue;
+
+            const matches = [];
+
+            // Compare against every other user
+            for (let j = 0; j < userIds.length; j++) {
+                if (i === j) continue; // Don't compare self
+
+                const u2 = userIds[j];
+                const v2 = vectorMap[u2];
+
+                if (!v2 || Object.keys(v2).length === 0) continue;
+
+                // Calculate Cosine Similarity
+                let dotProduct = 0;
+                let mag1 = 0;
+                let mag2 = 0;
+
+                // v1 magnitude & dot product
+                for (const [k, val] of Object.entries(v1)) {
+                    mag1 += (val * val);
+                    if (v2[k]) dotProduct += (val * v2[k]);
+                }
+                
+                // v2 magnitude
+                for (const val of Object.values(v2)) {
+                    mag2 += (val * val);
+                }
+
+                mag1 = Math.sqrt(mag1);
+                mag2 = Math.sqrt(mag2);
+
+                if (mag1 > 0 && mag2 > 0) {
+                    const similarity = dotProduct / (mag1 * mag2);
+                    if (similarity > 0.1) { // Threshold to filter noise
+                        matches.push({ 
+                            id: u2, 
+                            score: Number(similarity.toFixed(4)),
+                            reason: "Portfolio Composition Match" 
+                        });
+                    }
+                }
+            }
+
+            // Sort by score desc and take top 5
+            matrix[u1] = matches.sort((a,b) => b.score - a.score).slice(0, 5);
+        }
+
+        // --- CRITICAL FIX ---
+        // 1. Assign to class property (for getResult)
+        this.results = matrix;
+
+        // 2. Return to MetaExecutor (for wrapping)
+        return this.results;
+    }
+
+    async getResult() {
+        return this.results;
+    }
+}
+
+module.exports = PiSimilarityMatrix;