bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -0,0 +1,317 @@
+/**
+ * @fileoverview BullTrackers Configuration for Computation System v2
+ * 
+ * This is the ONLY file that contains BullTrackers-specific knowledge.
+ * The framework itself is completely domain-agnostic.
+ * 
+ * Business Rules:
+ * - Rules are injected into computations automatically
+ * - When a rule changes, computations using it are re-run
+ * - Computations should be "simple recipes" that call rules
+ */
+
+// Load business rules
+const rules = require('../rules');
+
+module.exports = {
+    // =========================================================================
+    // PROJECT CONFIGURATION
+    // =========================================================================
+    
+    project: {
+        id: 'bulltrackers',
+        name: 'BullTrackers Analytics',
+        version: '2.0.0'
+    },
+    
+    // System epoch - bump this to force all computations to re-run
+    epoch: 'v2.0.0',
+    
+    // =========================================================================
+    // BIGQUERY CONFIGURATION
+    // =========================================================================
+    
+    bigquery: {
+        projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
+        dataset: process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data',
+        location: 'europe-west1',
+        cacheTTLMs: 3600000  // 1 hour schema cache
+    },
+    
+    // =========================================================================
+    // TABLE DEFINITIONS
+    // =========================================================================
+    // 
+    // Each table entry tells the framework:
+    // - dateField: Which column is used for date partitioning (null if not partitioned)
+    // - entityField: Which column identifies the entity (user, asset, etc.)
+    // - dataField: If the actual data is in a JSON column, which one
+    //
+    // The framework will automatically discover the full schema from BigQuery.
+    // These hints just tell it how to interpret the data.
+    // =========================================================================
+    
+    tables: {
+        // User Portfolio Snapshots
+        'portfolio_snapshots': {
+            dateField: 'date',
+            entityField: 'user_id',
+            dataField: 'portfolio_data',
+            description: 'Daily portfolio snapshots for all users'
+        },
+        
+        // User Trade History
+        'trade_history_snapshots': {
+            dateField: 'date',
+            entityField: 'user_id',
+            dataField: 'history_data',
+            description: 'Daily trade history snapshots'
+        },
+        
+        // Social Posts
+        'social_post_snapshots': {
+            dateField: 'date',
+            entityField: 'user_id',
+            dataField: 'posts_data',
+            description: 'Daily social post snapshots'
+        },
+        
+        // Asset Prices
+        'asset_prices': {
+            dateField: 'date',
+            entityField: 'instrument_id',
+            dataField: null,  // Flat table
+            description: 'Daily asset prices'
+        },
+        
+        // PI Rankings
+        'pi_rankings': {
+            dateField: 'date',
+            entityField: 'pi_id',
+            dataField: 'rankings_data',
+            description: 'Daily PI rankings snapshot'
+        },
+        
+        // PI Master List (not date-partitioned)
+        'pi_master_list': {
+            dateField: null,  // Not date-partitioned
+            entityField: 'cid',
+            dataField: null,
+            description: 'Master list of all Popular Investors'
+        },
+        
+        // PI Ratings
+        'pi_ratings': {
+            dateField: 'date',
+            entityField: 'pi_id',
+            dataField: null,
+            description: 'Daily PI ratings'
+        },
+        
+        // PI Page Views
+        'pi_page_views': {
+            dateField: 'date',
+            entityField: 'pi_id',
+            dataField: null,
+            description: 'Daily PI page view metrics'
+        },
+        
+        // Watchlist Membership
+        'watchlist_membership': {
+            dateField: 'date',
+            entityField: 'pi_id',
+            dataField: null,
+            description: 'Daily watchlist membership counts'
+        },
+        
+        // PI Alert History
+        'pi_alert_history': {
+            dateField: 'date',
+            entityField: 'pi_id',
+            dataField: 'metadata',
+            description: 'Daily alert trigger history'
+        },
+        
+        // Instrument Insights
+        'instrument_insights': {
+            dateField: 'date',
+            entityField: 'instrument_id',
+            dataField: 'insights_data',
+            description: 'Daily instrument insights'
+        },
+        
+        // Ticker Mappings (not date-partitioned)
+        'ticker_mappings': {
+            dateField: null,
+            entityField: 'instrument_id',
+            dataField: null,
+            description: 'Instrument ID to ticker symbol mappings'
+        },
+        
+        // Computation Results
+        'computation_results': {
+            dateField: 'date',
+            entityField: null,  // Keyed by computation_name
+            dataField: 'result_data',
+            description: 'Stored computation results'
+        },
+        // NEW: Sector Mappings Table
+        'sector_mappings': {
+            dateField: null,      // Static data
+            entityField: 'symbol', // Key the data by symbol for fast lookup
+            dataField: null,
+            description: 'Ticker to Sector mappings migrated from Firestore'
+        }
+    },
+    
+    // NEW: Data to load globally for every computation
+    referenceData: [
+        'sector_mappings'
+    ],
+    
+    // =========================================================================
+    // RESULT STORAGE CONFIGURATION
+    // =========================================================================
+    // 
+    // Using a separate v2 table to avoid conflicts with v1 schema.
+    // v1 table: computation_results (date, computation_name, category, result_data, metadata, created_at)
+    // v2 table: computation_results_v2 (with entity_id, code_hash, etc.)
+    // =========================================================================
+    
+    resultStore: {
+        // Using v3 table since v2 has streaming buffer data that blocks DML
+        // Jobs-based inserts are FREE and don't have streaming buffer issues
+        table: 'computation_results_v3',
+        partitionField: 'date',
+        clusterFields: ['computation_name', 'category']
+    },
+    
+    // =========================================================================
+    // COMPUTATIONS
+    // =========================================================================
+    // 
+    // Computations are registered here. During development, we add them one
+    // by one as they're migrated from v1.
+    // =========================================================================
+    
+    computations: [
+        // Add migrated computations here:
+        // require('../computations/UserRiskScore'),
+    ],
+    
+    // =========================================================================
+    // PREDEFINED FILTER SETS
+    // =========================================================================
+    // 
+    // Computations can reference these by name instead of hardcoding filters.
+    // =========================================================================
+    
+    filterSets: {
+        'popular_investors': {
+            user_type: 'POPULAR_INVESTOR'
+        },
+        'signed_in_users': {
+            user_type: 'SIGNED_IN_USER'
+        },
+        'all_tracked_users': {
+            user_type: ['POPULAR_INVESTOR', 'SIGNED_IN_USER']
+        }
+    },
+    
+    // =========================================================================
+    // BUSINESS RULES
+    // =========================================================================
+    // 
+    // Rules are automatically injected into computations.
+    // When a rule changes, all computations using it are re-run.
+    // 
+    // Usage in computation:
+    //   const positions = rules.portfolio.extractPositions(data);
+    //   const sharpe = rules.metrics.calculateSharpeRatio(returns);
+    // =========================================================================
+    
+    rules,
+    
+    // =========================================================================
+    // EXECUTION CONFIGURATION
+    // =========================================================================
+    
+    execution: {
+        // Max concurrent entity processing (per-entity computations)
+        // Higher = faster but more memory. Tune based on your Cloud Function memory.
+        entityConcurrency: 20,
+        
+        // Batch size for BigQuery inserts
+        insertBatchSize: 500,
+        
+        // Memory safety: max entities to load for a dependency
+        // If a dependency has more entities than this, use getDependency(name, entityId) instead
+        // This prevents OOM when running many concurrent per-entity computations
+        // Example: 20 concurrent * 50KB per entity * 50000 entities = 50GB (bad!)
+        // With limit: 20 concurrent * 50KB per entity * 10000 entities = 10GB (still risky at 2GB RAM)
+        // Recommendation: Set this based on your Cloud Function memory
+        maxDependencyEntities: 10000
+    },
+    
+    // =========================================================================
+    // SCHEDULING CONFIGURATION
+    // =========================================================================
+    // 
+    // Controls how computations are scheduled and dispatched.
+    // Cloud Tasks handles throttling and retry via queue configuration.
+    // =========================================================================
+    
+    scheduling: {
+        // Default schedule for computations that don't declare one
+        default: {
+            frequency: 'daily',
+            time: '02:00',
+            timezone: 'UTC'
+        },
+        
+        // Minimum gap between dependent computations (minutes)
+        // Pass 1 @ 14:00 → Pass 2 must be >= 14:15
+        dependencyGapMinutes: 15
+    },
+    
+    // =========================================================================
+    // CLOUD TASKS CONFIGURATION
+    // =========================================================================
+    // 
+    // Single queue handles all scheduled triggers.
+    // Queue settings (maxConcurrent, retry) are configured in GCP, not here.
+    // =========================================================================
+    
+    cloudTasks: {
+        projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
+        location: 'europe-west1',
+        queueName: 'computation-triggers',
+        dispatcherUrl: process.env.DISPATCHER_URL || 
+            'https://europe-west1-stocks-12345.cloudfunctions.net/computeDispatcher',
+        // Service account for OIDC authentication when invoking Dispatcher
+        // This SA needs roles/cloudfunctions.invoker on the Dispatcher function
+        serviceAccountEmail: process.env.CLOUD_TASKS_SA_EMAIL || 
+            'computation-scheduler@stocks-12345.iam.gserviceaccount.com'
+    },
+    
+    // =========================================================================
+    // ON-DEMAND API CONFIGURATION
+    // =========================================================================
+    // 
+    // Frontend-triggered computation requests.
+    // Routes through Dispatcher for validation.
+    // =========================================================================
+    
+    onDemand: {
+        // Rate limiting per user
+        maxRequestsPerMinute: 5,
+        
+        // Request timeout (ms) - frontend is waiting
+        timeout: 60000,
+        
+        // Which computations can be triggered on-demand
+        // null = all computations allowed
+        // array = only listed computations allowed
+        allowedComputations: null
+    }
+};

package/functions/computation-system-v2/framework/core/Computation.js

@@ -0,0 +1,73 @@
+/**
+ * @fileoverview Base Computation Class
+ */
+
+class Computation {
+    constructor() {
+        this.results = {};
+        this._meta = {};
+    }
+    
+    static getConfig() {
+        throw new Error('Computation.getConfig() must be implemented by subclass');
+    }
+    
+    static validateConfig() {
+        const errors = [];
+        try {
+            const config = this.getConfig();
+            
+            if (!config.name || typeof config.name !== 'string') errors.push('name is required and must be a string');
+            if (!config.requires || typeof config.requires !== 'object') errors.push('requires is required and must be an object');
+            if (config.dependencies && !Array.isArray(config.dependencies)) errors.push('dependencies must be an array');
+            
+            // NEW: Validation for conditional dependencies
+            if (config.conditionalDependencies) {
+                if (!Array.isArray(config.conditionalDependencies)) {
+                    errors.push('conditionalDependencies must be an array');
+                } else {
+                    config.conditionalDependencies.forEach((cd, idx) => {
+                        if (!cd.computation || typeof cd.computation !== 'string') {
+                            errors.push(`conditionalDependencies[${idx}].computation must be a string`);
+                        }
+                        if (!cd.condition || typeof cd.condition !== 'function') {
+                            errors.push(`conditionalDependencies[${idx}].condition must be a function`);
+                        }
+                    });
+                }
+            }
+
+            if (config.type && !['global', 'per-entity'].includes(config.type)) errors.push('type must be "global" or "per-entity"');
+            
+        } catch (e) {
+            errors.push(`getConfig() threw: ${e.message}`);
+        }
+        return { valid: errors.length === 0, errors };
+    }
+    
+    async process(context) { throw new Error('Computation.process() must be implemented by subclass'); }
+    async getResult() { return this.results; }
+    static getSchema() { return null; }
+    static getWeight() { return 1.0; }
+    
+    setResult(entityId, result) { this.results[entityId] = result; }
+    setGlobalResult(result) { this.results = result; }
+    get(obj, path, defaultValue = null) {
+        if (!obj || !path) return defaultValue;
+        const keys = path.split('.');
+        let current = obj;
+        for (const key of keys) {
+            if (current === null || current === undefined) return defaultValue;
+            current = current[key];
+        }
+        return current !== undefined ? current : defaultValue;
+    }
+    log(level, message) {
+        const config = this.constructor.getConfig();
+        const prefix = `[${config.name}]`;
+        if (this._meta.logger) this._meta.logger.log(level, `${prefix} ${message}`);
+        else console.log(`${level}: ${prefix} ${message}`);
+    }
+}
+
+module.exports = { Computation };

package/functions/computation-system-v2/framework/core/Manifest.js

@@ -0,0 +1,223 @@
+/**
+ * @fileoverview Manifest Builder
+ * * Responsible for:
+ * 1. Configuration validation
+ * 2. Hash generation (Version Control)
+ * 3. Schedule validation
+ * 4. Dependency resolution (delegated to Graph utils)
+ */
+
+const crypto = require('crypto');
+const { RulesRegistry } = require('./Rules');
+const { ScheduleValidator } = require('../scheduling/ScheduleValidator');
+const { Graph } = require('../utils/Graph');
+
+class ManifestBuilder {
+    constructor(config, logger = null) {
+        this.config = config;
+        this.logger = logger;
+        this.epoch = config.epoch || 'v2.0.0';
+        
+        this.sharedLayers = config.sharedLayers || {};
+        this.layerHashes = this._hashSharedLayers();
+        this.rulesRegistry = new RulesRegistry(config, logger);
+        this.scheduleValidator = new ScheduleValidator(config, logger);
+    }
+    
+    getRulesRegistry() { return this.rulesRegistry; }
+    getScheduleValidator() { return this.scheduleValidator; }
+
+    build(computations) {
+        this._log('INFO', `Building manifest for ${computations.length} computations...`);
+        
+        const manifestMap = new Map();
+        const adjacency = new Map();
+        
+        // 1. Process Computations
+        for (const ComputationClass of computations) {
+            const entry = this._processComputation(ComputationClass);
+            if (entry) {
+                manifestMap.set(entry.name, entry);
+                adjacency.set(entry.name, entry.dependencies);
+            }
+        }
+        
+        // 2. Validate Dependencies
+        const nodes = Array.from(manifestMap.keys());
+        const cycle = Graph.detectCycle(nodes, adjacency);
+        if (cycle) {
+            throw new Error(`[Manifest] Circular dependency detected: ${cycle}`);
+        }
+
+        // 3. Topological Sort (calculates passes)
+        const sortedItems = Graph.topologicalSort(nodes, adjacency);
+        
+        // 4. Hydrate Sorted List
+        const finalManifest = sortedItems.map(item => {
+            const entry = manifestMap.get(item.id);
+            entry.pass = item.pass;
+            return entry;
+        });
+
+        // 5. Finalize Hashes
+        this._computeFinalHashes(finalManifest, manifestMap);
+        
+        // 6. Schedule Validation
+        this._validateSchedules(finalManifest);
+
+        return finalManifest;
+    }
+
+    _processComputation(ComputationClass) {
+        if (typeof ComputationClass.getConfig !== 'function') {
+            this._log('ERROR', `Missing static getConfig(): ${ComputationClass.name}`);
+            return null;
+        }
+        
+        const validation = ComputationClass.validateConfig();
+        if (!validation.valid) {
+            this._log('ERROR', `Invalid config for ${ComputationClass.name}: ${validation.errors.join(', ')}`);
+            return null;
+        }
+        
+        const config = ComputationClass.getConfig();
+        const name = this._normalize(config.name);
+        const codeString = ComputationClass.toString();
+        const codeHash = this._hashCode(codeString);
+        
+        // Hash Composition
+        const usedLayers = this._detectLayerUsage(codeString);
+        const { usedRules, hashes: ruleHashes } = this.rulesRegistry.detectUsage(codeString);
+        
+        let compositeHash = codeHash + `|EPOCH:${this.epoch}`;
+        
+        const layerHashes = {};
+        for (const [layerName, exports] of Object.entries(usedLayers)) {
+            const h = this._hashLayerExports(layerName, exports);
+            if (h) {
+                layerHashes[layerName] = h;
+                compositeHash += `|LAYER:${layerName}:${h}`;
+            }
+        }
+        
+        for (const [mod, h] of Object.entries(ruleHashes).sort()) {
+            compositeHash += `|RULE:${mod}:${h}`;
+        }
+        
+        return {
+            name,
+            originalName: config.name,
+            class: ComputationClass,
+            category: config.category || 'default',
+            type: config.type || 'global',
+            requires: config.requires || {},
+            dependencies: (config.dependencies || []).map(d => this._normalize(d)),
+            isHistorical: config.isHistorical || false,
+            isTest: config.isTest || false,
+            schedule: this.scheduleValidator.parseSchedule(config.schedule),
+            storage: this._parseStorageConfig(config.storage),
+            ttlDays: config.ttlDays,
+            pass: 0, // Set later by Graph.js
+            hash: this._hashCode(compositeHash), // Intrinsic hash
+            weight: ComputationClass.getWeight ? ComputationClass.getWeight() : 1.0,
+            composition: {
+                epoch: this.epoch,
+                code: codeHash,
+                layers: layerHashes,
+                rules: ruleHashes,
+                deps: {}
+            }
+        };
+    }
+
+    _computeFinalHashes(sorted, manifestMap) {
+        for (const entry of sorted) {
+            let hashInput = entry.hash;
+            if (entry.dependencies.length > 0) {
+                const depHashes = entry.dependencies.sort().map(d => {
+                    const h = manifestMap.get(d)?.hash;
+                    entry.composition.deps[d] = h;
+                    return h;
+                });
+                hashInput += `|DEPS:${depHashes.join('|')}`;
+            }
+            entry.hash = this._hashCode(hashInput);
+        }
+    }
+
+    _validateSchedules(manifest) {
+        const validation = this.scheduleValidator.validate(manifest);
+        validation.errors.forEach(e => this._log('ERROR', `Schedule error: ${e.message}`));
+        validation.warnings.forEach(w => this._log('WARN', `Schedule warning: ${w.message}`));
+    }
+
+    // -- Helpers --
+    
+    _hashSharedLayers() {
+        const hashes = {};
+        for (const [name, exports] of Object.entries(this.sharedLayers)) {
+            hashes[name] = {};
+            Object.keys(exports).sort().forEach(k => {
+                const val = exports[k];
+                const str = typeof val === 'function' ? val.toString() : JSON.stringify(val);
+                hashes[name][k] = this._hashCode(`LAYER:${name}:${k}:${str}`);
+            });
+        }
+        return hashes;
+    }
+
+    _hashLayerExports(layerName, exports) {
+        const vals = [];
+        for (const exp of exports) {
+            const h = this.layerHashes[layerName]?.[exp];
+            if (h) vals.push(h);
+        }
+        return vals.length ? this._hashCode(vals.sort().join('|')) : null;
+    }
+
+    _detectLayerUsage(code) {
+        const used = {};
+        for (const [name, exports] of Object.entries(this.sharedLayers)) {
+            const found = Object.keys(exports).filter(exp => 
+                code.includes(exp) // Simple include check, similar to original regex
+            );
+            if (found.length) used[name] = found;
+        }
+        return used;
+    }
+
+    _parseStorageConfig(cfg) {
+        return {
+            bigquery: cfg?.bigquery !== false,
+            firestore: {
+                enabled: cfg?.firestore?.enabled === true,
+                path: cfg?.firestore?.path || null,
+                merge: cfg?.firestore?.merge || false,
+                includeMetadata: cfg?.firestore?.includeMetadata !== false
+            }
+        };
+    }
+
+    _groupByPass(manifest) {
+        const passes = {};
+        manifest.forEach(e => {
+            if (!passes[e.pass]) passes[e.pass] = [];
+            passes[e.pass].push(e);
+        });
+        return passes;
+    }
+
+    _hashCode(s) {
+        const cleaned = s.replace(/\s+/g, '');
+        return crypto.createHash('sha256').update(cleaned).digest('hex').substring(0, 16);
+    }
+    
+    _normalize(n) { return n.toLowerCase().replace(/[^a-z0-9]/g, ''); }
+    
+    _log(l, m) { this.logger ? this.logger.log(l, `[Manifest] ${m}`) : console.log(`[Manifest] ${m}`); }
+    
+    // Public alias for groupByPass matching the Interface
+    groupByPass(m) { return this._groupByPass(m); }
+}
+
+module.exports = { ManifestBuilder };

package/functions/computation-system-v2/framework/core/RuleInjector.js

@@ -0,0 +1,53 @@
+/**
+ * @fileoverview Rule Injector (Proxy Pattern)
+ * * Wraps the RulesRegistry with a Proxy to:
+ * 1. Lazy-load rule modules only when accessed
+ * 2. Track exactly which rules are used during execution
+ * 3. Eliminate the need for hardcoded dependency lists
+ */
+
+class RuleInjector {
+    constructor(registry) {
+        this.registry = registry;
+    }
+
+    /**
+     * Creates a proxied rules object and a tracking set.
+     * @returns {Object} { rules: Proxy, used: Set<string> }
+     */
+    createContext() {
+        const used = new Set();
+        const registry = this.registry;
+        
+        // cache loaded modules for this instance to avoid repeated lookups
+        const cache = {};
+
+        const rulesProxy = new Proxy({}, {
+            get: (target, prop) => {
+                const moduleName = prop.toString();
+                
+                // 1. Track usage
+                used.add(moduleName);
+
+                // 2. Return from local cache if available
+                if (cache[moduleName]) return cache[moduleName];
+
+                // 3. Lazy Load from Registry
+                // We access the raw context from the registry
+                const fullContext = registry.getContext();
+                
+                if (fullContext[moduleName]) {
+                    cache[moduleName] = fullContext[moduleName];
+                    return cache[moduleName];
+                }
+
+                // Return undefined if rule module doesn't exist
+                return undefined;
+            }
+        });
+
+        return { rules: rulesProxy, used };
+    }
+}
+
+module.exports = { RuleInjector };