bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/handlers/scheduler.js

@@ -0,0 +1,327 @@
+/**
+ * @fileoverview Unified Computation Scheduler
+ * 
+ * Single Cloud Function triggered every minute by Cloud Scheduler.
+ * Checks all computations, dispatches those that are due to Cloud Tasks.
+ * 
+ * Architecture:
+ * 
+ *   Cloud Scheduler (every minute, * * * * *)
+ *          │
+ *          ▼
+ *   ┌─────────────────────────────────────────────┐
+ *   │  Scheduler Cloud Function (this file)       │
+ *   │  1. Floor current time to minute boundary   │
+ *   │  2. Check each computation's schedule       │
+ *   │  3. Enqueue due computations to Cloud Tasks │
+ *   └─────────────────────────────────────────────┘
+ *          │
+ *          ▼  (via Cloud Tasks queue)
+ *   ┌─────────────────────────────────────────────┐
+ *   │  Dispatcher Cloud Function                  │
+ *   │  - Validates dependencies                   │
+ *   │  - Executes computation                     │
+ *   │  - Returns 503 if blocked (Cloud Tasks      │
+ *   │    will retry with backoff)                 │
+ *   └─────────────────────────────────────────────┘
+ * 
+ * Clock Drift Handling:
+ * - Scheduler might run at 14:00:58 instead of 14:00:00
+ * - We floor to minute boundary: 14:00:58 → 14:00
+ * - Schedule check uses 14:00, payload uses 14:00
+ * - System behaves as if it ran exactly on time
+ * 
+ * Rate Limiting:
+ * - Uses p-limit to control concurrent Cloud Tasks API calls
+ * - Prevents hitting GCP API quotas
+ */
+
+const { CloudTasksClient } = require('@google-cloud/tasks');
+const pLimit = require('p-limit');
+const { ManifestBuilder, ScheduleValidator } = require('../framework');
+const config = require('../config/bulltrackers.config');
+
+// Concurrency limit for Cloud Tasks API calls
+const CLOUD_TASKS_CONCURRENCY = 10;
+
+// Singleton instances
+let manifest = null;
+let scheduleValidator = null;
+let tasksClient = null;
+
+/**
+ * Initialize manifest and schedule validator.
+ */
+async function initialize() {
+    if (manifest) return;
+    
+    console.log('[Scheduler] Initializing...');
+    
+    const builder = new ManifestBuilder(config, { log: (l, m) => console.log(`[${l}] ${m}`) });
+    manifest = builder.build(config.computations || []);
+    scheduleValidator = builder.getScheduleValidator();
+    tasksClient = new CloudTasksClient();
+    
+    console.log(`[Scheduler] Initialized with ${manifest.length} computations`);
+}
+
+/**
+ * Main scheduler handler.
+ * Triggered by Cloud Scheduler every minute.
+ * 
+ * @param {Object} req - HTTP request
+ * @param {Object} res - HTTP response
+ */
+async function schedulerHandler(req, res) {
+    const startTime = Date.now();
+    
+    try {
+        await initialize();
+        
+        // Get current time, floored to minute boundary
+        // This handles clock drift - if we run at 14:00:58, we treat it as 14:00:00
+        const now = floorToMinute(new Date());
+        const targetDate = formatDate(now);
+        const currentTime = formatTime(now);
+        
+        console.log(`[Scheduler] Running for ${targetDate} ${currentTime}`);
+        
+        // Find computations due at this time
+        const dueComputations = findDueComputations(now);
+        
+        if (dueComputations.length === 0) {
+            console.log(`[Scheduler] No computations due at ${currentTime}`);
+            return res.status(200).json({
+                status: 'ok',
+                time: currentTime,
+                dispatched: 0,
+                message: 'No computations due'
+            });
+        }
+        
+        console.log(`[Scheduler] ${dueComputations.length} computations due: ${dueComputations.map(c => c.name).join(', ')}`);
+        
+        // Dispatch to Cloud Tasks with rate limiting
+        // Pass 'now' for idempotent task naming (retry-safe)
+        const results = await dispatchComputations(dueComputations, targetDate, now);
+        
+        const duration = Date.now() - startTime;
+        const succeeded = results.filter(r => r.status === 'dispatched').length;
+        const failed = results.filter(r => r.status === 'error').length;
+        
+        console.log(`[Scheduler] Dispatched ${succeeded}/${dueComputations.length} in ${duration}ms`);
+        
+        return res.status(200).json({
+            status: 'ok',
+            time: currentTime,
+            date: targetDate,
+            dispatched: succeeded,
+            failed,
+            duration,
+            results
+        });
+        
+    } catch (error) {
+        console.error('[Scheduler] Error:', error);
+        return res.status(500).json({
+            status: 'error',
+            message: error.message
+        });
+    }
+}
+
+/**
+ * Find all computations that are due at the given time.
+ * 
+ * @param {Date} now - Current time (floored to minute)
+ * @returns {Array} Array of manifest entries that are due
+ */
+function findDueComputations(now) {
+    const due = [];
+    const currentHour = now.getUTCHours();
+    const currentMinute = now.getUTCMinutes();
+    const currentTime = `${String(currentHour).padStart(2, '0')}:${String(currentMinute).padStart(2, '0')}`;
+    
+    const dayOfWeek = now.getUTCDay();      // 0 = Sunday
+    const dayOfMonth = now.getUTCDate();    // 1-31
+    
+    for (const entry of manifest) {
+        const schedule = entry.schedule;
+        
+        // Check if this computation is due now
+        if (isScheduleDue(schedule, currentTime, dayOfWeek, dayOfMonth)) {
+            due.push(entry);
+        }
+    }
+    
+    return due;
+}
+
+/**
+ * Check if a schedule is due at the given time.
+ * 
+ * @param {Object} schedule - Schedule object
+ * @param {string} currentTime - Current time in HH:MM format
+ * @param {number} dayOfWeek - Day of week (0-6, Sunday=0)
+ * @param {number} dayOfMonth - Day of month (1-31)
+ * @returns {boolean}
+ */
+function isScheduleDue(schedule, currentTime, dayOfWeek, dayOfMonth) {
+    const scheduleTime = schedule.time || '02:00';
+    const [scheduleHour, scheduleMinute] = scheduleTime.split(':').map(Number);
+    const [currentHour, currentMinuteNum] = currentTime.split(':').map(Number);
+    
+    // Check frequency-specific conditions
+    switch (schedule.frequency) {
+        case 'hourly':
+            // Hourly runs every hour at the specified minute
+            // e.g., time: '00:30' means run at XX:30
+            // Only check the minute portion matches
+            return scheduleMinute === currentMinuteNum;
+            
+        case 'daily':
+            // Daily runs at exact time (hour:minute must match)
+            return scheduleTime === currentTime;
+            
+        case 'weekly':
+            // Weekly runs at exact time on specified day
+            if (scheduleTime !== currentTime) return false;
+            const targetDay = schedule.dayOfWeek ?? 0;  // Default Sunday
+            return dayOfWeek === targetDay;
+            
+        case 'monthly':
+            // Monthly runs at exact time on specified day of month
+            if (scheduleTime !== currentTime) return false;
+            const targetDayOfMonth = schedule.dayOfMonth ?? 1;  // Default 1st
+            return dayOfMonth === targetDayOfMonth;
+            
+        default:
+            // Unknown frequency, default to daily behavior
+            return scheduleTime === currentTime;
+    }
+}
+
+/**
+ * Dispatch computations to Cloud Tasks queue.
+ * Uses p-limit for rate limiting.
+ * 
+ * @param {Array} computations - Array of manifest entries
+ * @param {string} targetDate - Target date (YYYY-MM-DD)
+ * @param {Date} scheduledTime - The floored time this scheduler run represents (for idempotent task names)
+ * @returns {Promise<Array>} Results for each dispatch
+ */
+async function dispatchComputations(computations, targetDate, scheduledTime) {
+    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
+    
+    const { projectId, location, queueName, dispatcherUrl } = config.cloudTasks;
+    const queuePath = tasksClient.queuePath(projectId, location, queueName);
+    
+    // Use the floored scheduledTime for idempotent task naming
+    // This ensures retries or slow loops don't create duplicate tasks
+    const timeSlot = formatTimeCompact(scheduledTime);
+    
+    const tasks = computations.map(entry => limit(async () => {
+        try {
+            const taskPayload = {
+                computationName: entry.originalName,
+                targetDate,
+                source: 'scheduled',
+                scheduledAt: scheduledTime.toISOString()
+            };
+            
+            const task = {
+                httpRequest: {
+                    httpMethod: 'POST',
+                    url: dispatcherUrl,
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: Buffer.from(JSON.stringify(taskPayload)).toString('base64'),
+                    // OIDC token for authenticated Cloud Function invocation
+                    // The Dispatcher should be deployed with "Require authentication"
+                    oidcToken: {
+                        serviceAccountEmail: config.cloudTasks.serviceAccountEmail,
+                        audience: dispatcherUrl
+                    }
+                },
+                // Task name uses the floored time slot - idempotent across retries
+                // If scheduler runs twice for the same minute, Cloud Tasks deduplicates
+                name: `${queuePath}/tasks/${entry.name}-${targetDate}-${timeSlot}`
+            };
+            
+            await tasksClient.createTask({ parent: queuePath, task });
+            
+            return {
+                computation: entry.originalName,
+                status: 'dispatched',
+                targetDate
+            };
+            
+        } catch (error) {
+            // Handle "already exists" gracefully (duplicate prevention)
+            if (error.code === 6) {  // ALREADY_EXISTS
+                return {
+                    computation: entry.originalName,
+                    status: 'skipped',
+                    reason: 'Task already exists (duplicate prevention)'
+                };
+            }
+            
+            console.error(`[Scheduler] Failed to dispatch ${entry.originalName}:`, error.message);
+            return {
+                computation: entry.originalName,
+                status: 'error',
+                error: error.message
+            };
+        }
+    }));
+    
+    return Promise.all(tasks);
+}
+
+/**
+ * Floor a date to the nearest minute boundary.
+ * 14:00:58 → 14:00:00
+ */
+function floorToMinute(date) {
+    const floored = new Date(date);
+    floored.setUTCSeconds(0);
+    floored.setUTCMilliseconds(0);
+    return floored;
+}
+
+/**
+ * Format date as YYYY-MM-DD.
+ */
+function formatDate(date) {
+    return date.toISOString().split('T')[0];
+}
+
+/**
+ * Format time as HH:MM.
+ */
+function formatTime(date) {
+    const hours = String(date.getUTCHours()).padStart(2, '0');
+    const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+    return `${hours}:${minutes}`;
+}
+
+/**
+ * Format time as HHMM (compact, for task names).
+ */
+function formatTimeCompact(date) {
+    const hours = String(date.getUTCHours()).padStart(2, '0');
+    const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+    return `${hours}${minutes}`;
+}
+
+// Export for Cloud Functions
+module.exports = {
+    schedulerHandler,
+    initialize,
+    
+    // For testing
+    _findDueComputations: findDueComputations,
+    _isScheduleDue: isScheduleDue,
+    _floorToMinute: floorToMinute
+};

package/functions/computation-system-v2/index.js

@@ -0,0 +1,163 @@
+/**
+ * @fileoverview Computation System v2 Entry Point
+ * * Usage:
+ * const { execute, analyze, runComputation } = require('./computation-system-v2');
+ * * // Analyze what can run
+ * const report = await analyze({ date: '2026-01-24' });
+ * * // Execute computations (Monolith/Scheduler mode)
+ * const result = await execute({ date: '2026-01-24' });
+ * * // Run single computation (Worker mode)
+ * const task = await runComputation({ 
+ * date: '2026-01-24', 
+ * computation: 'UserPortfolioSummary' 
+ * });
+ */
+
+const { Orchestrator } = require('./framework');
+const config = require('./config/bulltrackers.config');
+
+// Add computations to config
+config.computations = [
+    require('./computations/UserPortfolioSummary'),
+    require('./computations/PopularInvestorProfileMetrics'),
+    require('./computations/PopularInvestorRiskAssessment'),
+    // Add more computations here as they're migrated
+];
+
+// Singleton orchestrator instance
+let orchestrator = null;
+
+/**
+ * Get or create the orchestrator instance.
+ * @param {Object} [customConfig] - Override default config
+ * @param {Object} [logger] - Custom logger
+ * @returns {Promise<Orchestrator>}
+ */
+async function getOrchestrator(customConfig = null, logger = null) {
+    if (!orchestrator || customConfig) {
+        const cfg = customConfig || config;
+        orchestrator = new Orchestrator(cfg, logger);
+        // Note: Orchestrator initializes lazily on first execute/analyze call, 
+        // but we can force it here if needed.
+    }
+    return orchestrator;
+}
+
+/**
+ * Analyze what can run for a given date.
+ * @param {Object} options
+ * @param {string} options.date - Target date (YYYY-MM-DD)
+ * @param {Object} [options.config] - Override config
+ * @param {Object} [options.logger] - Custom logger
+ * @returns {Promise<Object>} Analysis report
+ */
+async function analyze(options) {
+    const { date, config: customConfig = null, logger = null } = options;
+    const orch = await getOrchestrator(customConfig, logger);
+    return orch.analyze({ date });
+}
+
+/**
+ * Execute computations for a given date.
+ * (Used by Scheduler or Monolithic runs)
+ */
+async function execute(options) {
+    const { 
+        date, 
+        pass = null, 
+        computation = null, 
+        dryRun = false,
+        entities = null,
+        config: customConfig = null, 
+        logger = null 
+    } = options;
+    
+    const orch = await getOrchestrator(customConfig, logger);
+    return orch.execute({ date, pass, computation, dryRun, entities });
+}
+
+/**
+ * WORKER ENTRY POINT: Run a single computation.
+ * (Used by Cloud Functions / Dispatcher)
+ */
+async function runComputation(options) {
+    const {
+        date,
+        computation,
+        entityIds = null,
+        dryRun = false,
+        config: customConfig = null,
+        logger = null
+    } = options;
+
+    const orch = await getOrchestrator(customConfig, logger);
+
+    // 1. Ensure manifest is built
+    if (!orch.manifest) await orch.initialize();
+
+    // 2. Find the specific entry
+    // We normalize to handle case-insensitivity
+    const normalizedName = computation.toLowerCase().replace(/[^a-z0-9]/g, '');
+    const entry = orch.manifest.find(c => c.name === normalizedName);
+
+    if (!entry) {
+        throw new Error(`Computation not found: ${computation}`);
+    }
+
+    // 3. Delegate to Orchestrator's runSingle
+    // This handles dependencies, data fetching, middleware, etc.
+    return orch.runSingle(entry, date, { 
+        entityIds, 
+        dryRun 
+    });
+}
+
+/**
+ * Get the manifest (list of all computations with metadata).
+ */
+async function getManifest(options = {}) {
+    const { config: customConfig = null, logger = null } = options;
+    const orch = await getOrchestrator(customConfig, logger);
+    if (!orch.manifest) await orch.initialize();
+    return orch.manifest;
+}
+
+/**
+ * Warm the schema cache for all tables.
+ */
+async function warmCache(options = {}) {
+    const { config: customConfig = null, logger = null } = options;
+    const orch = await getOrchestrator(customConfig, logger);
+    const allTables = Object.keys(orch.config.tables);
+    return orch.schemaRegistry.warmCache(allTables);
+}
+
+/**
+ * Reset the singleton (useful for testing).
+ */
+function reset() {
+    orchestrator = null;
+}
+
+// Import handlers for Cloud Functions
+const handlers = require('./handlers');
+
+module.exports = {
+    // Main API
+    analyze,
+    execute,
+    runComputation, // <--- New Method
+    getManifest,
+    warmCache,
+    reset,
+    
+    // For advanced usage
+    getOrchestrator,
+    config,
+    
+    // Cloud Function handlers
+    ...handlers,
+    
+    // Re-export framework components
+    ...require('./framework')
+};

package/functions/computation-system-v2/rules/index.js

@@ -0,0 +1,49 @@
+/**
+ * @fileoverview Business Rules Index
+ * 
+ * Central registry of all business rules.
+ * These are automatically injected into computations.
+ * 
+ * Available rule modules:
+ * - portfolio: Portfolio snapshots (positions, value, exposure)
+ * - metrics: Financial metrics (Sharpe, Sortino, VaR, etc.)
+ * - rankings: PI rankings data (AUM, copiers, risk scores)
+ * - trades: Trade history (closed positions, stats)
+ * - social: Social data (posts, ratings, page views)
+ * - instruments: Instrument data (prices, sentiment, tickers)
+ * 
+ * Rules are available in computations as:
+ *   rules.portfolio.extractPositions(...)
+ *   rules.metrics.calculateSharpeRatio(...)
+ *   rules.rankings.getRiskScore(...)
+ *   rules.trades.calculateTradeStats(...)
+ *   rules.social.calculateEngagement(...)
+ *   rules.instruments.getPrice(...)
+ */
+
+const portfolio = require('./portfolio');
+const metrics = require('./metrics');
+const rankings = require('./rankings');
+const trades = require('./trades');
+const social = require('./social');
+const instruments = require('./instruments');
+
+module.exports = {
+    // Portfolio snapshots - positions, value, exposure, direction
+    portfolio,
+    
+    // Financial metrics - Sharpe, Sortino, drawdown, VaR
+    metrics,
+    
+    // PI rankings - AUM, copiers, risk scores, trading style
+    rankings,
+    
+    // Trade history - closed trades, win rate, P&L stats
+    trades,
+    
+    // Social data - posts, ratings, page views, engagement
+    social,
+    
+    // Instruments - prices, sentiment, tickers
+    instruments
+};