bulltrackers-module 1.0.733 → 1.0.734

package/functions/computation-system-v2/framework/utils/Graph.js

@@ -0,0 +1,205 @@
+/**
+ * @fileoverview Graph Algorithms Utility
+ * Provides generic implementations for graph operations:
+ * - Topological Sort (Kahn's Algorithm) with level/pass calculation
+ * - Cycle Detection (Tarjan's Algorithm)
+ */
+
+class Graph {
+    /**
+     * Performs a topological sort on a directed acyclic graph (DAG).
+     * Computes 'pass' (level) numbers for parallel execution layers.
+     * @param {string[]} nodes - List of node identifiers
+     * @param {Map<string, string[]>} adjacency - Map of node -> [dependencies]
+     * @returns {Array<{id: string, pass: number}>} Sorted items with pass numbers
+     * @throws {Error} If a cycle is detected or dependencies are missing
+     */
+    static topologicalSort(nodes, adjacency) {
+        // 1. Validate: Ensure all dependencies actually exist in the nodes list.
+        // This prevents "ghost dependencies" from breaking the sort logic silently.
+        const nodeSet = new Set(nodes);
+        for (const [node, deps] of adjacency) {
+            for (const dep of deps) {
+                if (!nodeSet.has(dep)) {
+                    throw new Error(`Integrity Error: Node '${node}' depends on '${dep}', but '${dep}' is not in the node list.`);
+                }
+            }
+        }
+
+        const inDegree = new Map();
+        const reverseAdj = new Map(); // dependent -> [dependents that depend on it]
+
+        // Initialize
+        for (const node of nodes) {
+            inDegree.set(node, 0);
+            if (!reverseAdj.has(node)) reverseAdj.set(node, []);
+        }
+
+        // Build graph properties
+        for (const [node, deps] of adjacency) {
+            // Adjacency is Node -> [Dependencies]
+            // So if A depends on B, the edge is B -> A
+            inDegree.set(node, deps.length);
+
+            for (const dep of deps) {
+                if (!reverseAdj.has(dep)) reverseAdj.set(dep, []);
+                reverseAdj.get(dep).push(node);
+            }
+        }
+
+        const queue = [];
+        const sorted = [];
+        const passes = new Map(); // node -> pass number
+
+        // 2. Enqueue 0-degree nodes (Pass 1)
+        for (const [node, degree] of inDegree) {
+            if (degree === 0) {
+                queue.push(node);
+                passes.set(node, 1);
+            }
+        }
+
+        // Sort initial queue for deterministic behavior
+        queue.sort();
+
+        // 3. Process Queue
+        while (queue.length > 0) {
+            const current = queue.shift();
+            const currentPass = passes.get(current);
+
+            sorted.push({
+                id: current,
+                pass: currentPass
+            });
+
+            // Find nodes that depend on current
+            const dependents = reverseAdj.get(current) || [];
+
+            for (const dependent of dependents) {
+                const newDegree = inDegree.get(dependent) - 1;
+                inDegree.set(dependent, newDegree);
+
+                // Update pass: dependent must run after current
+                const existingPass = passes.get(dependent) || 0;
+                if (currentPass + 1 > existingPass) {
+                    passes.set(dependent, currentPass + 1);
+                }
+
+                if (newDegree === 0) {
+                    queue.push(dependent);
+                }
+            }
+
+            // Re-sort queue to ensure deterministic processing order within the same level
+            queue.sort();
+        }
+
+        // 4. Verify Sort Integrity
+        if (sorted.length !== nodes.length) {
+            // Auto-diagnose: Try to find the cycle to give a helpful error
+            const cycle = Graph.detectCycle(nodes, adjacency);
+            if (cycle) {
+                throw new Error(`Graph cycle detected during sort: ${cycle}`);
+            }
+            throw new Error('Graph sort failed (unknown integrity error, possibly disconnected islands or ghost deps)');
+        }
+
+        return sorted;
+    }
+
+    /**
+     * Detects cycles in a graph using an iterative version of Tarjan's SCC algorithm.
+     * Iterative implementation is used to avoid stack overflow on very deep dependency chains.
+     * @param {string[]} nodes - List of node identifiers
+     * @param {Map<string, string[]>} adjacency - Map of node -> [dependencies]
+     * @returns {string|null} Description of cycle if found, null otherwise
+     */
+    static detectCycle(nodes, adjacency) {
+        let index = 0;
+        const indices = new Map();
+        const lowLinks = new Map();
+        const onStack = new Set();
+        const sccStack = [];
+        const nodeSet = new Set(nodes);
+        
+        // Explicit stack for DFS state to avoid recursion limit
+        // State: { v: node, iter: iterator of neighbors, parent: node }
+        const workStack = [];
+
+        const getNeighbors = (node) => {
+            const deps = adjacency.get(node) || [];
+            return deps.filter(w => nodeSet.has(w))[Symbol.iterator]();
+        };
+
+        for (const node of nodes) {
+            if (indices.has(node)) continue;
+
+            // Initialize DFS for this component
+            workStack.push({ v: node, iter: getNeighbors(node), parent: null });
+
+            while (workStack.length > 0) {
+                const state = workStack[workStack.length - 1];
+                const { v, iter, parent } = state;
+
+                // First time visiting this node?
+                if (!indices.has(v)) {
+                    indices.set(v, index);
+                    lowLinks.set(v, index);
+                    index++;
+                    sccStack.push(v);
+                    onStack.add(v);
+                }
+
+                const next = iter.next();
+
+                if (!next.done) {
+                    const w = next.value;
+                    if (!indices.has(w)) {
+                        // Tree edge: Push child onto stack (simulate recursion)
+                        workStack.push({ v: w, iter: getNeighbors(w), parent: v });
+                    } else if (onStack.has(w)) {
+                        // Back edge: Update lowLink immediately
+                        lowLinks.set(v, Math.min(lowLinks.get(v), indices.get(w)));
+                    }
+                } else {
+                    // All neighbors processed: Pop from work stack (simulate return)
+                    workStack.pop();
+
+                    // Propagate lowLink to parent (if any)
+                    if (parent) {
+                        lowLinks.set(parent, Math.min(lowLinks.get(parent), lowLinks.get(v)));
+                    }
+
+                    // Check if v is a root of an SCC
+                    if (lowLinks.get(v) === indices.get(v)) {
+                        const scc = [];
+                        let w;
+                        do {
+                            w = sccStack.pop();
+                            onStack.delete(w);
+                            scc.push(w);
+                        } while (w !== v);
+
+                        // If SCC has > 1 node, it's a cycle
+                        if (scc.length > 1) {
+                            return scc.join(' -> ') + ' -> ' + scc[0];
+                        }
+                        
+                        // Check for self-loop (single node cycle)
+                        if (scc.length === 1) {
+                            const self = scc[0];
+                            const deps = adjacency.get(self) || [];
+                            if (deps.includes(self)) {
+                                return `${self} -> ${self}`;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return null;
+    }
+}
+
+module.exports = { Graph };

package/functions/computation-system-v2/handlers/dispatcher.js

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview Computation Dispatcher
+ * * RESPONSIBILITIES:
+ * 1. Receive HTTP request (from Cloud Tasks or User)
+ * 2. Delegate execution to the Orchestrator
+ * 3. Translate Orchestrator STATUS (blocked, completed) into HTTP CODES (503, 200)
+ * * Why is this file so short?
+ * All validation logic (Schedule, Dependencies, Data) has moved to 
+ * framework/core/RunAnalyzer.js to ensure consistency between 
+ * the Scheduler and the Worker.
+ */
+
+const system = require('../index');
+
+exports.dispatcherHandler = async (req, res) => {
+    const startTime = Date.now();
+    
+    try {
+        const {
+            computationName,
+            targetDate,
+            source = 'scheduled',    // 'scheduled' | 'on-demand'
+            entityIds,               // Optional: specific entities
+            dryRun = false,
+            force = false            // Optional: run even if hash matches
+        } = req.body || {};
+        
+        // 1. Basic Validation
+        if (!computationName) {
+            return res.status(400).json({
+                status: 'error',
+                reason: 'MISSING_COMPUTATION',
+                message: 'computationName is required'
+            });
+        }
+        
+        const date = targetDate || new Date().toISOString().split('T')[0];
+        console.log(`[Dispatcher] Received ${source} request: ${computationName} for ${date}`);
+        
+        // 2. DELEGATE TO ORCHESTRATOR
+        // The Orchestrator calls RunAnalyzer internally to check:
+        // - Is it scheduled?
+        // - Are dependencies ready?
+        // - Is data available?
+        // - Has the code changed?
+        const result = await system.runComputation({
+            date,
+            computation: computationName,
+            entityIds,
+            dryRun
+        });
+
+        const duration = Date.now() - startTime;
+
+        // 3. HANDLE SUCCESS (Completed or Skipped/Up-to-date)
+        if (result.status === 'completed' || result.status === 'skipped') {
+            console.log(`[Dispatcher] ${computationName} ${result.status}: ${result.resultCount || 0} entities in ${duration}ms`);
+            
+            return res.status(200).json({
+                status: result.status,
+                computation: result.name,
+                date,
+                entityCount: result.resultCount,
+                duration,
+                hash: result.hash,
+                reason: result.reason
+            });
+        } 
+        
+        // 4. HANDLE BLOCKS (The Critical Retry Logic)
+        // If Orchestrator says "blocked", we MUST return 503 for Cloud Tasks to retry.
+        if (result.status === 'blocked' || result.status === 'impossible') {
+            console.log(`[Dispatcher] ${computationName} blocked: ${result.reason}`);
+            
+            // Scheduled tasks need 503 to trigger retry
+            // On-demand users need 200 to see the error message immediately
+            const httpStatus = source === 'scheduled' ? 503 : 200;
+            
+            return res.status(httpStatus).json({
+                status: result.status,
+                reason: result.reason,
+                message: `Computation blocked: ${result.reason}`
+            });
+        }
+
+        // 5. Fallback for other statuses (e.g., 'not_scheduled')
+        return res.status(200).json(result);
+        
+    } catch (error) {
+        const duration = Date.now() - startTime;
+        console.error(`[Dispatcher] Error after ${duration}ms:`, error);
+        
+        // Handle "Not Found" specifically
+        if (error.message && error.message.includes('Computation not found')) {
+            return res.status(400).json({
+                status: 'error',
+                reason: 'UNKNOWN_COMPUTATION',
+                message: error.message
+            });
+        }
+
+        // Return 500 to trigger Cloud Tasks retry for crash/network errors
+        return res.status(500).json({
+            status: 'error',
+            reason: 'EXECUTION_FAILED',
+            message: error.message
+        });
+    }
+};

package/functions/computation-system-v2/handlers/index.js

@@ -0,0 +1,23 @@
+/**
+ * @fileoverview Cloud Function Handlers
+ * 
+ * Export handlers for deployment as Cloud Functions:
+ * - computeScheduler: Single scheduler triggered every minute
+ * - computeDispatcher: Receives tasks from Cloud Tasks queue
+ * - computeOnDemand: Receives requests from frontend
+ */
+
+const { schedulerHandler } = require('./scheduler');
+const { dispatcherHandler } = require('./dispatcher');
+const { onDemandHandler } = require('./onDemand');
+
+module.exports = {
+    // Unified scheduler - triggered every minute by Cloud Scheduler
+    computeScheduler: schedulerHandler,
+    
+    // Main dispatcher - handles scheduled tasks from Cloud Tasks
+    computeDispatcher: dispatcherHandler,
+    
+    // On-demand API - handles frontend requests
+    computeOnDemand: onDemandHandler
+};

package/functions/computation-system-v2/handlers/onDemand.js

@@ -0,0 +1,289 @@
+/**
+ * @fileoverview On-Demand Computation API
+ * 
+ * Frontend-facing endpoint for user-triggered computation requests.
+ * 
+ * Responsibilities:
+ * 1. Authenticate user (Firebase Auth / JWT)
+ * 2. Rate limit requests per user
+ * 3. Validate user can access requested entities
+ * 4. Forward to Dispatcher for validation and execution
+ * 
+ * This is a lightweight gateway - all computation logic is in the Dispatcher.
+ */
+
+const { dispatcherHandler, initialize } = require('./dispatcher');
+const config = require('../config/bulltrackers.config');
+
+// Simple in-memory rate limiter (for Cloud Functions)
+// In production, consider using Redis or Firestore for distributed rate limiting
+const rateLimitStore = new Map();
+
+/**
+ * Main on-demand handler.
+ * 
+ * @param {Object} req - HTTP request
+ * @param {Object} res - HTTP response
+ */
+async function onDemandHandler(req, res) {
+    // Set CORS headers
+    res.set('Access-Control-Allow-Origin', '*');
+    res.set('Access-Control-Allow-Methods', 'POST, OPTIONS');
+    res.set('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+    
+    // Handle preflight
+    if (req.method === 'OPTIONS') {
+        return res.status(204).send('');
+    }
+    
+    try {
+        // 1. Authenticate user
+        const user = await authenticateRequest(req);
+        if (!user) {
+            return res.status(401).json({
+                status: 'error',
+                reason: 'UNAUTHORIZED',
+                message: 'Authentication required'
+            });
+        }
+        
+        // 2. Rate limit
+        const rateLimitResult = checkRateLimit(user.id);
+        if (!rateLimitResult.allowed) {
+            return res.status(429).json({
+                status: 'error',
+                reason: 'RATE_LIMITED',
+                message: 'Too many requests. Please wait.',
+                retryAfter: rateLimitResult.retryAfter
+            });
+        }
+        
+        const { computation, date, entityIds, force } = req.body || {};
+        
+        // 3. Validate request
+        if (!computation) {
+            return res.status(400).json({
+                status: 'error',
+                reason: 'MISSING_COMPUTATION',
+                message: 'computation field is required'
+            });
+        }
+        
+        // 4. Validate user can access requested entities
+        if (entityIds && entityIds.length > 0) {
+            const accessCheck = validateEntityAccess(user, entityIds);
+            if (!accessCheck.allowed) {
+                return res.status(403).json({
+                    status: 'error',
+                    reason: 'ACCESS_DENIED',
+                    message: accessCheck.message
+                });
+            }
+        }
+        
+        // 5. Check if computation is allowed for on-demand
+        const allowedComputations = config.onDemand?.allowedComputations;
+        if (allowedComputations && !allowedComputations.includes(computation)) {
+            return res.status(403).json({
+                status: 'error',
+                reason: 'COMPUTATION_NOT_ALLOWED',
+                message: `${computation} is not available for on-demand requests`
+            });
+        }
+        
+        // 6. Forward to Dispatcher
+        console.log(`[OnDemand] User ${user.id} requesting ${computation} for entities: ${entityIds?.join(', ') || 'all'}`);
+        
+        // Create a mock request/response to call dispatcher
+        const dispatcherRequest = {
+            body: {
+                computationName: computation,
+                targetDate: date || getTodayDate(),
+                source: 'on-demand',
+                entityIds: entityIds,
+                force: force || false,
+                requestedBy: user.id
+            }
+        };
+        
+        // Capture dispatcher response
+        const dispatcherResponse = await callDispatcher(dispatcherRequest);
+        
+        // Forward response to client
+        return res.status(dispatcherResponse.status).json(dispatcherResponse.body);
+        
+    } catch (error) {
+        console.error('[OnDemand] Error:', error);
+        return res.status(500).json({
+            status: 'error',
+            reason: 'INTERNAL_ERROR',
+            message: 'An internal error occurred'
+        });
+    }
+}
+
+/**
+ * Authenticate the request.
+ * Returns user object or null if not authenticated.
+ */
+async function authenticateRequest(req) {
+    const authHeader = req.headers.authorization;
+    
+    if (!authHeader || !authHeader.startsWith('Bearer ')) {
+        return null;
+    }
+    
+    const token = authHeader.substring(7);
+    
+    try {
+        // Option 1: Firebase Admin SDK (if available)
+        if (global.firebaseAdmin) {
+            const decodedToken = await global.firebaseAdmin.auth().verifyIdToken(token);
+            return {
+                id: decodedToken.uid,
+                email: decodedToken.email,
+                type: decodedToken.user_type || 'unknown'
+            };
+        }
+        
+        // Option 2: Simple JWT decode (for development/testing)
+        // In production, always verify with Firebase Admin
+        const payload = decodeJwt(token);
+        if (payload && payload.sub) {
+            return {
+                id: payload.sub,
+                email: payload.email,
+                type: payload.user_type || 'unknown'
+            };
+        }
+        
+        return null;
+    } catch (error) {
+        console.warn('[OnDemand] Auth error:', error.message);
+        return null;
+    }
+}
+
+/**
+ * Simple JWT decode (without verification - for development only).
+ */
+function decodeJwt(token) {
+    try {
+        const parts = token.split('.');
+        if (parts.length !== 3) return null;
+        
+        const payload = Buffer.from(parts[1], 'base64').toString('utf8');
+        return JSON.parse(payload);
+    } catch (e) {
+        return null;
+    }
+}
+
+/**
+ * Check rate limit for a user.
+ */
+function checkRateLimit(userId) {
+    const maxRequests = config.onDemand?.maxRequestsPerMinute || 5;
+    const windowMs = 60 * 1000;  // 1 minute
+    
+    const now = Date.now();
+    const userKey = `ratelimit:${userId}`;
+    
+    // Get or create user's request history
+    let history = rateLimitStore.get(userKey);
+    if (!history) {
+        history = [];
+        rateLimitStore.set(userKey, history);
+    }
+    
+    // Remove old entries
+    const cutoff = now - windowMs;
+    while (history.length > 0 && history[0] < cutoff) {
+        history.shift();
+    }
+    
+    // Check limit
+    if (history.length >= maxRequests) {
+        const oldestRequest = history[0];
+        const retryAfter = Math.ceil((oldestRequest + windowMs - now) / 1000);
+        return { allowed: false, retryAfter };
+    }
+    
+    // Record this request
+    history.push(now);
+    
+    return { allowed: true };
+}
+
+/**
+ * Validate user can access the requested entities.
+ */
+function validateEntityAccess(user, entityIds) {
+    // Default policy: users can only access their own data
+    // This can be extended based on user roles
+    
+    for (const entityId of entityIds) {
+        // Allow if entityId matches user's ID
+        if (entityId === user.id) {
+            continue;
+        }
+        
+        // Allow if user is admin (could add role checking)
+        // if (user.role === 'admin') continue;
+        
+        // Deny access
+        return {
+            allowed: false,
+            message: `You cannot access data for entity: ${entityId}`
+        };
+    }
+    
+    return { allowed: true };
+}
+
+/**
+ * Call the dispatcher and capture response.
+ */
+async function callDispatcher(req) {
+    // Initialize dispatcher if needed
+    await initialize();
+    
+    // Create a mock response object to capture the dispatcher's response
+    let responseStatus = 200;
+    let responseBody = null;
+    
+    const mockRes = {
+        status: (code) => {
+            responseStatus = code;
+            return mockRes;
+        },
+        json: (body) => {
+            responseBody = body;
+            return mockRes;
+        }
+    };
+    
+    await dispatcherHandler(req, mockRes);
+    
+    return {
+        status: responseStatus,
+        body: responseBody
+    };
+}
+
+/**
+ * Get today's date in YYYY-MM-DD format.
+ */
+function getTodayDate() {
+    return new Date().toISOString().split('T')[0];
+}
+
+// Export for Cloud Functions
+module.exports = {
+    onDemandHandler,
+    
+    // For testing
+    _authenticateRequest: authenticateRequest,
+    _checkRateLimit: checkRateLimit,
+    _validateEntityAccess: validateEntityAccess
+};