bulltrackers-module 1.0.105 → 1.0.107

package/functions/computation-system/helpers/worker_helpers.js

@@ -1,340 +0,0 @@
-/**
- * @fileoverview Main pipe: pipe.computationSystem.handleNodeTask
- *
- * This function is the "Worker" for the event-driven DAG.
- * It is triggered by a Pub/Sub message for a *single node* (computation).
- *
- * Its responsibilities are to:
- * 1. Receive a task: { date, nodeName }.
- * 2. Execute that single computation.
- * 3. On success:
- * a. Mark the node as 'success' in the StateManager.
- * b. Ask the StateManager for children that are now 'ready'.
- * c. Publish new tasks for those ready children.
- * 4. On failure:
- * a. Log the error.
- * b. Acknowledge the message (do not re-throw).
- * c. The system will "heal" when the missing dependency eventually runs
- * and re-triggers this failed node.
- */
-
-const { FieldPath } = require('@google-cloud/firestore');
-// Import data loaders
-const {
-    getPortfolioPartRefs,
-    loadFullDayMap,
-    loadDataByRefs,
-    loadDailyInsights,
-    loadDailySocialPostInsights
-} = require('../utils/data_loader.js');
-// Import utils
-const { commitBatchInChunks, categorizeCalculations } = require('../utils/utils.js');
-const stateManager = require('../utils/computation_state_manager.js');
-
-// --- Internal Helper: findCalcClass (no changes) ---
-function findCalcClass(calculations, category, name) {
-    if (calculations[category]) {
-        if (typeof calculations[category][name] === 'function') {
-            return calculations[category][name];
-        }
-        if (calculations[category].historical && typeof calculations[category].historical[name] === 'function') {
-            return calculations[category].historical[name];
-        }
-    }
-    return null;
-}
-
-// --- NEW: Internal helper for previous date string ---
-function getPrevStr(dateStr) {
-    const prev = new Date(dateStr + 'T00:00:00Z');
-    prev.setUTCDate(prev.getUTCDate() - 1);
-    return prev.toISOString().slice(0, 10);
-}
-
-/**
- * Streams data and runs a *single* Batch 1-style computation.
- * --- THIS IS THE FULLY UPDATED FUNCTION ---
- */
-async function runStreamingComputation(dateStr, node, config, dependencies, calculations) {
-    const { db, logger } = dependencies;
-    
-    // --- MODIFIED: Destructure 'requires' from the node ---
-    const { category, name: calcName, calcClass: CalculationClass, requires } = node;
-    const passName = `[Worker ${dateStr}/${calcName}]`;
-    
-    logger.log('INFO', `${passName} Starting streaming computation... (Requires: ${requires.join(', ') || 'none'})`);
-
-    // --- Dynamic Data Loading ---
-    let todayInsightsData = null;
-    let yesterdayInsightsData = null;
-    let todaySocialPostInsightsData = null;
-    let yesterdaySocialPostInsightsData = null;
-    let todayRefs = [];
-    let yesterdayPortfolios = {};
-    const prevStr = getPrevStr(dateStr); // Get yesterday's date string
-
-    if (requires.includes('today_insights')) {
-        todayInsightsData = await loadDailyInsights(config, dependencies, dateStr);
-    }
-    if (requires.includes('yesterday_insights')) {
-        yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
-    }
-    if (requires.includes('today_social')) {
-        todaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, dateStr);
-    }
-    if (requires.includes('yesterday_social')) {
-        yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
-    }
-    if (requires.includes('today_portfolio')) {
-        todayRefs = await getPortfolioPartRefs(config, dependencies, dateStr);
-    }
-    if (requires.includes('yesterday_portfolio')) {
-        const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
-        if (yesterdayRefs.length > 0) {
-            yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-        } else if (requires.includes('today_portfolio')) { 
-            // Only warn if today_portfolio was also expected, otherwise this might be normal
-            logger.log('WARN', `${passName} Requires 'yesterday_portfolio' but no data was found for ${prevStr}.`);
-        }
-    }
-    // --- End Dynamic Data Loading ---
-
-    // Check if any primary data source was loaded. If not, we can't run.
-    if (todayRefs.length === 0 && !todayInsightsData && !todaySocialPostInsightsData) {
-        logger.log('WARN', `${passName} No primary data sources (portfolio, insights, social) found for ${dateStr}. Skipping.`);
-        return null;
-    }
-
-    // --- Context Object (no change) ---
-    const { instrumentToTicker, instrumentToSector } = await dependencies.calculationUtils.loadInstrumentMappings();
-    const context = {
-        instrumentMappings: instrumentToTicker,
-        sectorMapping: instrumentToSector,
-        todayDateStr: dateStr,
-        yesterdayDateStr: prevStr,
-        dependencies: dependencies,
-        config: config
-    };
-
-    const calc = new CalculationClass();
-    const batchSize = config.partRefBatchSize || 10;
-    
-    // --- MODIFIED: Stream and Process ---
-    let processedOnce = false; // Flag for insights/social calcs
-    const allContextArgs = [context, todayInsightsData, yesterdayInsightsData, todaySocialPostInsightsData, yesterdaySocialPostInsightsData];
-
-    // Only stream portfolios if they are required and were found
-    if (requires.includes('today_portfolio') && todayRefs.length > 0) {
-        for (let i = 0; i < todayRefs.length; i += batchSize) {
-            const batchRefs = todayRefs.slice(i, i + batchSize);
-            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-            for (const uid in todayPortfoliosChunk) {
-                const p = todayPortfoliosChunk[uid];
-                if (!p) continue;
-                
-                // User-type filtering
-                const userType = p.PublicPositions ? 'speculator' : 'normal';
-                if ((userType === 'normal' && category === 'speculators') || (userType === 'speculator' && !['speculators', 'sanity'].includes(category))) {
-                    continue;
-                }
-                
-                let processArgs;
-                if (requires.includes('yesterday_portfolio')) {
-                    const pYesterday = yesterdayPortfolios[uid];
-                    if (!pYesterday) continue; // Skip if user is missing yesterday's data
-                    processArgs = [p, pYesterday, uid, ...allContextArgs];
-                } else {
-                    processArgs = [p, null, uid, ...allContextArgs];
-                }
-                
-                await Promise.resolve(calc.process(...processArgs));
-                processedOnce = true;
-            }
-        }
-    }
-    
-    // If this is an insights/social calc and no portfolios ran (or were needed), run it once.
-    if (!processedOnce && (requires.includes('today_insights') || requires.includes('today_social'))) {
-        logger.log('INFO', `${passName} Running once for insights/social data.`);
-        await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
-    }
-    // --- End Stream ---
-
-    // --- Get Result and Commit (no change) ---
-    const result = await Promise.resolve(calc.getResult());
-    
-    if (result === null) {
-        throw new Error(`Calculation ${calcName} returned null. Dependency likely missing.`);
-    }
-
-    const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-    const pendingWrites = [];
-    const summaryData = {};
-
-    if (result && Object.keys(result).length > 0) {
-        let isSharded = false;
-        const shardedCollections = {
-            'sharded_user_profile': config.shardedUserProfileCollection,
-            'sharded_user_profitability': config.shardedProfitabilityCollection
-        };
-
-        for (const resultKey in shardedCollections) {
-            if (result[resultKey]) {
-                isSharded = true;
-                const shardCollectionName = shardedCollections[resultKey];
-                const shardedData = result[resultKey];
-                for (const shardId in shardedData) {
-                    const shardDocData = shardedData[shardId];
-                    if (shardDocData && Object.keys(shardDocData).length > 0) {
-                        const shardRef = db.collection(shardCollectionName).doc(shardId);
-                        pendingWrites.push({ ref: shardRef, data: shardDocData });
-                    }
-                }
-                const { [resultKey]: _, ...otherResults } = result;
-                if (Object.keys(otherResults).length > 0) {
-                    const computationDocRef = resultsCollectionRef.doc(category)
-                        .collection(config.computationsSubcollection)
-                        .doc(calcName);
-                    pendingWrites.push({ ref: computationDocRef, data: otherResults });
-                }
-            }
-        }
-
-        if (!isSharded) {
-            const computationDocRef = resultsCollectionRef.doc(category)
-                                              .collection(config.computationsSubcollection)
-                                              .doc(calcName);
-            pendingWrites.push({ ref: computationDocRef, data: result });
-        }
-
-        summaryData[`${category}.${calcName}`] = true; // Use dot notation for summary
-        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-        
-        await commitBatchInChunks(config, dependencies, pendingWrites, `${passName} Commit`);
-    } else {
-        logger.log('WARN', `${passName} Calculation produced no results. Skipping write.`);
-    }
-}
-
-/**
- * Runs a *single* meta/backtest computation.
- * (This function remains unchanged as 'meta' calcs handle their own data loading)
- */
-async function runMetaComputation(dateStr, node, config, dependencies) {
-    const { db, logger } = dependencies;
-    const { category, name: calcName, calcClass: CalculationClass } = node;
-    const passName = `[Worker ${dateStr}/${calcName}]`;
-    
-    logger.log('INFO', `${passName} Starting meta computation...`);
-
-    const calc = new CalculationClass();
-    const result = await Promise.resolve(calc.process(dateStr, dependencies, config));
-
-    // Handle null result as a dependency failure
-    if (result === null) {
-        throw new Error(`Meta-calculation ${calcName} returned null. Dependency likely missing.`);
-    }
-
-    const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-    const pendingWrites = [];
-    const summaryData = {};
-
-    if (result && Object.keys(result).length > 0) {
-        const computationDocRef = resultsCollectionRef.doc(category)
-                                          .collection(config.computationsSubcollection)
-                                          .doc(calcName);
-        pendingWrites.push({ ref: computationDocRef, data: result });
-        
-        summaryData[`${category}.${calcName}`] = true; // Use dot notation for summary
-        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-
-        await commitBatchInChunks(config, dependencies, pendingWrites, `${passName} Commit`);
-    } else {
-        logger.log('WARN', `${passName} Meta-calculation produced no results. Skipping write.`);
-    }
-}
-
-
-// --- Main Exported Function ---
-
-/**
- * Main pipe: pipe.computationSystem.handleNodeTask
- * (This function remains unchanged)
- */
-async function handleNodeTask(message, context, config, dependencies, calculations) {
-    const { logger, pubsubUtils } = dependencies;
-    const graph = dependencies.computationGraph; // Get the graph from dependencies
-    
-    let task;
-    try {
-        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
-    } catch (e) {
-        logger.log('ERROR', '[Compute Worker] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return; // Acknowledge the message
-    }
-
-    const { date, nodeName } = task;
-    const taskId = `[Worker ${date}/${nodeName}]`;
-    
-    if (!graph || !graph.isBuilt) {
-        logger.log('ERROR', `${taskId} CRITICAL: ComputationGraph not available. Cannot proceed.`);
-        return; // Acknowledge
-    }
-    
-    const node = graph.getNode(nodeName);
-    if (!node) {
-        logger.log('ERROR', `${taskId} CRITICAL: Node name not found in graph. Task is invalid.`);
-        return; // Acknowledge
-    }
-
-    logger.log('INFO', `🚀 ${taskId} Received task. Category: ${node.category}`);
-    
-    try {
-        // 1. Route to the correct execution helper
-        // --- MODIFIED: Check for 'meta' requirement key ---
-        if (node.requires.includes('meta')) {
-            await runMetaComputation(date, node, config, dependencies);
-        } else {
-            // All other calcs (pnl, behavioural, etc.) use the streaming helper
-            await runStreamingComputation(date, node, config, dependencies, calculations);
-        }
-
-        // 2. On Success: Mark success and trigger children
-        const readyChildren = await stateManager.markNodeSuccessAndGetReadyChildren(dependencies, date, nodeName);
-
-        if (readyChildren.length > 0) {
-            const nextTasks = readyChildren.map(childName => ({
-                date: date,
-                nodeName: childName
-            }));
-            
-            const topicName = config.computeNodeTopicName;
-            await pubsubUtils.batchPublishTasks(dependencies, {
-                topicName: topicName,
-                tasks: nextTasks,
-                taskType: `Compute Node Trigger`
-            });
-            logger.log('SUCCESS', `${taskId} Complete. Triggering ${readyChildren.length} child node(s): ${readyChildren.join(', ')}`);
-        
-        } else {
-            logger.log('SUCCESS', `${taskId} Complete. No new children are ready.`);
-        }
-
-    } catch (error) {
-         logger.log('WARN', `${taskId} FAILED (Will retry when dependencies are met).`, { 
-             err: error.message, 
-             stack: error.stack 
-         });
-         return;
-    }
-}
-
-module.exports = {
-    handleNodeTask,
-    // Exporting these for testing
-    runStreamingComputation,
-    runMetaComputation
-};

package/functions/computation-system/utils/computation_state_manager.js

@@ -1,178 +0,0 @@
-/**
- * @fileoverview Manages the state of a computation DAG run in Firestore.
- *
- * This allows the system to be event-driven, resilient, and self-healing.
- * It tracks which nodes have completed and which are ready to run.
- */
-
-const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-
-const STATE_COLLECTION = 'computation_state'; // Stores state docs, e.g., /computation_state/2025-10-31
-
-/**
- * Initializes the state document for a new DAG run.
- * @param {object} dependencies - Contains db, logger.
- * @param {string} dateStr - The date (YYYY-MM-DD) to run for.
- * @param {Array<object>} allNodes - Array of all node objects from the graph.
- */
-async function initializeState(dependencies, dateStr, allNodes) {
-    const { db, logger } = dependencies;
-    const stateDocRef = db.collection(STATE_COLLECTION).doc(dateStr);
-    
-    logger.log('INFO', `[StateManager] Initializing state for ${dateStr}...`);
-
-    const initialState = {
-        status: 'processing',
-        lastUpdated: FieldValue.serverTimestamp(),
-        nodes: {}
-    };
-
-    allNodes.forEach(node => {
-        initialState.nodes[node.name] = {
-            status: 'pending',
-            category: node.category,
-            parentCount: node.parents.size,
-            parents: Array.from(node.parents),
-            children: Array.from(node.children)
-        };
-    });
-
-    await stateDocRef.set(initialState);
-    logger.log('INFO', `[StateManager] State for ${dateStr} initialized with ${allNodes.length} nodes.`);
-}
-
-/**
- * Marks a node as 'success' and returns all children that are now ready to run.
- * @param {object} dependencies - Contains db, logger.
- * @param {string} dateStr - The date (YYYY-MM-DD) of the run.
- * @param {string} nodeName - The name of the node that just completed.
- * @returns {Promise<string[]>} A list of child node names that are ready to run.
- */
-async function markNodeSuccessAndGetReadyChildren(dependencies, dateStr, nodeName) {
-    const { db, logger } = dependencies;
-    const stateDocRef = db.collection(STATE_COLLECTION).doc(dateStr);
-    
-    logger.log('INFO', `[StateManager] Marking node '${nodeName}' as success for ${dateStr}.`);
-
-    const readyChildren = [];
-    
-    try {
-        await db.runTransaction(async (transaction) => {
-            const stateDoc = await transaction.get(stateDocRef);
-            if (!stateDoc.exists) {
-                logger.log('ERROR', `[StateManager] State doc for ${dateStr} does not exist. Cannot mark success.`);
-                return;
-            }
-            
-            const state = stateDoc.data();
-            const nodes = state.nodes;
-            
-            // 1. Mark this node as success
-            const update = {};
-            update[`nodes.${nodeName}.status`] = 'success';
-            update['lastUpdated'] = FieldValue.serverTimestamp();
-            
-            // 2. Check all children of this node
-            const children = nodes[nodeName]?.children || [];
-            for (const childName of children) {
-                const childNode = nodes[childName];
-                if (!childNode) {
-                    logger.log('WARN', `[StateManager] Child node ${childName} not found in state doc.`);
-                    continue;
-                }
-
-                // If child is already done or failed, skip it
-                if (childNode.status !== 'pending') {
-                    continue;
-                }
-
-                // Check if all parents of this child are now 'success'
-                let allParentsSuccess = true;
-                for (const parentName of childNode.parents) {
-                    const parentStatus = (parentName === nodeName) ? 'success' : nodes[parentName]?.status;
-                    if (parentStatus !== 'success') {
-                        allParentsSuccess = false;
-                        break;
-                    }
-                }
-
-                if (allParentsSuccess) {
-                    logger.log('INFO', `[StateManager] All dependencies for '${childName}' are met. Marking as 'ready'.`);
-                    update[`nodes.${childName}.status`] = 'ready'; // Mark as 'ready' to avoid double-runs
-                    readyChildren.push(childName);
-                }
-            }
-            
-            transaction.update(stateDocRef, update);
-        });
-
-        return readyChildren;
-
-    } catch (error) {
-        logger.log('ERROR', `[StateManager] Transaction failed for ${nodeName} on ${dateStr}.`, { err: error.message });
-        return []; // Return empty on error
-    }
-}
-
-/**
- * Finds all dates that are incomplete (status 'processing') or missing.
- * @param {object} dependencies - Contains db, logger.
- * @param {string[]} allExpectedDates - All dates that *should* exist.
- * @returns {Promise<string[]>} A list of date strings (YYYY-MM-DD) to process.
- */
-async function getIncompleteDates(dependencies, allExpectedDates) {
-    const { db, logger } = dependencies;
-    if (allExpectedDates.length === 0) return [];
-    
-    logger.log('INFO', `[StateManager] Checking for incomplete dates...`);
-    
-    const datesToProcess = new Set();
-    const refs = allExpectedDates.map(dateStr => db.collection(STATE_COLLECTION).doc(dateStr));
-    const snapshots = await db.getAll(...refs);
-
-    for (let i = 0; i < snapshots.length; i++) {
-        const dateStr = allExpectedDates[i];
-        const doc = snapshots[i];
-
-        if (!doc.exists) {
-            // Date is completely missing
-            datesToProcess.add(dateStr);
-            continue;
-        }
-
-        const data = doc.data();
-        if (data.status === 'processing') {
-            // Date was started but not finished (e.g., a node failed)
-            datesToProcess.add(dateStr);
-            continue;
-        }
-        
-        // If status is 'complete', we're good.
-    }
-    
-    logger.log('INFO', `[StateManager] Found ${datesToProcess.size} dates to process.`);
-    return Array.from(datesToProcess);
-}
-
-/**
- * Marks an entire DAG run as 'complete'.
- * @param {object} dependencies - Contains db, logger.
- * @param {string} dateStr - The date (YYYY-MM-DD) to mark as complete.
- */
-async function markDagComplete(dependencies, dateStr) {
-    const { db, logger } = dependencies;
-    const stateDocRef = db.collection(STATE_COLLECTION).doc(dateStr);
-    
-    logger.log('SUCCESS', `[StateManager] Marking DAG for ${dateStr} as complete.`);
-    await stateDocRef.update({
-        status: 'complete',
-        lastUpdated: FieldValue.serverTimestamp()
-    });
-}
-
-module.exports = {
-    initializeState,
-    markNodeSuccessAndGetReadyChildren,
-    getIncompleteDates,
-    markDagComplete
-};