bulltrackers-module 1.0.92 → 1.0.94

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -17,22 +17,35 @@ const {
 
 // --- MODIFIED: Import new meta calculation objects ---
 const {
-    historicalCalculations, dailyCalculations, metaCalculations,
-    HISTORICAL_CALC_NAMES, META_CALC_NAMES,
-    withRetry,
+    categorizeCalculations, // <-- IMPORT THE NEW FUNCTION
+    // REMOVED: withRetry
     getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
-    commitBatchInChunks, unifiedUtils
+    commitBatchInChunks
+    // REMOVED: unifiedUtils
 } = require('../utils/utils.js');
 // --- END MODIFIED ---
 
 /**
  * Main pipe: pipe.computationSystem.runOrchestration
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {object} calculations - The injected calculations object from 'aiden-shared-calculations-unified'.
  * @returns {Promise<Object>} Summary of all passes.
  */
-async function runComputationOrchestrator(config, dependencies) {
-    const { logger, db } = dependencies; // Added db here
+async function runComputationOrchestrator(config, dependencies, calculations) { // <-- ADDED 'calculations'
+    const { logger, db, calculationUtils } = dependencies; // Added db and calculationUtils
+    const { withRetry } = calculationUtils; // <-- Get withRetry from injected utils
+
+    // --- NEW: Categorize the injected calculations ---
+    const {
+        historicalCalculations,
+        dailyCalculations,
+        metaCalculations,
+        HISTORICAL_CALC_NAMES,
+        META_CALC_NAMES
+    } = categorizeCalculations(calculations);
+    // --- END NEW ---
+
     // --- MODIFIED: Add pass3_results to summary ---
     const summary = { pass1_results: [], pass2_results: [], pass3_results: [] };
     // --- END MODIFIED ---
@@ -46,9 +59,12 @@ async function runComputationOrchestrator(config, dependencies) {
     const masterDailyList = Object.entries(dailyCalculations).flatMap(([cat, calcs]) =>
         Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
     );
-    const insightsCalculations = require('aiden-shared-calculations-unified').calculations.insights || {};
+    
+    // --- MODIFIED: Use the injected 'calculations' object ---
+    const insightsCalculations = calculations.insights || {};
     const masterInsightsList = Object.keys(insightsCalculations).map(name => ({ category: 'insights', calcName: name }));
-    const socialPostCalculations = require('aiden-shared-calculations-unified').calculations.socialPosts || {};
+    const socialPostCalculations = calculations.socialPosts || {};
+    // --- END MODIFIED ---
     const masterSocialPostList = Object.keys(socialPostCalculations).map(name => ({ category: 'socialPosts', calcName: name }));
     
     // --- NEW: Create master list for meta calcs ---
@@ -227,7 +243,7 @@ async function streamAndProcess(
     todaySocialPostInsights = null,
     yesterdaySocialPostInsights = null
 ) {
-    const { db, logger } = dependencies;
+    const { db, logger, calculationUtils } = dependencies; // <-- Get calculationUtils
     logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
 
     // Calculate yesterday's date string
@@ -235,7 +251,10 @@ async function streamAndProcess(
     yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
     const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
 
-    const { instrumentToTicker, instrumentToSector } = await unifiedUtils.loadInstrumentMappings();
+    // --- MODIFIED: Get mapping functions from injected utils ---
+    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
+    // --- END MODIFIED ---
+
     // Add date strings to the context
     const context = { 
         instrumentMappings: instrumentToTicker, 

package/functions/computation-system/utils/data_loader.js

@@ -2,18 +2,22 @@
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
  */
-const { withRetry } = require('aiden-shared-calculations-unified').utils;
-const { FieldPath } = require('@google-cloud/firestore'); // <<< --- ADD FieldPath
+
+// <<< FIX: REMOVED all top-level 'require' and 'dependencies' lines >>>
 
 /**
  * Sub-pipe: pipe.computationSystem.dataLoader.getPortfolioPartRefs
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {string} dateString - The date in YYYY-MM-DD format.
  * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
  */
 async function getPortfolioPartRefs(config, dependencies, dateString) {
-    const { db, logger } = dependencies;
+    // <<< FIX: Destructure all dependencies here, inside the function >>>
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // <<< END FIX >>>
+
     logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
@@ -47,12 +51,16 @@ async function getPortfolioPartRefs(config, dependencies, dateString) {
 /**
  * Sub-pipe: pipe.computationSystem.dataLoader.loadDataByRefs
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {Firestore.DocumentReference[]} refs - An array of DocumentReferences to load.
  * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
  */
 async function loadDataByRefs(config, dependencies, refs) {
-    const { db, logger } = dependencies;
+    // <<< FIX: Destructure all dependencies here, inside the function >>>
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // <<< END FIX >>>
+
     if (!refs || refs.length === 0) { return {}; }
     const mergedPortfolios = {};
     const batchSize = config.partRefBatchSize || 50;
@@ -86,7 +94,10 @@ async function loadDataByRefs(config, dependencies, refs) {
  * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
  */
 async function loadFullDayMap(config, dependencies, partRefs) {
+    // <<< FIX: Destructure only what's needed for this specific function >>>
     const { logger } = dependencies;
+    // <<< END FIX >>>
+
     if (partRefs.length === 0) return {};
     logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
 
@@ -100,12 +111,16 @@ async function loadFullDayMap(config, dependencies, partRefs) {
  * Sub-pipe: pipe.computationSystem.dataLoader.loadDailyInsights
  * Fetches the daily instrument insights document for a specific date.
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {string} dateString - The date in YYYY-MM-DD format.
  * @returns {Promise<object|null>} The insights data object or null if not found/error.
  */
 async function loadDailyInsights(config, dependencies, dateString) {
-    const { db, logger } = dependencies;
+    // <<< FIX: Destructure all dependencies here, inside the function >>>
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // <<< END FIX >>>
+
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights'; // Use config or default
     logger.log('INFO', `Loading daily insights for date: ${dateString} from ${insightsCollectionName}`);
     try {
@@ -129,12 +144,16 @@ async function loadDailyInsights(config, dependencies, dateString) {
  * Sub-pipe: pipe.computationSystem.dataLoader.loadDailySocialPostInsights
  * Fetches all analyzed social post documents for a specific date.
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {string} dateString - The date in YYYY-MM-DD format.
  * @returns {Promise<object|null>} An object map of { [postId]: postData } or null.
  */
 async function loadDailySocialPostInsights(config, dependencies, dateString) {
-    const { db, logger } = dependencies;
+    // <<< FIX: Destructure all dependencies here, inside the function >>>
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // <<< END FIX >>>
+
     // Use the new config property, or fall back to a default
     const socialInsightsCollectionName = config.socialInsightsCollectionName || 'daily_social_insights';
     logger.log('INFO', `Loading social post insights for date: ${dateString} from ${socialInsightsCollectionName}`);
@@ -170,4 +189,4 @@ module.exports = {
     loadFullDayMap,
     loadDailyInsights,
     loadDailySocialPostInsights, 
-};
+};

package/functions/computation-system/utils/utils.js

@@ -1,62 +1,79 @@
 /**
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
- * DYNAMIC: Automatically categorizes meta and historical calcs based on file path.
+ * DYNAMIC: Categorization logic is now an exported function.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-const { calculations, utils } = require('aiden-shared-calculations-unified');
-const { withRetry } = utils;
-
-// --- Dynamic Calculation Categorization ---
-
-// These sets will be populated dynamically by inspecting the file structure
-const HISTORICAL_CALC_NAMES = new Set();
-const META_CALC_NAMES = new Set();
-
-const historicalCalculations = {};
-const dailyCalculations = {};
-const metaCalculations = {};
-
-for (const category in calculations) {
-    // 1. Check for 'meta' category first (by top-level directory name)
-    if (category === 'meta') {
-        if (!metaCalculations[category]) metaCalculations[category] = {};
-        for (const calcName in calculations[category]) {
-            const CalculationClass = calculations[category][calcName];
-            metaCalculations[category][calcName] = CalculationClass;
-            META_CALC_NAMES.add(calcName);
-        }
-        continue; // Done with this category
-    }
 
-    // 2. Process other categories (e.g., 'pnl', 'capital_flow')
-    for (const subKey in calculations[category]) {
-        const item = calculations[category][subKey];
-
-        // Check if the key is 'historical' and it contains an object of calculations
-        if (subKey === 'historical' && typeof item === 'object' && item !== null && !Array.isArray(item)) {
-            // This is the historical subdirectory, e.g., calculations.capital_flow.historical
-            
-            if (!historicalCalculations[category]) historicalCalculations[category] = {};
-            
-            for (const calcName in item) {
-                const CalculationClass = item[calcName];
-                // Add it to the historical list, using the parent 'category'
-                historicalCalculations[category][calcName] = CalculationClass;
-                HISTORICAL_CALC_NAMES.add(calcName);
+// --- REMOVED ---
+// const { utils } = require('aiden-shared-calculations-unified');
+// const { withRetry } = utils;
+// --- END REMOVED ---
+
+
+/**
+ * NEW FUNCTION: This logic is now wrapped and exported.
+ * It receives the 'calculations' object from the entry point.
+ * @param {object} calculations - The full calculations object from the 'aiden-shared-calculations-unified' package.
+ * @returns {object} An object containing categorized calculations and name sets.
+ */
+function categorizeCalculations(calculations) {
+    const HISTORICAL_CALC_NAMES = new Set();
+    const META_CALC_NAMES = new Set();
+
+    const historicalCalculations = {};
+    const dailyCalculations = {};
+    const metaCalculations = {};
+
+    for (const category in calculations) {
+        // 1. Check for 'meta' category first (by top-level directory name)
+        if (category === 'meta') {
+            if (!metaCalculations[category]) metaCalculations[category] = {};
+            for (const calcName in calculations[category]) {
+                const CalculationClass = calculations[category][calcName];
+                metaCalculations[category][calcName] = CalculationClass;
+                META_CALC_NAMES.add(calcName);
             }
+            continue; // Done with this category
         }
-        // Check if the item is a function (a standard daily calc at the root of the category)
-        else if (typeof item === 'function') {
-            // This is a standard daily calc, e.g., calculations.pnl.average-daily-pnl
-            const calcName = subKey;
-            const CalculationClass = item;
-            
-            if (!dailyCalculations[category]) dailyCalculations[category] = {};
-            dailyCalculations[category][calcName] = CalculationClass;
+
+        // 2. Process other categories (e.g., 'pnl', 'capital_flow')
+        for (const subKey in calculations[category]) {
+            const item = calculations[category][subKey];
+
+            // Check if the key is 'historical' and it contains an object of calculations
+            if (subKey === 'historical' && typeof item === 'object' && item !== null && !Array.isArray(item)) {
+                // This is the historical subdirectory, e.g., calculations.capital_flow.historical
+                
+                if (!historicalCalculations[category]) historicalCalculations[category] = {};
+                
+                for (const calcName in item) {
+                    const CalculationClass = item[calcName];
+                    // Add it to the historical list, using the parent 'category'
+                    historicalCalculations[category][calcName] = CalculationClass;
+                    HISTORICAL_CALC_NAMES.add(calcName);
+                }
+            }
+            // Check if the item is a function (a standard daily calc at the root of the category)
+            else if (typeof item === 'function') {
+                // This is a standard daily calc, e.g., calculations.pnl.average-daily-pnl
+                const calcName = subKey;
+                const CalculationClass = item;
+                
+                if (!dailyCalculations[category]) dailyCalculations[category] = {};
+                dailyCalculations[category][calcName] = CalculationClass;
+            }
         }
     }
+    
+    return {
+        historicalCalculations,
+        dailyCalculations,
+        metaCalculations,
+        HISTORICAL_CALC_NAMES,
+        META_CALC_NAMES
+    };
 }
 // --- End Dynamic Categorization ---
 
@@ -64,12 +81,16 @@ for (const category in calculations) {
 /**
  * Sub-pipe: pipe.computationSystem.computationUtils.commitBatchInChunks
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {Array<object>} writes - Array of { ref: DocumentReference, data: object }.
  * @param {string} operationName - Name for logging.
  */
 async function commitBatchInChunks(config, dependencies, writes, operationName) {
-    const { db, logger } = dependencies;
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
+    
     const batchSizeLimit = config.batchSizeLimit || 450;
     
     if (writes.length === 0) {
@@ -136,7 +157,10 @@ async function processJobsInParallel(jobs, taskFunction, passName, config) {
  * Internal helper: Finds the earliest date document in a collection.
  */
 async function getFirstDateFromCollection(config, dependencies, collectionName) {
-    const { db, logger } = dependencies;
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
     let earliestDate = null;
     try {
         const blockDocRefs = await withRetry(
@@ -177,7 +201,7 @@ async function getFirstDateFromCollection(config, dependencies, collectionName)
 /**
  * Sub-pipe: pipe.computationSystem.computationUtils.getFirstDateFromSourceData
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @returns {Promise<Date>} The earliest date found or a default fallback date.
  */
 async function getFirstDateFromSourceData(config, dependencies) {
@@ -207,9 +231,9 @@ async function getFirstDateFromSourceData(config, dependencies) {
 
 module.exports = {
     FieldValue, FieldPath,
-    historicalCalculations, dailyCalculations, metaCalculations, 
-    unifiedUtils: utils, 
-    HISTORICAL_CALC_NAMES, META_CALC_NAMES,
-    withRetry, commitBatchInChunks,
+    // unifiedUtils: utils, // This is no longer defined here
+    categorizeCalculations, // EXPORT THE NEW FUNCTION
+    // withRetry, // This is no longer defined here
+    commitBatchInChunks,
     getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
 };

package/functions/price-backfill/helpers/handler_helpers.js

@@ -5,8 +5,9 @@
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-// Import the mapping loader from your shared calculations package
-const { loadInstrumentMappings } = require('aiden-shared-calculations-unified').utils;
+// --- REMOVED ---
+// const { loadInstrumentMappings } = require('aiden-shared-calculations-unified').utils;
+// --- END REMOVED ---
 const pLimit = require('p-limit');
 
 // How many tickers to fetch in parallel
@@ -19,16 +20,21 @@ const SHARD_SIZE = 40;
 /**
  * Main pipe: pipe.maintenance.runBackfillAssetPrices
  * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, calculationUtils.
  */
+// --- MODIFIED: Removed calculationUtils, as it's in dependencies ---
 exports.runBackfillAssetPrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
+    const { db, logger, headerManager, proxyManager, calculationUtils } = dependencies;
+    const { loadInstrumentMappings } = calculationUtils; // <-- Get function from dependencies
+    // --- END MODIFIED ---
     
     logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
     
     let mappings;
     try {
+        // --- MODIFIED: Use the injected utils ---
         mappings = await loadInstrumentMappings();
+        // --- END MODIFIED ---
         if (!mappings || !mappings.instrumentToTicker) {
             throw new Error("Failed to load instrument mappings.");
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.92",
+  "version": "1.0.94",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -32,7 +32,6 @@
     "@google-cloud/firestore": "^7.11.3",
     "sharedsetup": "latest",
     "require-all": "^3.0.0",
-    "aiden-shared-calculations-unified": "1.0.16",
     "@google-cloud/pubsub": "latest",
     "express": "^4.19.2",
     "cors": "^2.8.5",