bulltrackers-module 1.0.91 → 1.0.93

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -17,22 +17,35 @@ const {
 
 // --- MODIFIED: Import new meta calculation objects ---
 const {
-    historicalCalculations, dailyCalculations, metaCalculations,
-    HISTORICAL_CALC_NAMES, META_CALC_NAMES,
-    withRetry,
+    categorizeCalculations, // <-- IMPORT THE NEW FUNCTION
+    // REMOVED: withRetry
     getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
-    commitBatchInChunks, unifiedUtils
+    commitBatchInChunks
+    // REMOVED: unifiedUtils
 } = require('../utils/utils.js');
 // --- END MODIFIED ---
 
 /**
  * Main pipe: pipe.computationSystem.runOrchestration
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {object} calculations - The injected calculations object from 'aiden-shared-calculations-unified'.
  * @returns {Promise<Object>} Summary of all passes.
  */
-async function runComputationOrchestrator(config, dependencies) {
-    const { logger, db } = dependencies; // Added db here
+async function runComputationOrchestrator(config, dependencies, calculations) { // <-- ADDED 'calculations'
+    const { logger, db, calculationUtils } = dependencies; // Added db and calculationUtils
+    const { withRetry } = calculationUtils; // <-- Get withRetry from injected utils
+
+    // --- NEW: Categorize the injected calculations ---
+    const {
+        historicalCalculations,
+        dailyCalculations,
+        metaCalculations,
+        HISTORICAL_CALC_NAMES,
+        META_CALC_NAMES
+    } = categorizeCalculations(calculations);
+    // --- END NEW ---
+
     // --- MODIFIED: Add pass3_results to summary ---
     const summary = { pass1_results: [], pass2_results: [], pass3_results: [] };
     // --- END MODIFIED ---
@@ -46,9 +59,12 @@ async function runComputationOrchestrator(config, dependencies) {
     const masterDailyList = Object.entries(dailyCalculations).flatMap(([cat, calcs]) =>
         Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
     );
-    const insightsCalculations = require('aiden-shared-calculations-unified').calculations.insights || {};
+    
+    // --- MODIFIED: Use the injected 'calculations' object ---
+    const insightsCalculations = calculations.insights || {};
     const masterInsightsList = Object.keys(insightsCalculations).map(name => ({ category: 'insights', calcName: name }));
-    const socialPostCalculations = require('aiden-shared-calculations-unified').calculations.socialPosts || {};
+    const socialPostCalculations = calculations.socialPosts || {};
+    // --- END MODIFIED ---
     const masterSocialPostList = Object.keys(socialPostCalculations).map(name => ({ category: 'socialPosts', calcName: name }));
     
     // --- NEW: Create master list for meta calcs ---
@@ -227,7 +243,7 @@ async function streamAndProcess(
     todaySocialPostInsights = null,
     yesterdaySocialPostInsights = null
 ) {
-    const { db, logger } = dependencies;
+    const { db, logger, calculationUtils } = dependencies; // <-- Get calculationUtils
     logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
 
     // Calculate yesterday's date string
@@ -235,7 +251,10 @@ async function streamAndProcess(
     yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
     const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
 
-    const { instrumentToTicker, instrumentToSector } = await unifiedUtils.loadInstrumentMappings();
+    // --- MODIFIED: Get mapping functions from injected utils ---
+    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
+    // --- END MODIFIED ---
+
     // Add date strings to the context
     const context = { 
         instrumentMappings: instrumentToTicker, 

package/functions/computation-system/utils/utils.js

@@ -1,67 +1,95 @@
 /**
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
+ * DYNAMIC: Categorization logic is now an exported function.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-const { calculations, utils } = require('aiden-shared-calculations-unified');
-const { withRetry } = utils;
-
-// --- Calculation Categorization (Stateless) ---
-const HISTORICAL_CALC_NAMES = new Set([
-    'paper-vs-diamond-hands', 'smart-money-flow', 'profitability-migration',
-    'user-profitability-tracker', 'sector-rotation', 'crowd-conviction-score',
-    'risk-appetite-change', 'drawdown-response', 'gain-response',
-    'tsl-effectiveness', 'position-count-pnl', 'diversification-pnl',
-    
-    'deposit-withdrawal-percentage',
-    'new-allocation-percentage',
-    'reallocation-increase-percentage',
-    'asset-crowd-flow'
-]);
+// --- REMOVED ---
+// const { utils } = require('aiden-shared-calculations-unified');
+// const { withRetry } = utils;
+// --- END REMOVED ---
 
-// --- Set for meta calculation names (Generated Dynamically) ---
-const META_CALC_NAMES = new Set(); // This will be populated dynamically
 
+/**
+ * NEW FUNCTION: This logic is now wrapped and exported.
+ * It receives the 'calculations' object from the entry point.
+ * @param {object} calculations - The full calculations object from the 'aiden-shared-calculations-unified' package.
+ * @returns {object} An object containing categorized calculations and name sets.
+ */
+function categorizeCalculations(calculations) {
+    const HISTORICAL_CALC_NAMES = new Set();
+    const META_CALC_NAMES = new Set();
 
-const historicalCalculations = {};
-const dailyCalculations = {};
-const metaCalculations = {}; 
+    const historicalCalculations = {};
+    const dailyCalculations = {};
+    const metaCalculations = {};
 
-for (const category in calculations) {
-    for (const calcName in calculations[category]) {
-        
-        // --- UPDATED LOGIC ---
-        // 1. Check if it's a 'meta' calculation by its category
+    for (const category in calculations) {
+        // 1. Check for 'meta' category first (by top-level directory name)
         if (category === 'meta') {
             if (!metaCalculations[category]) metaCalculations[category] = {};
-            metaCalculations[category][calcName] = calculations[category][calcName];
-            META_CALC_NAMES.add(calcName); // Add to the set for export
-        
-        // 2. Check if it's a historical calculation
-        } else if (HISTORICAL_CALC_NAMES.has(calcName)) {
-            if (!historicalCalculations[category]) historicalCalculations[category] = {};
-            historicalCalculations[category][calcName] = calculations[category][calcName];
-        
-        // 3. Otherwise, it's a standard daily calculation
-        } else {
-            if (!dailyCalculations[category]) dailyCalculations[category] = {};
-            dailyCalculations[category][calcName] = calculations[category][calcName];
+            for (const calcName in calculations[category]) {
+                const CalculationClass = calculations[category][calcName];
+                metaCalculations[category][calcName] = CalculationClass;
+                META_CALC_NAMES.add(calcName);
+            }
+            continue; // Done with this category
+        }
+
+        // 2. Process other categories (e.g., 'pnl', 'capital_flow')
+        for (const subKey in calculations[category]) {
+            const item = calculations[category][subKey];
+
+            // Check if the key is 'historical' and it contains an object of calculations
+            if (subKey === 'historical' && typeof item === 'object' && item !== null && !Array.isArray(item)) {
+                // This is the historical subdirectory, e.g., calculations.capital_flow.historical
+                
+                if (!historicalCalculations[category]) historicalCalculations[category] = {};
+                
+                for (const calcName in item) {
+                    const CalculationClass = item[calcName];
+                    // Add it to the historical list, using the parent 'category'
+                    historicalCalculations[category][calcName] = CalculationClass;
+                    HISTORICAL_CALC_NAMES.add(calcName);
+                }
+            }
+            // Check if the item is a function (a standard daily calc at the root of the category)
+            else if (typeof item === 'function') {
+                // This is a standard daily calc, e.g., calculations.pnl.average-daily-pnl
+                const calcName = subKey;
+                const CalculationClass = item;
+                
+                if (!dailyCalculations[category]) dailyCalculations[category] = {};
+                dailyCalculations[category][calcName] = CalculationClass;
+            }
         }
     }
+    
+    return {
+        historicalCalculations,
+        dailyCalculations,
+        metaCalculations,
+        HISTORICAL_CALC_NAMES,
+        META_CALC_NAMES
+    };
 }
-// --- End Categorization ---
+// --- End Dynamic Categorization ---
 
 
 /**
  * Sub-pipe: pipe.computationSystem.computationUtils.commitBatchInChunks
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @param {Array<object>} writes - Array of { ref: DocumentReference, data: object }.
  * @param {string} operationName - Name for logging.
  */
 async function commitBatchInChunks(config, dependencies, writes, operationName) {
-    const { db, logger } = dependencies;
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
+    
     const batchSizeLimit = config.batchSizeLimit || 450;
     
     if (writes.length === 0) {
@@ -128,7 +156,10 @@ async function processJobsInParallel(jobs, taskFunction, passName, config) {
  * Internal helper: Finds the earliest date document in a collection.
  */
 async function getFirstDateFromCollection(config, dependencies, collectionName) {
-    const { db, logger } = dependencies;
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
     let earliestDate = null;
     try {
         const blockDocRefs = await withRetry(
@@ -169,7 +200,7 @@ async function getFirstDateFromCollection(config, dependencies, collectionName)
 /**
  * Sub-pipe: pipe.computationSystem.computationUtils.getFirstDateFromSourceData
  * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
  * @returns {Promise<Date>} The earliest date found or a default fallback date.
  */
 async function getFirstDateFromSourceData(config, dependencies) {
@@ -199,9 +230,9 @@ async function getFirstDateFromSourceData(config, dependencies) {
 
 module.exports = {
     FieldValue, FieldPath,
-    historicalCalculations, dailyCalculations, metaCalculations, 
-    unifiedUtils: utils, 
-    HISTORICAL_CALC_NAMES, META_CALC_NAMES,
-    withRetry, commitBatchInChunks,
+    // unifiedUtils: utils, // This is no longer defined here
+    categorizeCalculations, // EXPORT THE NEW FUNCTION
+    // withRetry, // This is no longer defined here
+    commitBatchInChunks,
     getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
 };

package/functions/price-backfill/helpers/handler_helpers.js

@@ -5,8 +5,9 @@
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-// Import the mapping loader from your shared calculations package
-const { loadInstrumentMappings } = require('aiden-shared-calculations-unified').utils;
+// --- REMOVED ---
+// const { loadInstrumentMappings } = require('aiden-shared-calculations-unified').utils;
+// --- END REMOVED ---
 const pLimit = require('p-limit');
 
 // How many tickers to fetch in parallel
@@ -19,16 +20,21 @@ const SHARD_SIZE = 40;
 /**
  * Main pipe: pipe.maintenance.runBackfillAssetPrices
  * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, calculationUtils.
  */
+// --- MODIFIED: Removed calculationUtils, as it's in dependencies ---
 exports.runBackfillAssetPrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
+    const { db, logger, headerManager, proxyManager, calculationUtils } = dependencies;
+    const { loadInstrumentMappings } = calculationUtils; // <-- Get function from dependencies
+    // --- END MODIFIED ---
     
     logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
     
     let mappings;
     try {
+        // --- MODIFIED: Use the injected utils ---
         mappings = await loadInstrumentMappings();
+        // --- END MODIFIED ---
         if (!mappings || !mappings.instrumentToTicker) {
             throw new Error("Failed to load instrument mappings.");
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.91",
+  "version": "1.0.93",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -32,7 +32,6 @@
     "@google-cloud/firestore": "^7.11.3",
     "sharedsetup": "latest",
     "require-all": "^3.0.0",
-    "aiden-shared-calculations-unified": "1.0.16",
     "@google-cloud/pubsub": "latest",
     "express": "^4.19.2",
     "cors": "^2.8.5",