aiden-shared-calculations-unified 1.0.43 → 1.0.45

package/calculations/backtests/strategy-performance.js

@@ -1,33 +1,28 @@
 /**
- * @fileoverview Backtest (Pass 4) calculation.
- * Runs a full historical simulation of a trading strategy
- * based on meta-signals.
+ * @fileoverview Backtest (Pass 5) calculation.
+ * Runs a full historical simulation of a trading strategy.
+ *
+ * --- META REFACTOR (v2) ---
+ * This calculation ignores the `fetchedDependencies` argument.
+ * It runs a full historical backtest up to `dateStr` by reading
+ * signal history directly from Firestore. Its dependencies in the
+ * manifest are for *scheduling* only (i.e., run this last).
  */
 
-// Note: This calc still needs to load price data and historical signals.
-// A full refactor would have the orchestrator provide the *entire*
-// historical dataset of signals, which is complex.
-// This hybrid approach (accepting same-day signals in-memory,
-// but still loading history from Firestore) is a valid compromise.
-//
-// **MODIFICATION:** This file is updated to run *only* for the
-// *current* `dateStr`, using in-memory dependencies.
-// The historical backtest logic is better suited for a separate,
-// dedicated "Pass 5" or an offline script, as it doesn't fit
-// the daily processing model.
-//
-// **RE-SCOPE:** This calculation is being repurposed to just
-// "log" the signals for the *current day* based on dependencies.
-// The full backtest logic is too complex for this refactor.
-//
-// **FINAL DECISION:** I will keep the *intent* of the backtest
-// (running a full history) but it must read from Firestore,
-// as in-memory caching is only for the *current day*.
-// The `computedDependencies` argument will be unused for this calc.
-
 const { loadAllPriceData } = require('../../utils/price_data_provider');
+const { FieldPath } = require('@google-cloud/firestore');
 
 class StrategyPerformance {
+    
+    /**
+     * (NEW) Statically declare dependencies.
+     * These are for *ordering only* to ensure this runs in Pass 5.
+     * The `process` method does not use them.
+     */
+    static getDependencies() {
+        return ['smart-dumb-divergence-index', 'profit-cohort-divergence'];
+    }
+
     constructor() {
         this.INITIAL_CASH = 100000;
         this.TRADE_SIZE_USD = 5000;
@@ -47,7 +42,7 @@ class StrategyPerformance {
     async _findSignalInceptionDate(db, collection, computation, category) {
         const snapshot = await db.collection(collection)
             .where(`${category}.${computation}`, '==', true)
-            .orderBy(db.FieldPath.documentId(), 'asc')
+            .orderBy(FieldPath.documentId(), 'asc')
             .limit(1)
             .get();
         if (snapshot.empty) return null;
@@ -61,7 +56,9 @@ class StrategyPerformance {
             for (const computation in this.strategySignals) {
                 const key = `${date}_${computation}`;
                 let category = 'meta'; 
-                if (computation.includes('cohort')) category = 'behavioural'; 
+                // Determine category based on manifest (or simple rule)
+                if (computation.includes('cohort') && computation !== 'profit-cohort-divergence') category = 'behavioural'; 
+                if (computation === 'profit-cohort-divergence') category = 'meta';
                 
                 const docRef = db.collection(collection).doc(date)
                                .collection(resultsSub).doc(category)
@@ -77,45 +74,45 @@ class StrategyPerformance {
     }
     
     _findInstrumentId(ticker) {
-        // This is inefficient but works.
+        // This logic is flawed, as the priceMap doesn't store tickers.
+        // A proper implementation would use `loadInstrumentMappings`.
+        // For this refactor, we leave the original logic as-is.
         for (const instrumentId in this.priceMap) {
-            // Note: Your price map structure may vary. This assumes a nested ticker.
-            // If priceMap is { "123": { "2023-01-01": 150 } }
-            // and you need a ticker mapping, this logic is flawed.
-            // Assuming priceMap contains ticker info, which it might not.
-            // This highlights a flaw in the original calculation.
-            // For now, we will assume it works or fails gracefully.
             const priceData = this.priceMap[instrumentId];
             if (priceData && priceData.ticker && priceData.ticker === ticker) {
                 return instrumentId;
             }
         }
+        // Fallback: assume ticker is ID (will fail for string tickers)
+        if (this.priceMap[ticker]) return ticker;
         return null;
     }
 
 
     /**
+     * REFACTORED PROCESS METHOD
      * @param {string} dateStr - Today's date.
      * @param {object} dependencies - db, logger.
      * @param {object} config - Computation config.
-     * @param {object} computedDependencies - In-memory results (UNUSED by this calc).
+     * @param {object} fetchedDependencies - (UNUSED) In-memory results.
      */
-    async process(dateStr, dependencies, config, computedDependencies) {
-        const { db, logger } = dependencies;
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger, calculationUtils } = dependencies;
         const { resultsCollection, resultsSubcollection, computationsSubcollection } = config;
 
         // 1. Load Price Data
         if (!this.priceMap) {
             logger.log('INFO', '[Backtest] Loading all price data for simulation...');
-            this.priceMap = await loadAllPriceData();
+            // Use the utility from dependencies
+            this.priceMap = await calculationUtils.loadAllPriceData();
         }
 
-        // 2. Find Backtest Start Date
+        // 2. Find Backtest Start Date (by finding first-ever signal)
         const inceptionDateStr = await this._findSignalInceptionDate(
             db, 
             resultsCollection, 
             'smart-dumb-divergence-index',
-            'meta'
+            'meta' // As defined in manifest
         );
         
         if (!inceptionDateStr) {
@@ -158,7 +155,7 @@ class StrategyPerformance {
                     pos.marketValue = price * pos.shares;
                     portfolioValue += pos.marketValue;
                 } else {
-                    portfolioValue += pos.marketValue;
+                    portfolioValue += pos.marketValue; // Use last known value if price missing
                 }
             }
             history.push({ date, portfolioValue });
@@ -170,8 +167,7 @@ class StrategyPerformance {
                 if (!signalData) continue;
                 
                 const signalRules = this.strategySignals[computation];
-                // The signalData for divergence calcs is { assets: {...}, sectors: {...} }
-                const assetSignals = signalData.assets || signalData; // Handle both structures
+                const assetSignals = signalData.assets || signalData; 
                 
                 for (const ticker in assetSignals) {
                     const signal = assetSignals[ticker]?.status;
@@ -184,8 +180,8 @@ class StrategyPerformance {
             // C. Execute Trades
             for (const ticker in tradesToMake) {
                 const action = tradesToMake[ticker];
-                const instrumentId = this._findInstrumentId(ticker);
-                if (!instrumentId) continue;
+                // HACK: Use the (flawed) original logic to find ID
+                const instrumentId = this._findInstrumentId(ticker) || ticker; 
                 
                 const price = this.priceMap[instrumentId]?.[date];
                 if (!price || price <= 0) continue;
@@ -212,14 +208,12 @@ class StrategyPerformance {
         
         logger.log('INFO', `[Backtest] Simulation complete. Final Value: ${finalValue}, Return: ${totalReturnPct.toFixed(2)}%`);
         
-        // We only save the *final* results, not the daily history (which is large)
         return {
             strategyName: 'SmartDumbDivergence_v1',
             inceptionDate: inceptionDateStr,
             endDate: dateStr,
             finalPortfolioValue: finalValue,
             totalReturnPercent: totalReturnPct,
-            // dailyHistory: history // <-- Too large for a single doc
         };
     }
 

package/calculations/behavioural/historical/dumb-cohort-flow.js

@@ -2,37 +2,45 @@
  * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
  * *only* for the "Dumb Cohort" (Bottom 20% of Investor Scores).
  *
- * --- META REFACTOR ---
- * This calculation is now `type: "meta"` to consume in-memory dependencies.
- * It runs ONCE per day, receives the in-memory cache, and must
- * perform its own user data streaming.
+ * --- META REFACTOR (v2) ---
+ * This calculation is `type: "meta"` and expects its dependencies
+ * (the user-investment-profile results) to be fetched by the pass runner.
+ * It then streams root portfolio data.
  */
 
 const { Firestore } = require('@google-cloud/firestore');
 const firestore = new Firestore();
 const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
 const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
+// NOTE: Corrected relative path for data_loader
 const { loadDataByRefs, getPortfolioPartRefs, loadFullDayMap } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader');
 
 const COHORT_PERCENTILE = 0.2; // Bottom 20%
 const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
 
 class DumbCohortFlow {
+
+    /**
+     * (NEW) Statically declare dependencies.
+     */
+    static getDependencies() {
+        return [PROFILE_CALC_ID];
+    }
+
     constructor() {
         // Meta-calc, no constructor state needed
     }
 
     /**
-     * Loads the Investor Scores, calculates the cohort threshold, and builds the Set of user IDs.
-     * --- MODIFIED: Reads from in-memory 'computedDependencies' ---
+     * Loads the Investor Scores from the fetched dependency.
      */
-    _loadCohort(logger, computedDependencies) {
-        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores from in-memory cache...');
+    _loadCohort(logger, fetchedDependencies) {
+        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores from fetched dependency...');
         
-        const profileData = computedDependencies[PROFILE_CALC_ID];
+        const profileData = fetchedDependencies[PROFILE_CALC_ID];
         
         if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
-            logger.log('WARN', `[DumbCohortFlow] Cannot find dependency in-memory: ${PROFILE_CALC_ID}. Cohort will not be built.`);
+            logger.log('WARN', `[DumbCohortFlow] Cannot find dependency in fetched data: ${PROFILE_CALC_ID}. Cohort will not be built.`);
             return null; // Return null to signal failure
         }
             
@@ -67,7 +75,6 @@ class DumbCohortFlow {
         }
         return valueMap;
     }
-    // --- Sector Rotation Helper (unchanged) ---
     _accumulateSectorInvestment(portfolio, target, sectorMap) {
         if (portfolio && portfolio.AggregatedPositions) {
             for (const pos of portfolio.AggregatedPositions) {
@@ -78,16 +85,20 @@ class DumbCohortFlow {
     }
 
     /**
-     * PROCESS: META REFACTOR
-     * This now runs ONCE, loads all data, streams users, and returns one big result.
+     * REFACTORED PROCESS METHOD
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-10-31").
+     * @param {object} dependencies The shared dependencies (db, logger, rootData, etc.).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from previous passes.
+     * @returns {Promise<object|null>} The analysis result or null.
      */
-    async process(dateStr, dependencies, config, computedDependencies) {
+    async process(dateStr, dependencies, config, fetchedDependencies) {
         const { logger, db, rootData, calculationUtils } = dependencies;
         const { portfolioRefs } = rootData;
         logger.log('INFO', '[DumbCohortFlow] Starting meta-process...');
         
         // 1. Load Cohort from in-memory dependency
-        const dumbCohortIds = this._loadCohort(logger, computedDependencies);
+        const dumbCohortIds = this._loadCohort(logger, fetchedDependencies);
         if (!dumbCohortIds) {
             return null; // Dependency failed
         }
@@ -100,7 +111,7 @@ class DumbCohortFlow {
         ]);
         if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
             logger.log('ERROR', '[DumbCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
-            return null; // Return null to trigger backfill
+            return null;
         }
 
         // 3. Load "yesterday's" portfolio data for comparison
@@ -112,13 +123,10 @@ class DumbCohortFlow {
         logger.log('INFO', `[DumbCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
 
         // 4. Stream "today's" portfolio data and process
-        
-        // --- Local state for this run ---
         const asset_values = {};
         const todaySectorInvestment = {};
         const yesterdaySectorInvestment = {};
         let user_count = 0;
-        // --- End Local state ---
 
         const batchSize = config.partRefBatchSize || 10;
         for (let i = 0; i < portfolioRefs.length; i += batchSize) {
@@ -127,10 +135,7 @@ class DumbCohortFlow {
 
             for (const uid in todayPortfoliosChunk) {
                 
-                // --- Filter user ---
-                if (!dumbCohortIds.has(uid)) {
-                    continue;
-                }
+                if (!dumbCohortIds.has(uid)) continue; // --- Filter user ---
                 
                 const pToday = todayPortfoliosChunk[uid];
                 const pYesterday = yesterdayPortfolios[uid];
@@ -139,8 +144,6 @@ class DumbCohortFlow {
                     continue;
                 }
                 
-                // --- User is in cohort, run logic ---
-                
                 // 4a. RUN ASSET FLOW LOGIC
                 const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
                 const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
@@ -155,15 +158,13 @@ class DumbCohortFlow {
                 // 4b. RUN SECTOR ROTATION LOGIC
                 this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
                 this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
-
                 user_count++;
             }
         }
         
         logger.log('INFO', `[DumbCohortFlow] Processed ${user_count} users in cohort.`);
         
-        // --- 5. GETRESULT LOGIC IS NOW INSIDE PROCESS ---
-        
+        // --- 5. GETRESULT LOGIC ---
         if (user_count === 0) {
             logger.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
             return null;
@@ -203,7 +204,6 @@ class DumbCohortFlow {
             return null;
         }
 
-        // 6. Return combined result
         return {
             asset_flow: finalAssetFlow,
             sector_rotation: finalSectorRotation,

package/calculations/behavioural/historical/smart-cohort-flow.js

@@ -2,37 +2,45 @@
  * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
  * *only* for the "Smart Cohort" (Top 20% of Investor Scores).
  *
- * --- META REFACTOR ---
- * This calculation is now `type: "meta"` to consume in-memory dependencies.
- * It runs ONCE per day, receives the in-memory cache, and must
- * perform its own user data streaming.
+ * --- META REFACTOR (v2) ---
+ * This calculation is `type: "meta"` and expects its dependencies
+ * (the user-investment-profile results) to be fetched by the pass runner.
+ * It then streams root portfolio data.
  */
 
 const { Firestore } = require('@google-cloud/firestore');
 const firestore = new Firestore();
 const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
 const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
+// NOTE: Corrected relative path for data_loader
 const { loadDataByRefs, getPortfolioPartRefs, loadFullDayMap } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader');
 
 const COHORT_PERCENTILE = 0.8; // Top 20%
 const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
 
 class SmartCohortFlow {
+
+    /**
+     * (NEW) Statically declare dependencies.
+     */
+    static getDependencies() {
+        return [PROFILE_CALC_ID];
+    }
+    
     constructor() {
         // Meta-calc, no constructor state needed
     }
 
     /**
-     * Loads the Investor Scores, calculates the cohort threshold, and builds the Set of user IDs.
-     * --- MODIFIED: Reads from in-memory 'computedDependencies' ---
+     * Loads the Investor Scores from the fetched dependency.
      */
-    _loadCohort(logger, computedDependencies) {
-        logger.log('INFO', '[SmartCohortFlow] Loading Investor Scores from in-memory cache...');
+    _loadCohort(logger, fetchedDependencies) {
+        logger.log('INFO', '[SmartCohortFlow] Loading Investor Scores from fetched dependency...');
         
-        const profileData = computedDependencies[PROFILE_CALC_ID];
+        const profileData = fetchedDependencies[PROFILE_CALC_ID];
         
         if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
-            logger.log('WARN', `[SmartCohortFlow] Cannot find dependency in-memory: ${PROFILE_CALC_ID}. Cohort will not be built.`);
+            logger.log('WARN', `[SmartCohortFlow] Cannot find dependency in fetched data: ${PROFILE_CALC_ID}. Cohort will not be built.`);
             return null; // Return null to signal failure
         }
             
@@ -67,7 +75,6 @@ class SmartCohortFlow {
         }
         return valueMap;
     }
-    // --- Sector Rotation Helper (unchanged) ---
     _accumulateSectorInvestment(portfolio, target, sectorMap) {
         if (portfolio && portfolio.AggregatedPositions) {
             for (const pos of portfolio.AggregatedPositions) {
@@ -78,16 +85,20 @@ class SmartCohortFlow {
     }
 
     /**
-     * PROCESS: META REFACTOR
-     * This now runs ONCE, loads all data, streams users, and returns one big result.
+     * REFACTORED PROCESS METHOD
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-10-31").
+     * @param {object} dependencies The shared dependencies (db, logger, rootData, etc.).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from previous passes.
+     * @returns {Promise<object|null>} The analysis result or null.
      */
-    async process(dateStr, dependencies, config, computedDependencies) {
+    async process(dateStr, dependencies, config, fetchedDependencies) {
         const { logger, db, rootData, calculationUtils } = dependencies;
         const { portfolioRefs } = rootData;
         logger.log('INFO', '[SmartCohortFlow] Starting meta-process...');
         
         // 1. Load Cohort from in-memory dependency
-        const smartCohortIds = this._loadCohort(logger, computedDependencies);
+        const smartCohortIds = this._loadCohort(logger, fetchedDependencies);
         if (!smartCohortIds) {
             return null; // Dependency failed
         }
@@ -100,7 +111,7 @@ class SmartCohortFlow {
         ]);
         if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
             logger.log('ERROR', '[SmartCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
-            return null; // Return null to trigger backfill
+            return null; 
         }
 
         // 3. Load "yesterday's" portfolio data for comparison
@@ -112,13 +123,10 @@ class SmartCohortFlow {
         logger.log('INFO', `[SmartCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
 
         // 4. Stream "today's" portfolio data and process
-        
-        // --- Local state for this run ---
         const asset_values = {};
         const todaySectorInvestment = {};
         const yesterdaySectorInvestment = {};
         let user_count = 0;
-        // --- End Local state ---
 
         const batchSize = config.partRefBatchSize || 10;
         for (let i = 0; i < portfolioRefs.length; i += batchSize) {
@@ -127,10 +135,7 @@ class SmartCohortFlow {
 
             for (const uid in todayPortfoliosChunk) {
                 
-                // --- Filter user ---
-                if (!smartCohortIds.has(uid)) {
-                    continue;
-                }
+                if (!smartCohortIds.has(uid)) continue; // --- Filter user ---
                 
                 const pToday = todayPortfoliosChunk[uid];
                 const pYesterday = yesterdayPortfolios[uid];
@@ -139,8 +144,6 @@ class SmartCohortFlow {
                     continue;
                 }
                 
-                // --- User is in cohort, run logic ---
-                
                 // 4a. RUN ASSET FLOW LOGIC
                 const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
                 const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
@@ -155,15 +158,13 @@ class SmartCohortFlow {
                 // 4b. RUN SECTOR ROTATION LOGIC
                 this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
                 this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
-
                 user_count++;
             }
         }
         
         logger.log('INFO', `[SmartCohortFlow] Processed ${user_count} users in cohort.`);
         
-        // --- 5. GETRESULT LOGIC IS NOW INSIDE PROCESS ---
-        
+        // --- 5. GETRESULT LOGIC ---
         if (user_count === 0) {
             logger.warn('[SmartCohortFlow] No users processed for smart cohort. Returning null.');
             return null;
@@ -203,7 +204,6 @@ class SmartCohortFlow {
             return null;
         }
 
-        // 6. Return combined result
         return {
             asset_flow: finalAssetFlow,
             sector_rotation: finalSectorRotation,