bulltrackers-module 1.0.207 → 1.0.209

package/functions/computation-system/controllers/computation_controller.js

@@ -1,10 +1,10 @@
 /**
  * FIXED: computation_controller.js
- * V4.1: Supports Smart Shard Lookup via Wrapper
+ * V4.2: Added InsightsExtractor and UserClassifier to Context
  */
 
-const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives'); // TODO, SURELY THIS COULD BE DYNAMIC, DETECT WHAT IS EXPORTED? 
-const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');                                                                     // THE IDEA IS THAT WE CAN PRODUCE NEW COMPUTATIONS IN MATH PRIMITIVES, AND NOT NEED TO UPDATE ANY OTHER CODE FILE
+const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor, InsightsExtractor, UserClassifier } = require('../layers/math_primitives'); 
+const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
 
 class DataLoader {
     constructor(config, dependencies) {
@@ -64,7 +64,7 @@ class DataLoader {
     }
 }
 
-class ContextBuilder { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
+class ContextBuilder { 
     static buildPerUserContext(options) {
         const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
@@ -73,14 +73,27 @@ class ContextBuilder { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LIS
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             mappings: mappings || {},
-            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution : DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
+            math: { 
+                extract: DataExtractor, 
+                history: HistoryExtractor, 
+                compute: MathPrimitives, 
+                aggregate: Aggregators, 
+                validate: Validators, 
+                signals: SignalPrimitives, 
+                schemas: SCHEMAS, 
+                distribution : DistributionAnalytics, 
+                TimeSeries: TimeSeries, 
+                priceExtractor : priceExtractor,
+                insights: InsightsExtractor, // Mapped for new Meta calcs
+                classifier: UserClassifier   // Mapped for Smart/Dumb logic
+            },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
             meta: metadata, config, deps
         };
     }
 
-    static buildMetaContext(options) { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
+    static buildMetaContext(options) {
         const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
             date: { today: dateStr },
@@ -88,7 +101,20 @@ class ContextBuilder { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LIS
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             prices: prices || {},
             mappings: mappings || {},
-            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution: DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
+            math: { 
+                extract: DataExtractor, 
+                history: HistoryExtractor, 
+                compute: MathPrimitives, 
+                aggregate: Aggregators, 
+                validate: Validators, 
+                signals: SignalPrimitives, 
+                schemas: SCHEMAS, 
+                distribution: DistributionAnalytics, 
+                TimeSeries: TimeSeries, 
+                priceExtractor : priceExtractor,
+                insights: InsightsExtractor, // Mapped for new Meta calcs
+                classifier: UserClassifier   // Mapped for Smart/Dumb logic
+            },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
             meta: metadata, config, deps
@@ -159,4 +185,4 @@ class ComputationController {
     }
 }
 
-module.exports = { ComputationController };
+module.exports = { ComputationController };

package/functions/computation-system/layers/math_primitives.js

@@ -2,15 +2,121 @@
  * @fileoverview Math Layer - Single Source of Truth (V3 Final)
  * Encapsulates Schema Knowledge, Mathematical Primitives, and Signal Extractors.
  * * STRICT COMPLIANCE:
- * - Adheres to 'schema.md' definitions for Normal vs Speculator portfolios.
+ * - Adheres to 'schema.md' definitions.
  * - standardizes access to P&L, Weights, and Rates.
- * - Provides safe fallbacks for all fields.
  */
 
 const SCHEMAS = {
-    USER_TYPES: { NORMAL: 'normal', SPECULATOR: 'speculator' }
+    USER_TYPES: { NORMAL: 'normal', SPECULATOR: 'speculator' },
+    STYLES: { DAY_TRADER: 'Day Trader', SWING_TRADER: 'Swing Trader', INVESTOR: 'Investor' },
+    LABELS: { SMART: 'Smart Money', DUMB: 'Dumb Money', NEUTRAL: 'Neutral' }
 };
 
+class InsightsExtractor {
+    /**
+     * Extracts the raw array of insight objects from the context.
+     * Checks for standard context injection paths.
+     */
+    static getInsights(context) {
+        // Support multiple potential injection paths depending on controller version
+        return context.insights || context.daily_instrument_insights || [];
+    }
+
+    /**
+     * returns the specific insight object for a given instrument ID.
+     */
+    static getInsightForInstrument(insights, instrumentId) {
+        if (!insights || !Array.isArray(insights)) return null;
+        return insights.find(i => i.instrumentId === instrumentId) || null;
+    }
+
+    // --- Standard Metrics ---
+
+    static getTotalOwners(insight) {
+        return insight ? (insight.total || 0) : 0;
+    }
+
+    static getLongPercent(insight) {
+        return insight ? (insight.buy || 0) : 0;
+    }
+
+    static getShortPercent(insight) {
+        return insight ? (insight.sell || 0) : 0;
+    }
+
+    static getGrowthPercent(insight) {
+        return insight ? (insight.growth || 0) : 0;
+    }
+
+    // --- Derived Counts (Estimated) ---
+
+    static getLongCount(insight) {
+        const total = this.getTotalOwners(insight);
+        const buyPct = this.getLongPercent(insight);
+        return Math.floor(total * (buyPct / 100));
+    }
+
+    static getShortCount(insight) {
+        const total = this.getTotalOwners(insight);
+        const sellPct = this.getShortPercent(insight);
+        return Math.floor(total * (sellPct / 100));
+    }
+
+    /**
+     * Calculates the net change in users from yesterday based on growth %.
+     * Formula: NetChange = Total - (Total / (1 + Growth/100))
+     */
+    static getNetOwnershipChange(insight) {
+        const total = this.getTotalOwners(insight);
+        const growth = this.getGrowthPercent(insight);
+        if (total === 0) return 0;
+        
+        // Reverse engineer yesterday's count
+        // Today = Yesterday * (1 + growth)
+        // Yesterday = Today / (1 + growth)
+        const prevTotal = total / (1 + (growth / 100)); // TODO: Check precision issues
+        return Math.round(total - prevTotal);
+    }
+}
+
+class UserClassifier {
+    /**
+     * Classifies a user as Smart/Dumb based on Win Rate and Profitability.
+     * @param {Object} historySummary - Result from HistoryExtractor.getSummary()
+     */
+    static classifySmartDumb(historySummary) {
+        if (!historySummary || historySummary.totalTrades < 5) return SCHEMAS.LABELS.NEUTRAL; // Insufficient data
+
+        const { winRatio, avgProfitPct, avgLossPct } = historySummary;
+
+        // 1. The "Consistent Winner" (Smart)
+        if (winRatio > 60 && avgProfitPct > Math.abs(avgLossPct)) return SCHEMAS.LABELS.SMART;
+
+        // 2. The "Sniper" (Smart - Low Win Rate but huge winners)
+        if (winRatio > 30 && avgProfitPct > (Math.abs(avgLossPct) * 2.5)) return SCHEMAS.LABELS.SMART;
+
+        // 3. The "Bagholder" (Dumb - High win rate but one loss wipes them out)
+        if (winRatio > 80 && (Math.abs(avgLossPct) > avgProfitPct * 4)) return SCHEMAS.LABELS.DUMB;
+
+        // 4. The "Gambler" (Dumb - Losing money consistently)
+        if (winRatio < 40 && avgProfitPct < Math.abs(avgLossPct)) return SCHEMAS.LABELS.DUMB;
+
+        return SCHEMAS.LABELS.NEUTRAL;
+    }
+
+    /**
+     * Classifies trading style based on average holding time.
+     * @param {number} avgHoldingMinutes 
+     */
+    static classifyStyle(avgHoldingMinutes) {
+        if (avgHoldingMinutes <= 0) return SCHEMAS.STYLES.INVESTOR; // Default
+        
+        if (avgHoldingMinutes < 60 * 24) return SCHEMAS.STYLES.DAY_TRADER; // < 1 Day
+        if (avgHoldingMinutes < 60 * 24 * 30) return SCHEMAS.STYLES.SWING_TRADER; // < 1 Month
+        return SCHEMAS.STYLES.INVESTOR; // > 1 Month
+    }
+}
+
 class DataExtractor { // For generic access of data types
     // ========================================================================
     // 1. COLLECTION ACCESSORS
@@ -124,15 +230,12 @@ class DataExtractor { // For generic access of data types
     static getPortfolioDailyPnl(portfolio, userType) {
         if (!portfolio) return 0;
         
-        // 1. Speculator (Explicit 'NetProfit' field on root) ---> TODO : THIS IS FLAWED,
-        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {       //  SPECULATOR DATA DOES NOT CONTAIN THE ENTIRE PORTFOLIO DATA FOR THAT USER, IT CONTAINS A SNAPSHOT OF THE USERS' SPECIFIC ASSET WE REQUESTED FROM THE TASK ENGINE
-                                                                // You cannot ever infer the entire speculator portfolio daily / weekly / monthly or any time period value based on the data we have for speculators
-            return portfolio.NetProfit || 0;                    // Data for speculators only works when we are looking at the ticker-specific data, or the trade history data which is agnostic on user type
+        // 1. Speculator (Explicit 'NetProfit' field on root)
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {       
+            return portfolio.NetProfit || 0;                    
         }
         
-        // 2. Normal (Aggregated Calculation) ---> TODO : VERIFY THIS LOGIC, IT SHOULD BE OK
-        // Logic: Sum(Value - Invested) across the 'InstrumentType' breakdown
-        // This gives the net performance change for the day relative to the portfolio.
+        // 2. Normal (Aggregated Calculation)
         if (portfolio.AggregatedPositionsByInstrumentTypeID) {
             return portfolio.AggregatedPositionsByInstrumentTypeID.reduce((sum, agg) => {
                 return sum + ((agg.Value || 0) - (agg.Invested || 0));
@@ -167,8 +270,6 @@ class DataExtractor { // For generic access of data types
 
     static getStopLossRate(position) {
         const rate = position ? (position.StopLossRate || 0) : 0; 
-        // Fix for eToro bug: SL disabled positions can return 0.01 or similar small values.
-        // If the rate is positive but extremely small (<= 0.01), treat as 0 (disabled).
         if (rate > 0 && rate <= 0.01) return 0; // Normalizes bug value to 0
         if (rate < 0) return 0;
         return rate;
@@ -176,8 +277,6 @@ class DataExtractor { // For generic access of data types
 
     static getTakeProfitRate(position) {
         const rate = position ? (position.TakeProfitRate || 0) : 0;
-        // Fix for eToro bug: TP disabled positions can return INF or small values.
-        // If the rate is positive but extremely small (<= 0.01), treat as 0 (disabled).
         if (rate > 0 && rate <= 0.01) return 0; // Normalizes bug value to 0
         return rate;
     }
@@ -186,38 +285,17 @@ class DataExtractor { // For generic access of data types
         return position ? (position.HasTrailingStopLoss === true) : false; // Default false IF NOT FOUND
     }
 
-    /**
-     * Extract Open Date Time.
-     * Used for bagholder calculations.
-     */
     static getOpenDateTime(position) {
         if (!position || !position.OpenDateTime) return null;        
         return new Date(position.OpenDateTime);
     }
 }
 
-/**
- * --- NEW CLASS: priceExtractor ---
- * Handles schema-aware extraction of asset price history.
- * Mitigates typo risk by centralizing access to context.prices.
- */
 class priceExtractor {
-    /**
-     * Retrieves the sorted price history for a specific instrument.
-     * @param {object} pricesContext - The global context.prices object.
-     * @param {string|number} tickerOrId - The Ticker or InstrumentID to fetch.
-     * @returns {Array<{date: string, price: number}>} Sorted array of price objects.
-     */
     static getHistory(pricesContext, tickerOrId) {
         if (!pricesContext || !pricesContext.history) return [];
-        
-        // The history map is keyed by InstrumentID (e.g., "10000")
-        // We iterate to find the entry matching the request (by ID or Ticker)
-        // Optimization: If input is ID, direct lookup. If ticker, search.
-        
         let assetData = pricesContext.history[tickerOrId];
         
-        // Fallback: If direct lookup failed, maybe it's a ticker symbol?
         if (!assetData) {
             const id = Object.keys(pricesContext.history).find(key => {
                 const data = pricesContext.history[key];
@@ -228,7 +306,6 @@ class priceExtractor {
 
         if (!assetData || !assetData.prices) return [];
 
-        // Convert Map<Date, Price> to Array<{date, price}> and sort
         const priceMap = assetData.prices;
         const sortedDates = Object.keys(priceMap).sort((a, b) => a.localeCompare(b));
         
@@ -238,20 +315,12 @@ class priceExtractor {
         })).filter(item => item.price > 0);
     }
 
-    /**
-     * Returns all available assets with their histories.
-     * Useful for computations iterating over the entire market.
-     * @param {object} pricesContext - The global context.prices object.
-     * @returns {Map<string, Array<{date: string, price: number}>>} Map<Ticker, HistoryArray>
-     */
     static getAllHistories(pricesContext) {
         if (!pricesContext || !pricesContext.history) return new Map();
         
         const results = new Map();
         for (const [id, data] of Object.entries(pricesContext.history)) {
             const ticker = data.ticker || id;
-            // Reuse the single-asset logic for consistency, though slightly less efficient
-            // than inlining the sort. For safety/DRY, we reuse.
             const history = this.getHistory(pricesContext, id);
             if (history.length > 0) {
                 results.set(ticker, history);
@@ -262,26 +331,14 @@ class priceExtractor {
 }
 
 class HistoryExtractor {
-    // --- Schema Accessor (REFACTORED for Granular API) ---
-    /**
-     * Extracts the daily history snapshot from the User object.
-     * Returns the raw granular positions list.
-     */
     static getDailyHistory(user) {
-        // The new API returns { PublicHistoryPositions: [...] }
         return user?.history?.today || null;
     }
 
-    // --- Data Extractors (ADAPTER PATTERN) ---
-    /**
-     * Adapts granular trade history into the legacy 'TradedAssets' format.
-     * Groups trades by InstrumentID and calculates average holding time.
-     */
     static getTradedAssets(historyDoc) {
         const trades = historyDoc?.PublicHistoryPositions || [];
         if (!trades.length) return [];
         
-        // Group by InstrumentID
         const assetsMap = new Map();
         
         for (const t of trades) {
@@ -297,11 +354,9 @@ class HistoryExtractor {
             }
             
             const asset = assetsMap.get(instId);
-            
-            // Calculate Duration in Minutes
             const open = new Date(t.OpenDateTime);
             const close = new Date(t.CloseDateTime);
-            const durationMins = (close - open) / 60000; // ms -> min
+            const durationMins = (close - open) / 60000;
             
             if (durationMins > 0) {
                 asset.totalDuration += durationMins;
@@ -309,8 +364,6 @@ class HistoryExtractor {
             }
         }
         
-        // Convert Map to Array format expected by existing calculations
-        // (Returns objects with .instrumentId and .avgHoldingTimeInMinutes)
         return Array.from(assetsMap.values()).map(a => ({
             instrumentId: a.instrumentId,
             avgHoldingTimeInMinutes: a.count > 0 ? (a.totalDuration / a.count) : 0
@@ -325,10 +378,6 @@ class HistoryExtractor {
         return asset ? (asset.avgHoldingTimeInMinutes || 0) : 0;
     }
 
-    /**
-     * Adapts granular trade history into the legacy 'Summary' format.
-     * Calculates WinRatio, AvgProfit, etc. on the fly from the raw list.
-     */
     static getSummary(historyDoc) { 
         const trades = historyDoc?.PublicHistoryPositions || [];
         if (!trades.length) return null;
@@ -342,7 +391,6 @@ class HistoryExtractor {
         let totalDur = 0;
 
         for (const t of trades) {
-            // P&L Stats (NetProfit is %)
             if (t.NetProfit > 0) {
                 wins++;
                 totalProf += t.NetProfit;
@@ -352,10 +400,9 @@ class HistoryExtractor {
                 lossCount++;
             }
             
-            // Duration Stats
             const open = new Date(t.OpenDateTime);
             const close = new Date(t.CloseDateTime);
-            totalDur += (close - open) / 60000; // ms -> min
+            totalDur += (close - open) / 60000;
         }
 
         return { 
@@ -369,9 +416,6 @@ class HistoryExtractor {
 }
 
 class SignalPrimitives {
-    /**
-     * Safely extracts a specific numeric field for a specific ticker from a dependency.
-     */
     static getMetric(dependencies, calcName, ticker, fieldName, fallback = 0) {
         if (!dependencies || !dependencies[calcName]) return fallback;
         const tickerData = dependencies[calcName][ticker];
@@ -381,13 +425,9 @@ class SignalPrimitives {
         return (typeof val === 'number') ? val : fallback;
     }
 
-    /**
-     * Creates a unified Set of all keys (tickers) present across multiple dependency results.
-     */
     static getUnionKeys(dependencies, calcNames) {
         const keys = new Set();
         if (!dependencies) return [];
-        
         for (const name of calcNames) {
             const resultObj = dependencies[name];
             if (resultObj && typeof resultObj === 'object') {
@@ -397,40 +437,29 @@ class SignalPrimitives {
         return Array.from(keys);
     }
 
-    /**
-     * Hyperbolic Tangent Normalization.
-     * Maps inputs to a strict -Scale to +Scale range.
-     */
-    static normalizeTanh(value, scale = 10, sensitivity = 10.0) { // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/tanh
+    static normalizeTanh(value, scale = 10, sensitivity = 10.0) {
         if (value === 0) return 0;
         return Math.tanh(value / sensitivity) * scale;
     }
 
-    /**
-     * Standard Z-Score normalization.
-     */
-    static normalizeZScore(value, mean, stdDev) { // https://gist.github.com/Restuta/5cbf1d186f17febe5e899febb63e1b86
+    static normalizeZScore(value, mean, stdDev) {
         if (!stdDev || stdDev === 0) return 0;
         return (value - mean) / stdDev;
     }
 
-    /**
-     * Simple Divergence (A - B).
-     */
     static divergence(valueA, valueB) {
         return (valueA || 0) - (valueB || 0);
     }
 
-    static getPreviousState(previousComputed, calcName, ticker, fieldName = null) { // This is used for either fetching computations listed in getdependencies() OR self-history
-        if (!previousComputed || !previousComputed[calcName]) return null;          // Using this for self-history DOES NOT cause a circular dependency because we assign a special rule in orchestration_helpers
-                                                                                    // Which handles the self-reference.
+    static getPreviousState(previousComputed, calcName, ticker, fieldName = null) {
+        if (!previousComputed || !previousComputed[calcName]) return null;
         const tickerData = previousComputed[calcName][ticker];
         if (!tickerData) return null;
 
         if (fieldName) {
              return tickerData[fieldName];
         }
-        return tickerData; // Return whole object (useful for _state)
+        return tickerData; 
     }
 }
 
@@ -460,37 +489,17 @@ class MathPrimitives {
         return value > threshold ? 'winner' : 'loser';
     }
 
-    /**
-     * Calculates the probability of an asset hitting a specific price barrier (Stop Loss/Take Profit)
-     * within a given timeframe using the First Passage Time of Geometric Brownian Motion.
-     * * Formula: P(T < t) = Φ((b - vt) / σ√t) + exp(2vb/σ²) * Φ((b + vt) / σ√t)
-     * Where: 
-     * b = ln(Barrier/Price)
-     * v = drift - 0.5 * volatility^2
-     * @param {number} currentPrice - The current price of the asset
-     * @param {number} barrierPrice - The target price (SL or TP)
-     * @param {number} volatility - Annualized volatility (e.g., 0.40 for 40%)
-     * @param {number} days - Number of days to forecast (e.g., 3)
-     * @param {number} drift - (Optional) Annualized drift. Default 0 (Risk Neutral).
-     * @returns {number} Probability (0.0 to 1.0)
-     */
-    static calculateHitProbability(currentPrice, barrierPrice, volatility, days, drift = 0) { // https://www.ma.ic.ac.uk/~bin06/M3A22/m3f22chVII.pdf
+    static calculateHitProbability(currentPrice, barrierPrice, volatility, days, drift = 0) {
         if (currentPrice <= 0 || barrierPrice <= 0 || volatility <= 0 || days <= 0) return 0;
 
-        const t     = days / 365.0; // Convert days to years
+        const t     = days / 365.0; 
         const sigma = volatility;
         const mu    = drift;
-        
-        // The barrier in log-space
         const b = Math.log(barrierPrice / currentPrice);
-        
-        // Adjusted drift (nu)
         const nu = mu - 0.5 * Math.pow(sigma, 2);
-
         const sqrtT      = Math.sqrt(t);
         const sigmaSqrtT = sigma * sqrtT;
 
-        // Helper for Standard Normal CDF (Φ)
         const normCDF = (x) => {
             const t = 1 / (1 + 0.2316419 * Math.abs(x));
             const d = 0.3989423 * Math.exp(-x * x / 2);
@@ -498,41 +507,20 @@ class MathPrimitives {
             return x > 0 ? 1 - prob : prob;
         };
 
-        // Standard First Passage Time Formula parts
         const term1 = (b - nu * t) / sigmaSqrtT;
         const term2 = (2 * nu * b) / (sigma * sigma);
         const term3 = (b + nu * t) / sigmaSqrtT;
 
-        // If barrier is below price (Stop Loss for Long), b is negative.
-        // If barrier is above price (Take Profit for Long), we flip the logic essentially,
-        // but the formula works for the distance. 
-        // However, for strict GBM hitting time, we usually treat 'b' as the distance.
-        // For this implementation, we check direction relative to barrier.
-        
-        // If we are already at or past the barrier, probability is 100%
         if ((currentPrice > barrierPrice && barrierPrice > currentPrice) || 
             (currentPrice < barrierPrice && barrierPrice < currentPrice)) {
              return 1.0;
         }
 
-        // Calculate Probability
-        // Note: If nu is 0, the second term simplifies significantly, but we keep full form.
         const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) +  Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
 
         return Math.min(Math.max(probability, 0), 1);
     }
 
-    /**
-     * --- NEW PRIMITIVE ---
-     * Simulates future price paths using Geometric Brownian Motion (Monte Carlo).
-     * Used for testing portfolio resilience against potential market moves.
-     * @param {number} currentPrice - S0
-     * @param {number} volatility - Annualized volatility (sigma)
-     * @param {number} days - Time horizon in days (t)
-     * @param {number} simulations - Number of paths to generate (e.g., 1000)
-     * @param {number} drift - Annualized drift (mu), default 0
-     * @returns {Float32Array} Array of simulated end prices
-     */
     static simulateGBM(currentPrice, volatility, days, simulations = 1000, drift = 0) {
         if (currentPrice <= 0 || volatility <= 0 || days <= 0) return new Float32Array(0);
         
@@ -542,29 +530,17 @@ class MathPrimitives {
         const driftTerm = (mu - 0.5 * sigma * sigma) * t;
         const volTerm   = sigma * Math.sqrt(t);
         
-        // Use Float32Array for memory efficiency with large simulation counts
         const results = new Float32Array(simulations);
         
         for (let i = 0; i < simulations; i++) {
-            // Box-Muller transform for efficient standard normal distribution generation
             const u1 = Math.random();
             const u2 = Math.random();
             const z  = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
-            
-            // GBM Formula: St = S0 * exp((mu - 0.5*sigma^2)t + sigma*Wt)
             results[i] = currentPrice * Math.exp(driftTerm + volTerm * z);
         }
         return results;
     }
 
-    /**
-     * --- NEW PRIMITIVE ---
-     * Simulates "Population Breakdown" (Capitulation) Risk.
-     * Correlates simulated price drops with user pain thresholds.
-     * @param {Float32Array} pricePaths - Array of simulated prices (from simulateGBM).
-     * @param {Array<{entryPrice: number, thresholdPct: number}>} userProfiles - Array of user states.
-     * @returns {number} Expected % of population that capitulates (0.0 - 1.0).
-     */
     static simulatePopulationBreakdown(pricePaths, userProfiles) {
         if (!pricePaths.length || !userProfiles.length) return 0;
         
@@ -572,43 +548,30 @@ class MathPrimitives {
         const totalSims = pricePaths.length;
         const totalUsers = userProfiles.length;
 
-        // For each simulated future price scenario...
         for (let i = 0; i < totalSims; i++) {
             const simPrice = pricePaths[i];
             let capitulatedUsersInScenario = 0;
 
-            // ...check every user to see if they survive
             for (let j = 0; j < totalUsers; j++) {
                 const user = userProfiles[j];
-                // Calculate hypothetical P&L for this user in this scenario
-                // P&L% = (CurrentValue - EntryValue) / EntryValue
                 const hypotheticalPnL = ((simPrice - user.entryPrice) / user.entryPrice) * 100;
                 
-                // If hypothetical P&L is worse (lower) than their historical pain threshold, they capitulate.
-                // Note: thresholdPct is typically negative (e.g., -15.0)
                 if (hypotheticalPnL < user.thresholdPct) {
                     capitulatedUsersInScenario++;
                 }
             }
             
-            // Add the % of users who broke in this scenario to the accumulator
             totalBreakdownEvents += (capitulatedUsersInScenario / totalUsers);
         }
         
-        // Return the average capitulation rate across all simulations
         return totalBreakdownEvents / totalSims;
     }
 }
 
 class Aggregators {
-    /**
-     * Helper to bucket users by P&L Status.
-     * Used by legacy systems or specific aggregators.
-     */
-    static bucketUsersByPnlPerAsset(usersData, tickerMap) { // https://www.geeksforgeeks.org/javascript/bucket-sort-visualization-using-javascript/
+    static bucketUsersByPnlPerAsset(usersData, tickerMap) {
         const buckets = new Map();
         for (const [userId, portfolio] of Object.entries(usersData)) {
-            // Auto-detect type if not provided (Legacy compatibility) TODO : We do not need legacy compatability, legacy computations do not run.
             const userType  = portfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
             const positions = DataExtractor.getPositions(portfolio, userType);
             
@@ -630,13 +593,6 @@ class Aggregators {
         return Object.fromEntries(buckets);
     }
 
-    /**
-     * --- NEW PRIMITIVE ---
-     * Calculates Weighted Sentiment (Avg P&L) for a set of positions.
-     * Solves "Dirty Data" / Variable N issue by weighting by investment size.
-     * @param {Array} positions - Array of raw position objects.
-     * @returns {number} Weighted Average NetProfit %.
-     */
     static getWeightedSentiment(positions) {
         if (!positions || positions.length === 0) return 0;
 
@@ -644,9 +600,8 @@ class Aggregators {
         let totalWeight = 0;
 
         for (const pos of positions) {
-            // Use DataExtractor to be safe and schema-agnostic
             const pnl = DataExtractor.getNetProfit(pos);
-            const weight = DataExtractor.getPositionWeight(pos); // 'Invested' or 'Amount'
+            const weight = DataExtractor.getPositionWeight(pos);
 
             if (weight > 0) {
                 totalWeightedPnL += (pnl * weight);
@@ -659,12 +614,10 @@ class Aggregators {
     }
 }
 
-// Validation layer -- Used to validate the data incoming 
 class Validators {
     static validatePortfolio(portfolio, userType) {
         if (!portfolio) return { valid: false, errors: ['Portfolio is null'] };
         
-        // Handle both types of schema
         if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
              if (!portfolio.PublicPositions) return { valid: false, errors: ['Missing PublicPositions'] };
         } else {
@@ -676,38 +629,25 @@ class Validators {
 }
 
 class TimeSeries {
-    /**
-     * Updates a Rolling Mean and Variance using Welford's Online Algorithm (EMA variant). // https://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/
-     * @param {number} value - New data point.
-     * @param {Object} state - { mean: number, variance: number }
-     * @param {number} alpha - Decay factor (e.g., 0.1 for ~20 days).
-     */
     static updateEMAState(value, state, alpha = 0.1) {
         const mean = state ? (state.mean || 0) : 0;
-        const variance = state ? (state.variance || 1) : 1; // Default variance 1 to avoid div/0
+        const variance = state ? (state.variance || 1) : 1; 
 
         if (value === undefined || value === null || isNaN(value)) {
             return { mean, variance };
         }
 
-        // EMA Update for Mean
         const diff = value - mean;
         const newMean = mean + (alpha * diff);
-
-        // EMA Update for Variance
         const newVariance = (1 - alpha) * (variance + (alpha * diff * diff));
 
         return { mean: newMean, variance: newVariance };
     }
 
-    /**
-     * Calculates Pearson Correlation between two arrays. https://gist.github.com/matt-west/6500993
-     */
     static pearsonCorrelation(x, y) {
         if (!x || !y || x.length !== y.length || x.length === 0) return 0;
         
         const n = x.length;
-        // Simple sums
         let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0, sumY2 = 0;
 
         for (let i = 0; i < n; i++) {
@@ -727,14 +667,6 @@ class TimeSeries {
 
 
 class DistributionAnalytics {
-    /**
-     * Gaussian Kernel Density Estimation (KDE)
-     * Converts discrete price points into a continuous probability density curve.
-     * Optimized for memory: accepts pre-binned data.
-     * @param {Array<{value: number, weight: number}>} data - Points or Bins.
-     * @param {number} bandwidth - Smoothing factor (h).
-     * @param {number} steps - Resolution of the output curve.
-     */
     static computeKDE(data, bandwidth, steps = 60) {
         if (!data || data.length === 0) return [];
 
@@ -744,7 +676,6 @@ class DistributionAnalytics {
             if (p.value > max) max = p.value;
         }
         
-        // Pad range to capture tails
         min -= bandwidth * 3;
         max += bandwidth * 3;
         const stepSize = (max - min) / steps;
@@ -753,14 +684,12 @@ class DistributionAnalytics {
         for (let i = 0; i <= steps; i++) {
             const x = min + (i * stepSize);
             let density = 0;
-            // Vectorized-like summation https://cvw.cac.cornell.edu/vector/intro/how-vector-works#:~:text=Vectorization%20is%20a%20process%20by,performance%20increases%20obtained%20by%20vectorization.
             for (const p of data) {
                 const diff = (x - p.value);
-                // Optimization: Skip calculation for points too far away (> 3 std devs)
                 if (Math.abs(diff) > bandwidth * 3) continue;
                 
                 const u = diff / bandwidth;
-                const k = 0.39894228 * Math.exp(-0.5 * u * u); // Standard Normal PDF
+                const k = 0.39894228 * Math.exp(-0.5 * u * u); 
                 density += (p.weight * k) / bandwidth;
             }
             if (density > 0) curve.push({ price: x, density });
@@ -769,19 +698,19 @@ class DistributionAnalytics {
     }
 
     static integrateProfile(curve, startPrice, endPrice) {
+        if (!curve || !Array.isArray(curve)) return 0; // Fix for potential crash
         let sum = 0;
         for (let i = 0; i < curve.length - 1; i++) {
             const p1 = curve[i];
             const p2 = curve[i+1];
             if (p1.price >= startPrice && p2.price <= endPrice) {
-                // Trapezoidal Rule https://www.khanacademy.org/math/ap-calculus-ab/ab-integration-new/ab-6-2/a/understanding-the-trapezoid-rule
                 sum += (p2.price - p1.price) * ((p1.density + p2.density) / 2);
             }
         }
         return sum;
     }
 
-    static linearRegression(xValues, yValues) { // https://hazlo.medium.com/linear-regression-from-scratch-in-js-first-foray-into-ml-for-web-developers-867cfcae8fde
+    static linearRegression(xValues, yValues) {
         const n = xValues.length;
         if (n !== yValues.length || n < 2) return { slope: 0, r2: 0 };
         
@@ -799,23 +728,17 @@ class DistributionAnalytics {
     }
 }
 
-
-// TODO, THIS EXPORT CAN SURELY BE DYNAMICALLY PRODUCED? TAKE WHATEVER CLASSES ARE DEFINED? THE IDEA IS THAT WE CAN SIMPLY PRODUCE NEW CLASSES AND NOT NEED TO MAKE ANY FURTHER CHANGES SO :
-// 1. DYNAMICALLY FINDS THE CLASS, EXPORTS HERE
-// 2. DYNAMICALLY IMPORTS INTO COMPUTATION CONTROLLER
-// 3. DYNAMICALLY POPULATES THE MATH VALUE OF THE COMPUTATION CONTROLLER FOR STANDARD AND META COMPUTATION CONTEXTBUILDERS
-// 4. GO AHEAD AND WRITE YOUR COMPUTATION FILES USING THE NEW MATH PRIMITIVES DEFINED AS NEEDED.
-// THIS THEN MEANS THE MATH PRIMTIVIES CAN BE UPDATED EASILY WITHOUT FURTHER CHANGES TO OTHER SYSTEMS AND AVOID COMPLICATED DEBUGGING OR FORGETFUL INTEGRATIONS.
-
 module.exports = {
     Aggregators,
     DataExtractor,
     DistributionAnalytics,
     HistoryExtractor,
+    InsightsExtractor, // Exported
     MathPrimitives,
     SCHEMAS,
     SignalPrimitives,
     TimeSeries,
+    UserClassifier, // Exported
     Validators,
     priceExtractor
 };

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -12,18 +12,30 @@
  */
 async function checkDiscoveryNeed(userType, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
+    const { targetUsersPerBlock, normalBlockCountsDocPath, allHighValueBlocks } = config;
+    
+    // --- REFACTOR: ORGANIC SPECULATOR DISCOVERY ---
+    // We no longer "hunt" for speculators using block targets. 
+    // Speculators are now discovered organically via the Normal User Update Loop (Feedback Loop).
+    if (userType === 'speculator') {
+        logger.log('INFO', '[Orchestrator Helpers] Speculator discovery is Organic/Feedback-Driven. Skipping active discovery checks.');
+        return { needsDiscovery: false, blocksToFill: [] };
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Checking discovery need for ${userType}...`);
-    const { targetUsersPerBlock, speculatorBlockCountsDocPath, normalBlockCountsDocPath,speculatorInstruments,allHighValueBlocks} = config;
-    const isSpeculator = userType === 'speculator';
-    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { speculatorBlockCountsDocPath, normalBlockCountsDocPath }, userType);
-    let underPopulatedBlocks = [];
-    if (isSpeculator) {for (const instrument of speculatorInstruments) { for (const block of allHighValueBlocks) 
-        {const instrumentBlockKey = `${instrument}_${block.startId}`;
-        if ((blockCounts[instrumentBlockKey] || 0) < targetUsersPerBlock) {underPopulatedBlocks.push({ ...block, instrument });}}}} 
-        else {underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);}
-        const needsDiscovery = underPopulatedBlocks.length > 0;
-        if (!needsDiscovery) {logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);} 
-        else {logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks/instruments for ${userType}.`);}
+    
+    // Logic for Normal Users remains (fill blocks to target)
+    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { normalBlockCountsDocPath }, userType);
+    
+    const underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);
+    const needsDiscovery = underPopulatedBlocks.length > 0;
+
+    if (!needsDiscovery) {
+        logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);
+    } else {
+        logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks for ${userType}.`);
+    }
+
     return { needsDiscovery, blocksToFill: underPopulatedBlocks };
 }
 
@@ -38,29 +50,70 @@ async function checkDiscoveryNeed(userType, config, dependencies) {
 async function getDiscoveryCandidates(userType, blocksToFill, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
     logger.log('INFO', `[Orchestrator Helpers] Getting discovery candidates for ${userType}...`);
-    const {maxTasksPerRun,discoveryBatchSize,maxRandomCidsToDiscover,specBlocksCollection,pendingSpecCollection,normalUserCollection,invalidSpecCollection,speculatorInstrumentSet,snapshotsSubCollectionName,partsSubCollectionName} = config;
-    const isSpeculator = userType === 'speculator';
+    
+    const {
+        maxTasksPerRun,
+        discoveryBatchSize,
+        maxRandomCidsToDiscover,
+        specBlocksCollection,
+        pendingSpecCollection,
+        normalUserCollection,
+        invalidSpecCollection,
+        snapshotsSubCollectionName,
+        partsSubCollectionName
+    } = config;
+
     const dispatchedCids = new Set();
-    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies,{ normalUserCollectionName: normalUserCollection, snapshotsSubCollectionName, partsSubCollectionName });
+
+    // Fetch exclusion list to avoid re-discovering existing users
+    // Note: For Normal users, we check against existing Normal users.
+    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies, { 
+        normalUserCollectionName: normalUserCollection, 
+        snapshotsSubCollectionName, 
+        partsSubCollectionName 
+    });
+    
     const existingNormalUserIds = new Set(Object.keys(latestNormalPortfolios));
-    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {specBlocksCollection, pendingSpecCollection, normalUserCollection,invalidSpecCollection,existingNormalUserIds}, userType);
-    if (isSpeculator) {const prioritizedCandidates = await firestoreUtils.getPrioritizedSpeculators(dependencies,exclusionIds, speculatorInstrumentSet,latestNormalPortfolios);
-        logger.log('INFO', `Found ${prioritizedCandidates.length} potential new speculators from existing user pool.`);
-        prioritizedCandidates.forEach(cidStr => {if (dispatchedCids.size < maxTasksPerRun) {dispatchedCids.add(cidStr);}});
-        logger.log('INFO', `Added ${dispatchedCids.size} prioritized speculators.`);}
+    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        normalUserCollection,
+        invalidSpecCollection,
+        existingNormalUserIds
+    }, userType);
+
+    // --- RANDOM DISCOVERY LOOP (For Normal Users) ---
     let dispatchedRandomCidCount = 0;
-    const initialDispatchedCount = dispatchedCids.size;
-    while ((dispatchedRandomCidCount + initialDispatchedCount) < maxRandomCidsToDiscover &&dispatchedCids.size < maxTasksPerRun && blocksToFill.length > 0){
+    
+    while ((dispatchedRandomCidCount + dispatchedCids.size) < maxRandomCidsToDiscover && 
+           dispatchedCids.size < maxTasksPerRun && 
+           blocksToFill.length > 0) {
+        
         const blockIndex = dispatchedCids.size % blocksToFill.length;
         const block = blocksToFill[blockIndex];
         if (!block) break; 
+
         for (let j = 0; j < discoveryBatchSize; j++) {
-        if ((dispatchedRandomCidCount + initialDispatchedCount) >= maxRandomCidsToDiscover || dispatchedCids.size >= maxTasksPerRun) break;
-        let randomId;
-        let retryCount = 0;
-        const MAX_RETRIES = 50;
-        do {if (++retryCount > MAX_RETRIES) break;randomId = String(Math.floor(Math.random() * 1000000) + block.startId);} while (exclusionIds.has(randomId) ||dispatchedCids.has(randomId));
-        if (retryCount <= MAX_RETRIES) {dispatchedCids.add(randomId);dispatchedRandomCidCount++;}}}
+            if ((dispatchedRandomCidCount + dispatchedCids.size) >= maxRandomCidsToDiscover || 
+                dispatchedCids.size >= maxTasksPerRun) break;
+
+            let randomId;
+            let retryCount = 0;
+            const MAX_RETRIES = 50;
+            
+            do {
+                if (++retryCount > MAX_RETRIES) break;
+                // Generate random CID within the block range
+                randomId = String(Math.floor(Math.random() * 1000000) + block.startId);
+            } while (exclusionIds.has(randomId) || dispatchedCids.has(randomId));
+
+            if (retryCount <= MAX_RETRIES) {
+                dispatchedCids.add(randomId);
+                dispatchedRandomCidCount++;
+            }
+        }
+    }
+
     logger.log('INFO', `Generated ${dispatchedRandomCidCount} random CIDs for ${userType} discovery.`);
     logger.log('INFO', `Total candidates for dispatch: ${dispatchedCids.size}`);
     return dispatchedCids; 
@@ -75,34 +128,54 @@ async function getDiscoveryCandidates(userType, blocksToFill, config, dependenci
  * @returns {Promise<void>}
  */
 async function dispatchDiscovery(userType, candidates, config, dependencies) {
-    const { logger, firestoreUtils, pubsubUtils, pubsub } = dependencies; 
-    const {topicName,dispatchBatchSize,pubsubBatchSize, pendingSpecCollection, pendingMaxFieldsPerDoc,pendingMaxWritesPerBatch} = config;
-    if (candidates.size === 0) {logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);return;}
+    const { logger, pubsub } = dependencies; 
+    const { topicName, dispatchBatchSize } = config;
+
+    if (candidates.size === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);
+        return;
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Dispatching ${candidates.size} discovery tasks for ${userType}...`);
-    const isSpeculator = userType === 'speculator';
+
     const cidsArray = Array.from(candidates);
-    if (isSpeculator) 
-        {await firestoreUtils.clearCollection(dependencies, pendingSpecCollection); 
-        await firestoreUtils.batchWriteShardedIds(dependencies, {collectionPath: pendingSpecCollection,items: cidsArray, timestamp: new Date(),maxFieldsPerDoc: pendingMaxFieldsPerDoc,maxWritesPerBatch: pendingMaxWritesPerBatch});}
     const tasks = [];
-    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
-    if (batchCids.length > 0) {
-    const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
-    tasks.push({type: 'discover',cids: batchCids,blockId, userType});}}
+
+    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {
+        const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
+        if (batchCids.length > 0) {
+            const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
+            tasks.push({
+                type: 'discover',
+                cids: batchCids,
+                blockId, 
+                userType
+            });
+        }
+    }
+
     const topic = pubsub.topic(topicName);
     let totalCidsPublished = 0;
     let messagesPublished = 0;
+
     for (let i = 0; i < tasks.length; i += dispatchBatchSize) {
         const batchOfTasks = tasks.slice(i, i + dispatchBatchSize);
         const messagePayload = { tasks: batchOfTasks };
-    try {await topic.publishMessage({ json: messagePayload });
-        const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
-        totalCidsPublished += cidsInThisMessage;
-        messagesPublished++;
-        logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);} 
-    catch (publishError) {
-        logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });}}
-        logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks, grouped into ${messagesPublished} Pub/Sub messages for ${userType} discovery.`);
+
+        try {
+            await topic.publishMessage({ json: messagePayload });
+            
+            const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
+            totalCidsPublished += cidsInThisMessage;
+            messagesPublished++;
+            
+            logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);
+        } catch (publishError) {
+            logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });
+        }
+    }
+    
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks for ${userType} discovery.`);
 }
 
 module.exports = { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.207",
+  "version": "1.0.209",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [