bulltrackers-module 1.0.204 → 1.0.206

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -25,7 +25,7 @@ const PARALLEL_BATCH_SIZE = 7;
  */
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
     if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
 
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
@@ -33,21 +33,21 @@ async function runComputationPass(config, dependencies, computationManifest) {
     // Hardcoded earliest dates
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history: new Date('2025-11-05T00:00:00Z'),
-        social: new Date('2025-10-30T00:00:00Z'),
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
 
-    const passes = groupByPass(computationManifest);
+    const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length)
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
 
     const passEarliestDate = earliestDates.absoluteEarliest;
-    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
     // Legacy Batch Optimization for Price (Only used in legacy loop)
@@ -98,10 +98,10 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     // 2. Check Root Data Availability
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history: new Date('2025-11-05T00:00:00Z'),
-        social: new Date('2025-10-30T00:00:00Z'),
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
     };
 
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
@@ -129,8 +129,8 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
 
         // Fetch dependencies (results from this day or yesterday)
         const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-        const prevDate = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr = prevDate.toISOString().slice(0, 10);
+        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr     = prevDate.toISOString().slice(0, 10);
         const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
         if (standardToRun.length) {

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,27 +1,26 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: Context math mapping in runBatchPriceComputation (resolves 'undefined' crash).
- * IMPROVED: Explicit logging for every calculation run (Start, Success, Failure).
- * OPTIMIZED: Parallel Commits & Pipelined Shards.
+ * FIXED: 'commitResults' now isolates commits PER COMPUTATION.
+ * A single failure (e.g., size limit) will only fail that specific calculation,
+ * allowing others in the same pass/date to succeed and be recorded.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
 const { batchStoreSchemas }                  = require('../utils/schema_capture');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
-const { 
-    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
+const {
+    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
-// --- FIX 1: Import Math Layer Primitives for Correct Context Mapping ---
-const { 
-    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, 
-    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, 
-    TimeSeries, priceExtractor 
+const {
+    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
+    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
+    TimeSeries, priceExtractor
 } = require('../layers/math_primitives.js');
 
-const pLimit = require('p-limit'); 
+const pLimit = require('p-limit');
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -30,26 +29,25 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
 
 /**
  * --- PASSIVE DATA VALIDATION ---
- * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
  */
-function validateResultPatterns(logger, calcName, results, category) { 
+function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
 
-    const tickers = Object.keys(results);
+    const tickers    = Object.keys(results);
     const totalItems = tickers.length;
-    
-    if (totalItems < 5) return; 
+
+    if (totalItems < 5) return;
 
     const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
     if (!sampleTicker) return;
-    
+
     const keys = Object.keys(results[sampleTicker]);
-    
+
     keys.forEach(key => {
         if (key.startsWith('_')) return;
 
-        let nullCount = 0;
-        let nanCount = 0;
+        let nullCount      = 0;
+        let nanCount       = 0;
         let undefinedCount = 0;
 
         for (const t of tickers) {
@@ -60,28 +58,25 @@ function validateResultPatterns(logger, calcName, results, category) {
         }
 
         if (nanCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items. Code bug likely.`);
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
         } else if (undefinedCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items. Code bug likely.`);
-        } 
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
+        }
         else if (nullCount > (totalItems * 0.9)) {
-            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items. Check logic if this is unexpected.`);
+            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
         }
     });
 }
 
-/**
- * Checks if all root data dependencies for a given calculation are met.
- */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if      (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price');
+        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
+        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -89,30 +84,30 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs   = [], historyRefs = [];
-    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData    = null, socialData = null;
+    let portfolioRefs = [], historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData = null, socialData = null;
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
-        
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
+        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
+        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
+
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
         }
-        
+
         await Promise.all(tasks);
-        
+
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-        
-        return { 
-            portfolioRefs, 
-            historyRefs, 
-            todayInsights: insightsData, 
-            todaySocialPostInsights: socialData, 
+
+        return {
+            portfolioRefs,
+            historyRefs,
+            todayInsights: insightsData,
+            todaySocialPostInsights: socialData,
             status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }
         };
 
@@ -140,15 +135,15 @@ async function checkPriceDataAvailability(config, dependencies) {
 
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    const snap       = await docRef.get();
+    const docRef = db.collection(collection).doc(dateStr);
+    const snap = await docRef.get();
     return snap.exists ? snap.data() : {};
 }
 
 async function fetchGlobalComputationStatus(config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc('global_status');
-    const snap = await docRef.get();
+    const docRef     = db.collection(collection).doc('global_status');
+    const snap       = await docRef.get();
     return snap.exists ? snap.data() : {};
 }
 
@@ -156,13 +151,13 @@ async function updateComputationStatus(dateStr, updates, config, { db }) {
     if (!updates || Object.keys(updates).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef     = db.collection(collection).doc(dateStr);
-    await docRef.set(updates, { merge: true }); 
+    await docRef.set(updates, { merge: true });
 }
 
 async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc('global_status');
+    const docRef     = db.collection(collection).doc('global_status');
     const flattenUpdates = {};
     for (const [date, statuses] of Object.entries(updatesByDate)) {
         for (const [calc, status] of Object.entries(statuses)) {
@@ -172,14 +167,14 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     try {
         await docRef.update(flattenUpdates);
     } catch (err) {
-        if (err.code === 5) { 
-             const deepObj = {};
-             for (const [date, statuses] of Object.entries(updatesByDate)) {
-                 deepObj[date] = statuses;
-             }
-             await docRef.set(deepObj, { merge: true });
+        if (err.code === 5) {
+            const deepObj = {};
+            for (const [date, statuses] of Object.entries(updatesByDate)) {
+                deepObj[date] = statuses;
+            }
+            await docRef.set(deepObj, { merge: true });
         } else {
-             throw err;
+            throw err;
         }
     }
 }
@@ -188,7 +183,7 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
     for (const calc of calcsInPass) {
-        if (calc.dependencies)                { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
+        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
         if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
     }
     if (!calcsToFetch.size) return {};
@@ -197,23 +192,26 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     const names   = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                                .collection(config.computationsSubcollection).doc(name));
-                                names.push(name); } }
+        if (m) {
+            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                .collection(config.computationsSubcollection).doc(name));
+            names.push(name);
+        }
+    }
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => {  if(doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data();  } });
+        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
     }
     return fetched;
 }
 
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
-    const controller = new ComputationController(config, deps);
-    const calcs = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c => 
-        c.manifest.rootDataDependencies.includes('portfolio') || 
+    const controller     = new ComputationController(config, deps);
+    const calcs          = Object.values(state).filter(c => c && c.manifest);
+    const streamingCalcs = calcs.filter(c =>
+        c.manifest.rootDataDependencies.includes('portfolio') ||
         c.manifest.rootDataDependencies.includes('history')
     );
 
@@ -222,14 +220,14 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
 
     await controller.loader.loadMappings();
-    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
-    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
-    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
     let yP_chunk = {};
     let tH_chunk = {};
@@ -238,14 +236,14 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
         if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
 
-        const promises = streamingCalcs.map(calc => 
+        const promises = streamingCalcs.map(calc =>
             controller.executor.executePerUser(
                 calc,
                 calc.manifest,
                 dateStr,
                 tP_chunk,
-                yP_chunk, 
-                tH_chunk, 
+                yP_chunk,
+                tH_chunk,
                 fetchedDeps,
                 previousFetchedDeps
             )
@@ -260,20 +258,20 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     const logger = deps.logger;
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
-        const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
 
     const state = {};
     for (const c of calcs) {
-        try { 
-            const inst = new c.class(); 
-            inst.manifest = c; 
-            state[normalizeName(c.name)] = inst; 
+        try {
+            const inst = new c.class();
+            inst.manifest = c;
+            state[normalizeName(c.name)] = inst;
             logger.log('INFO', `${c.name} calculation running for ${dStr}`);
-        } 
-        catch(e)  { logger.log('WARN', `Failed to init ${c.name}`); }
+        }
+        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
@@ -282,8 +280,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
     const controller = new ComputationController(config, deps);
-    const dStr       = date.toISOString().slice(0, 10);
-    const state      = {};
+    const dStr = date.toISOString().slice(0, 10);
+    const state = {};
 
     for (const mCalc of calcs) {
         try {
@@ -297,28 +295,46 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
+/**
+ * --- REFACTORED: commitResults ---
+ * Commits results individually per calculation.
+ * If one calculation fails (e.g. size limit), others still succeed.
+ */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
-    const writes = [], schemas = [], sharded = {};
-    const successUpdates = {}; 
+    const successUpdates = {};
+    const schemas = [];
 
+    // Iterate PER CALCULATION to isolate failures
     for (const name in stateObj) {
         const calc = stateObj[name];
+        let hasData = false;
+
         try {
-            const result = await calc.getResult(); 
+            const result = await calc.getResult();
             if (!result) {
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Empty Result)`);
+                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
                 continue;
             }
-            
-            const standardRes = {};
-            let hasData = false;
 
+            const standardRes   = {};
+            const shardedWrites = [];
+            const calcWrites    = []; // Accumulate all writes for THIS specific calculation
+
+            // 1. Separate Standard and Sharded Data
             for (const key in result) {
                 if (key.startsWith('sharded_')) {
                     const sData = result[key];
-                    for (const c in sData) { 
-                        sharded[c] = sharded[c] || {}; 
-                        Object.assign(sharded[c], sData[c]); 
+                    // sData structure: { CollectionName: { DocId: { ...data } } }
+                    for (const colName in sData) {
+                        const docsMap = sData[colName];
+                        for (const docId in docsMap) {
+                            // Support both full path or collection-relative path
+                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
+                            shardedWrites.push({
+                                ref,
+                                data: { ...docsMap[docId], _completed: true }
+                            });
+                        }
                     }
                     if (Object.keys(sData).length > 0) hasData = true;
                 } else {
@@ -326,70 +342,75 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 }
             }
 
+            // 2. Prepare Standard Result Write
             if (Object.keys(standardRes).length) {
                 validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-
                 standardRes._completed = true;
-                writes.push({
-                    ref: deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                        .collection(config.computationsSubcollection).doc(name),
+
+                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                    .collection(config.computationsSubcollection).doc(name);
+
+                calcWrites.push({
+                    ref: docRef,
                     data: standardRes
                 });
                 hasData = true;
             }
-            
+
+            // 3. Queue Schema (Safe to accumulate)
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
-                schemas.push({ 
-                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata 
+                schemas.push({
+                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
                 });
             }
-            
+
+            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
             if (hasData) {
-                successUpdates[name] = true;
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Succeeded`);
+                // Combine standard + sharded writes for this unit of work
+                const allWritesForCalc = [...calcWrites, ...shardedWrites];
+
+                if (allWritesForCalc.length > 0) {
+                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
+
+                    // IF we get here, the commit succeeded.
+                    successUpdates[name] = true;
+                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
+                } else {
+                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
+                }
             } else {
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Unknown (No Data Written)`);
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
             }
 
-        } catch (e) { 
-            deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`);
-            deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Exception)`);
+        } catch (e) {
+            // CRITICAL: Catch errors here so the loop continues for other calculations
+            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
+            // Do NOT add to successUpdates
         }
     }
 
-    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
-    if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
-    for (const col in sharded) {
-        const sWrites = [];
-        for (const id in sharded[col]) {
-            const ref = id.includes('/') ? deps.db.doc(id) : deps.db.collection(col).doc(id);
-            sWrites.push({ ref, data: { ...sharded[col][id], _completed: true } });
-        }
-        if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
-    }
+    // Save Schemas (Best effort, isolated)
+    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
 
+    // Update Status Document (Only for the ones that succeeded)
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
-        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
     }
     return successUpdates;
 }
 
 /**
  * --- UPDATED: runBatchPriceComputation ---
- * Now supports subset/specific ticker execution via 'targetTickers'
- * OPTIMIZED: Implements concurrency for both Shard Processing and Write Commits
  */
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps; 
+    const { logger, db, calculationUtils } = deps;
     const controller = new ComputationController(config, deps);
-    
-    // 1. Call loadMappings() correctly and get the result
-    const mappings = await controller.loader.loadMappings(); 
-    
-    // 2. Resolve Shards (All or Subset)
+
+    const mappings = await controller.loader.loadMappings();
+
     let targetInstrumentIds = [];
     if (targetTickers && targetTickers.length > 0) {
         const tickerToInst = mappings.tickerToInstrument || {};
@@ -399,22 +420,20 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             return;
         }
     }
-    
+
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-    
+
     if (!allShardRefs.length) {
         logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
         return;
     }
 
-    // 3. Execution Planning
-    const OUTER_CONCURRENCY_LIMIT = 2; 
+    const OUTER_CONCURRENCY_LIMIT = 2;
     const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50; 
+    const WRITE_BATCH_LIMIT = 50;
 
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
 
-    // 4. Create Chunks of Shards
     const shardChunks = [];
     for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
         shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
@@ -422,114 +441,104 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
 
     const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
 
-    // 5. Process Shard Chunks Concurrently
-    const chunkPromises = shardChunks.map((shardChunkRefs, index) => outerLimit(async () => {
-        try {
-            logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-            
-            const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-            
-            // Optional Filtering for Subset Mode
-            if (targetInstrumentIds.length > 0) {
-                const requestedSet = new Set(targetInstrumentIds);
-                for (const loadedInstrumentId in pricesData) {
-                    if (!requestedSet.has(loadedInstrumentId)) {
-                        delete pricesData[loadedInstrumentId];
+    const chunkPromises = [];
+    for (let index = 0; index < shardChunks.length; index++) {
+        const shardChunkRefs = shardChunks[index];
+        chunkPromises.push(outerLimit(async () => {
+            try {
+                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
+
+                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+
+                if (targetInstrumentIds.length > 0) {
+                    const requestedSet = new Set(targetInstrumentIds);
+                    for (const loadedInstrumentId in pricesData) {
+                        if (!requestedSet.has(loadedInstrumentId)) {
+                            delete pricesData[loadedInstrumentId];
+                        }
                     }
                 }
-            }
-            const writes = [];
-            
-            // --- CALCULATION PHASE ---
-            for (const dateStr of dateStrings) {
-                // --- FIX 2: Manually map math primitives to their alias names ---
-                // This matches the ContextBuilder logic in ComputationController
-                // and fixes the "Cannot read properties of undefined (reading 'standardDeviation')" error.
-                const context = {
-                    mappings,
-                    prices: { history: pricesData },
-                    date: { today: dateStr },
-                    math: { 
-                        extract: DataExtractor, 
-                        history: HistoryExtractor, 
-                        compute: MathPrimitives, 
-                        aggregate: Aggregators, 
-                        validate: Validators, 
-                        signals: SignalPrimitives, 
-                        schemas: SCHEMAS, 
-                        distribution : DistributionAnalytics, 
-                        TimeSeries: TimeSeries, 
-                        priceExtractor : priceExtractor 
-                    }
-                };
-
-                for (const calcManifest of calcs) {
-                    try {
-                        // --- LOGGING FIX: Log start of calculation ---
-                        logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`);
-
-                        const instance = new calcManifest.class();
-                        await instance.process(context);
-                        const result = await instance.getResult();
-                        
-                        let hasContent = false;
-                        if (result && Object.keys(result).length > 0) {
-                            let dataToWrite = result;
-                            if (result.by_instrument) dataToWrite = result.by_instrument; 
-                            
-                            if (Object.keys(dataToWrite).length > 0) {
-                                hasContent = true;
-                                const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                    .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                    .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-                                
-                                writes.push({
-                                    ref: docRef,
-                                    data: { ...dataToWrite, _completed: true },
-                                    options: { merge: true }
-                                });
-                            }
-                        }
 
-                        // --- LOGGING FIX: Log success/completion ---
-                        if (hasContent) {
-                            logger.log('INFO', `[BatchPrice] \u2714 Finished ${calcManifest.name} for ${dateStr}. Found data.`);
-                        } else {
-                            logger.log('INFO', `[BatchPrice] - Finished ${calcManifest.name} for ${dateStr}. No result data.`);
+                // We now accumulate writes per calc to allow partial success, though batching optimization is tricky here.
+                // For safety, let's keep the existing structure but wrap individual calc processing in try/catch 
+                // inside the write phase if possible. 
+                // However, runBatchPrice is optimized for BULK throughput. 
+                // To prevent total failure, we will use a safe array.
+                const writes = [];
+
+                for (const dateStr of dateStrings) {
+                    const context = {
+                        mappings,
+                        prices: { history: pricesData },
+                        date: { today: dateStr },
+                        math: {
+                            extract: DataExtractor,
+                            history: HistoryExtractor,
+                            compute: MathPrimitives,
+                            aggregate: Aggregators,
+                            validate: Validators,
+                            signals: SignalPrimitives,
+                            schemas: SCHEMAS,
+                            distribution: DistributionAnalytics,
+                            TimeSeries: TimeSeries,
+                            priceExtractor: priceExtractor
+                        }
+                    };
+
+                    for (const calcManifest of calcs) {
+                        try {
+                            // logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`); // Verbose
+                            const instance = new calcManifest.class();
+                            await instance.process(context);
+                            const result = await instance.getResult();
+
+                            if (result && Object.keys(result).length > 0) {
+                                let dataToWrite = result;
+                                if (result.by_instrument) dataToWrite = result.by_instrument;
+
+                                if (Object.keys(dataToWrite).length > 0) {
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+
+                                    writes.push({
+                                        ref: docRef,
+                                        data: { ...dataToWrite, _completed: true },
+                                        options: { merge: true }
+                                    });
+                                }
+                            }
+                        } catch (err) {
+                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                         }
-
-                    } catch (err) {
-                        // --- LOGGING FIX: Explicit failure log ---
-                        logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                     }
                 }
-            }
-            
-            // --- PARALLEL COMMIT PHASE ---
-            if (writes.length > 0) {
-                const commitBatches = [];
-                for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
-                    commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
-                }
 
-                const commitLimit = pLimit(10); 
-                
-                await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                    const batch = db.batch();
-                    batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-                    
-                    try {
-                        await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                    } catch (commitErr) {
-                        logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                if (writes.length > 0) {
+                    const commitBatches = [];
+                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
+                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
                     }
-                })));
-            }
 
-        } catch (chunkErr) {
-            logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
-        }
-    }));
+                    const commitLimit = pLimit(10);
+
+                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
+                        const batch = db.batch();
+                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+
+                        try {
+                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
+                        } catch (commitErr) {
+                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                        }
+                    })));
+                }
+
+            } catch (chunkErr) {
+                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
+            }
+        }));
+    }
 
     await Promise.all(chunkPromises);
     logger.log('INFO', '[BatchPrice] Optimization pass complete.');
@@ -541,9 +550,9 @@ module.exports = {
     checkRootDataAvailability,
     fetchExistingResults,
     fetchComputationStatus,
-    fetchGlobalComputationStatus,   
-    updateComputationStatus, 
-    updateGlobalComputationStatus, 
+    fetchGlobalComputationStatus,
+    updateComputationStatus,
+    updateGlobalComputationStatus,
     runStandardComputationPass,
     runMetaComputationPass,
     runBatchPriceComputation

package/functions/computation-system/layers/math_primitives.js

@@ -95,8 +95,8 @@ class DataExtractor { // For generic access of data types
      * Extract Current Equity Value %.
      * Schema: 'Value' is the current value as a % of total portfolio equity.
      */
-    static getPositionValuePct(position) {            // TODO - VERIFY THIS WORKS FOR SPECULATORS,
-        return position ? (position.Value || 0) : 0; //  IS VALUE ACTUALLY THE VALUE OF POSITION AS A % OF TOTAL PORTFOLIO EQUITY? IS IT THE SAME FOR NORMAL USERS?
+    static getPositionValuePct(position) {            
+        return position ? (position.Value || 0) : 0; 
     }
 
     /**
@@ -262,39 +262,108 @@ class priceExtractor {
 }
 
 class HistoryExtractor {
-    // --- Schema Accessor (NEW) ---
+    // --- Schema Accessor (REFACTORED for Granular API) ---
     /**
      * Extracts the daily history snapshot from the User object.
-     * This decouples the computation from knowing 'user.history.today'.
+     * Returns the raw granular positions list.
      */
     static getDailyHistory(user) {
+        // The new API returns { PublicHistoryPositions: [...] }
         return user?.history?.today || null;
     }
 
-    // --- Data Extractors ---
+    // --- Data Extractors (ADAPTER PATTERN) ---
+    /**
+     * Adapts granular trade history into the legacy 'TradedAssets' format.
+     * Groups trades by InstrumentID and calculates average holding time.
+     */
     static getTradedAssets(historyDoc) {
-        if (!historyDoc || !Array.isArray(historyDoc.assets)) return [];
-        return historyDoc.assets;
+        const trades = historyDoc?.PublicHistoryPositions || [];
+        if (!trades.length) return [];
+        
+        // Group by InstrumentID
+        const assetsMap = new Map();
+        
+        for (const t of trades) {
+            const instId = t.InstrumentID;
+            if (!instId) continue;
+            
+            if (!assetsMap.has(instId)) {
+                assetsMap.set(instId, { 
+                    instrumentId: instId, 
+                    totalDuration: 0, 
+                    count: 0 
+                });
+            }
+            
+            const asset = assetsMap.get(instId);
+            
+            // Calculate Duration in Minutes
+            const open = new Date(t.OpenDateTime);
+            const close = new Date(t.CloseDateTime);
+            const durationMins = (close - open) / 60000; // ms -> min
+            
+            if (durationMins > 0) {
+                asset.totalDuration += durationMins;
+                asset.count++;
+            }
+        }
+        
+        // Convert Map to Array format expected by existing calculations
+        // (Returns objects with .instrumentId and .avgHoldingTimeInMinutes)
+        return Array.from(assetsMap.values()).map(a => ({
+            instrumentId: a.instrumentId,
+            avgHoldingTimeInMinutes: a.count > 0 ? (a.totalDuration / a.count) : 0
+        }));
     }
 
     static getInstrumentId(asset) { 
         return asset ? asset.instrumentId : null;
     }
 
-    static getAvgHoldingTimeMinutes(asset) { // Note, in minutes, we could convert values here into hours or days but we leave as-is for now.
+    static getAvgHoldingTimeMinutes(asset) { 
         return asset ? (asset.avgHoldingTimeInMinutes || 0) : 0;
     }
 
-    static getSummary(historyDoc) { // This returns the top-level summary of trade history
-        const all = historyDoc?.all;
-        if (!all) return null;
+    /**
+     * Adapts granular trade history into the legacy 'Summary' format.
+     * Calculates WinRatio, AvgProfit, etc. on the fly from the raw list.
+     */
+    static getSummary(historyDoc) { 
+        const trades = historyDoc?.PublicHistoryPositions || [];
+        if (!trades.length) return null;
+
+        let totalTrades = trades.length;
+        let wins = 0;
+        let totalProf = 0;
+        let totalLoss = 0;
+        let profCount = 0;
+        let lossCount = 0;
+        let totalDur = 0;
+
+        for (const t of trades) {
+            // P&L Stats (NetProfit is %)
+            if (t.NetProfit > 0) {
+                wins++;
+                totalProf += t.NetProfit;
+                profCount++;
+            } else if (t.NetProfit < 0) {
+                totalLoss += t.NetProfit;
+                lossCount++;
+            }
+            
+            // Duration Stats
+            const open = new Date(t.OpenDateTime);
+            const close = new Date(t.CloseDateTime);
+            totalDur += (close - open) / 60000; // ms -> min
+        }
 
-        return { // The all object contains instrumentid of -1 value, we do not include this, it's a junk backend-eToro placeholder.
-            totalTrades:             all.totalTrades             || 0,
-            winRatio:                all.winRatio                || 0,
-            avgProfitPct:            all.avgProfitPct            || 0,
-            avgLossPct:              all.avgLossPct              || 0,
-            avgHoldingTimeInMinutes: all.avgHoldingTimeInMinutes || 0
+        return { 
+            totalTrades:             totalTrades,
+            winRatio:                totalTrades > 0 ? (wins / totalTrades) * 100 : 0,
+            avgProfitPct:            profCount > 0 ? totalProf / profCount : 0,
+            avgLossPct:              lossCount > 0 ? totalLoss / lossCount : 0,
+            avgHoldingTimeInMinutes: totalTrades > 0 ? totalDur / totalTrades : 0
         };
     }
 }
@@ -354,7 +423,7 @@ class SignalPrimitives {
 
     static getPreviousState(previousComputed, calcName, ticker, fieldName = null) { // This is used for either fetching computations listed in getdependencies() OR self-history
         if (!previousComputed || !previousComputed[calcName]) return null;          // Using this for self-history DOES NOT cause a circular dependency because we assign a special rule in orchestration_helpers
-                                                                                    // Which handles the self-reference, see 2. SMART SELF-FETCH in orchestration_helpers
+                                                                                    // Which handles the self-reference.
         const tickerData = previousComputed[calcName][ticker];
         if (!tickerData) return null;
 
@@ -374,7 +443,7 @@ class MathPrimitives {
     static median(values) {
         if (!values || !values.length) return 0;
         const sorted = [...values].sort((a, b) => a - b);
-        const mid = Math.floor(sorted.length / 2);
+        const mid    = Math.floor(sorted.length / 2);
         return sorted.length % 2 === 0
             ? (sorted[mid - 1] + sorted[mid]) / 2
             : sorted[mid];
@@ -382,7 +451,7 @@ class MathPrimitives {
 
     static standardDeviation(values) {
         if (!values || !values.length) return 0;
-        const avg = this.average(values);
+        const avg         = this.average(values);
         const squareDiffs = values.map(val => Math.pow((val || 0) - avg, 2));
         return Math.sqrt(this.average(squareDiffs));
     }
@@ -398,7 +467,7 @@ class MathPrimitives {
      * Where: 
      * b = ln(Barrier/Price)
      * v = drift - 0.5 * volatility^2
-     * * @param {number} currentPrice - The current price of the asset
+     * @param {number} currentPrice - The current price of the asset
      * @param {number} barrierPrice - The target price (SL or TP)
      * @param {number} volatility - Annualized volatility (e.g., 0.40 for 40%)
      * @param {number} days - Number of days to forecast (e.g., 3)
@@ -408,9 +477,9 @@ class MathPrimitives {
     static calculateHitProbability(currentPrice, barrierPrice, volatility, days, drift = 0) { // https://www.ma.ic.ac.uk/~bin06/M3A22/m3f22chVII.pdf
         if (currentPrice <= 0 || barrierPrice <= 0 || volatility <= 0 || days <= 0) return 0;
 
-        const t = days / 365.0; // Convert days to years
+        const t     = days / 365.0; // Convert days to years
         const sigma = volatility;
-        const mu = drift;
+        const mu    = drift;
         
         // The barrier in log-space
         const b = Math.log(barrierPrice / currentPrice);
@@ -418,7 +487,7 @@ class MathPrimitives {
         // Adjusted drift (nu)
         const nu = mu - 0.5 * Math.pow(sigma, 2);
 
-        const sqrtT = Math.sqrt(t);
+        const sqrtT      = Math.sqrt(t);
         const sigmaSqrtT = sigma * sqrtT;
 
         // Helper for Standard Normal CDF (Φ)
@@ -448,8 +517,7 @@ class MathPrimitives {
 
         // Calculate Probability
         // Note: If nu is 0, the second term simplifies significantly, but we keep full form.
-        const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) + 
-                            Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
+        const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) +  Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
 
         return Math.min(Math.max(probability, 0), 1);
     }
@@ -468,11 +536,11 @@ class MathPrimitives {
     static simulateGBM(currentPrice, volatility, days, simulations = 1000, drift = 0) {
         if (currentPrice <= 0 || volatility <= 0 || days <= 0) return new Float32Array(0);
         
-        const t = days / 365.0;
-        const sigma = volatility;
-        const mu = drift;
+        const t         = days / 365.0;
+        const sigma     = volatility;
+        const mu        = drift;
         const driftTerm = (mu - 0.5 * sigma * sigma) * t;
-        const volTerm = sigma * Math.sqrt(t);
+        const volTerm   = sigma * Math.sqrt(t);
         
         // Use Float32Array for memory efficiency with large simulation counts
         const results = new Float32Array(simulations);
@@ -481,7 +549,7 @@ class MathPrimitives {
             // Box-Muller transform for efficient standard normal distribution generation
             const u1 = Math.random();
             const u2 = Math.random();
-            const z = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
+            const z  = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
             
             // GBM Formula: St = S0 * exp((mu - 0.5*sigma^2)t + sigma*Wt)
             results[i] = currentPrice * Math.exp(driftTerm + volTerm * z);
@@ -643,14 +711,14 @@ class TimeSeries {
         let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0, sumY2 = 0;
 
         for (let i = 0; i < n; i++) {
-            sumX += x[i];
-            sumY += y[i];
+            sumX  += x[i];
+            sumY  += y[i];
             sumXY += x[i] * y[i];
             sumX2 += x[i] * x[i];
             sumY2 += y[i] * y[i];
         }
 
-        const numerator = (n * sumXY) - (sumX * sumY);
+        const numerator   = (n * sumXY) - (sumX * sumY);
         const denominator = Math.sqrt(((n * sumX2) - (sumX * sumX)) * ((n * sumY2) - (sumY * sumY)));
 
         return (denominator === 0) ? 0 : numerator / denominator;
@@ -719,8 +787,8 @@ class DistributionAnalytics {
         
         let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0, sumYY = 0;
         for (let i = 0; i < n; i++) {
-            sumX += xValues[i];
-            sumY += yValues[i];
+            sumX  += xValues[i];
+            sumY  += yValues[i];
             sumXY += xValues[i] * yValues[i];
             sumXX += xValues[i] * xValues[i];
             sumYY += yValues[i] * yValues[i];

package/functions/task-engine/helpers/update_helpers.js

@@ -1,12 +1,12 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
+ * (OPTIMIZED V3: Removed obsolete username lookup logic)
  * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
- * (OPTIMIZED V2: Downgraded verbose per-user logs to TRACE to save costs)
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-const pLimit = require('p-limit');
+const crypto = require('crypto'); 
 
 // --- CIRCUIT BREAKER STATE ---
 // Persists across function invocations in the same instance.
@@ -28,7 +28,6 @@ function recordProxyOutcome(success) {
     if (success) {
         if (_consecutiveProxyFailures > 0) {
             // Optional: Only log recovery to reduce noise
-            // console.log('[ProxyCircuit] Proxy recovered.');
         }
         _consecutiveProxyFailures = 0;
     } else {
@@ -36,86 +35,11 @@ function recordProxyOutcome(success) {
     }
 }
 
-/**
- * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
- */
-async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
-    if (!cids?.length) return [];
-    logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
-    
-    // --- Set concurrency to 1 because appscript gets really fucked up with undocumented rate limits if we try spam it concurrently, a shame but that's life. DO NOT CHANGE THIS
-    const limit = pLimit(1);
-    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
-    const batches = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
-    
-    const batchPromises = batches.map((batch, index) => limit(async () => { 
-        const batchId = `batch-${index + 1}`;
-        logger.log('TRACE', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`); // DOWNGRADED TO TRACE
-        
-        const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
-        
-        let wasSuccess = false;
-        let proxyUsed = false;
-        let response;
-        const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
-        const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-        
-        // --- 1. Try Proxy (Circuit Breaker Protected) ---
-        if (shouldTryProxy()) {
-            try {
-                logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
-                response = await proxyManager.fetch(url, options);
-                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                
-                wasSuccess = true; 
-                proxyUsed = true;
-                recordProxyOutcome(true); // Reset failure count
-                logger.log('TRACE', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`); // DOWNGRADED TO TRACE
-
-            } catch (proxyError) { 
-                recordProxyOutcome(false); // Increment failure count
-                logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
-                // Fall through to direct...
-            }
-        } else {
-            logger.log('TRACE', `[lookupUsernames/${batchId}] Circuit Breaker Open. Skipping Proxy.`);
-        }
-
-        // --- 2. Direct Fallback ---
-        if (!wasSuccess) {
-            try { 
-                response = await fetch(url, options);  
-                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
-                logger.log('TRACE', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // DOWNGRADED TO TRACE
-                wasSuccess = true; // It worked eventually
-            } catch (fallbackError) { 
-                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); 
-                return null; // Give up on this batch
-            }
-        } 
-        
-        if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } 
-        
-        try { 
-            const data = await response.json(); return data;
-        } catch (parseError) { 
-            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; 
-        } 
-    }));
-
-    const results  = await Promise.allSettled(batchPromises);
-    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value); 
-    logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
-    return allUsers;
-}
-
-
 /**
  * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
  */
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config) {
+    // Note: 'username' param removed from signature as it is no longer needed.
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
@@ -137,7 +61,15 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             if (!historyHeader) { 
                 logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
-                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
+                
+                // --- REFACTOR: New Granular API Logic ---
+                // No username required. Uses CID (userId) directly.
+                const d = new Date();
+                d.setFullYear(d.getFullYear() - 1);
+                const oneYearAgoStr = d.toISOString();
+                const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; // Fallback for older nodes
+
+                const historyUrl = `https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat?StartTime=${oneYearAgoStr}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${userId}&client_request_id=${uuid}`;
                 const options = { headers: historyHeader.header };
                 let response;
 
@@ -278,4 +210,4 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); // DOWNGRADED TO TRACE
 }
 
-module.exports = { handleUpdate, lookupUsernames };
+module.exports = { handleUpdate };

package/functions/task-engine/utils/task_engine_utils.js

@@ -11,7 +11,7 @@
 
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
-const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+const { handleUpdate } = require('../helpers/update_helpers'); // Removed lookupUsernames import
 const pLimit = require('p-limit'); 
 
 /**
@@ -27,27 +27,25 @@ function parseTaskPayload(message, logger) {
 }
 
 /**
- * Sorts tasks into update (with username), lookup (missing username), and other (discover/verify).
+ * Sorts tasks into update and other (discover/verify).
+ * REFACTORED: Simplified. No username lookup logic needed.
  */
 async function prepareTaskBatches(tasks, batchManager, logger) {
-    const tasksToRun = [], cidsToLookup = new Map(), otherTasks = [];
-    await batchManager.loadUsernameMap();
+    const tasksToRun = [], otherTasks = [];
+    
     for (const task of tasks) {
-        if (task.type === 'update') { const username = batchManager.getUsername(task.userId); username ? tasksToRun.push({ task, username }) : cidsToLookup.set(String(task.userId), task); } else otherTasks.push(task); }
-    logger.log('INFO', `[TaskEngine] Sorting complete. Known: ${tasksToRun.length}, Lookup: ${cidsToLookup.size}, Other: ${otherTasks.length}`);
-    return { tasksToRun, cidsToLookup, otherTasks };
-}
-
-/**
- * Runs username lookups for missing CIDs and adds to tasksToRun.
- */
-async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger) {
-    if (!cidsToLookup.size) return;
-    logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
-    // Pass config to lookupUsernames
-    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config); // <--- PASS FULL CONFIG
-    for (const u of foundUsers) { const cid = String(u.CID), username = u.Value.UserName; batchManager.addUsernameMapUpdate(cid, username); const task = cidsToLookup.get(cid); if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); } }
-    if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
+        if (task.type === 'update') { 
+            // New API uses CID (userId), so we push directly to run.
+            tasksToRun.push(task); 
+        } else {
+            otherTasks.push(task); 
+        }
+    }
+    
+    // We explicitly return empty structures for compatibility if handler_creator expects them,
+    // though ideally handler_creator should also be simplified.
+    // For now, we return compatible object structure.
+    return { tasksToRun, cidsToLookup: new Map(), otherTasks };
 }
 
 /**
@@ -88,11 +86,16 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
     }
 
     // 2. Queue 'update' tasks
-    for (const { task, username } of tasksToRun) {
+    for (const task of tasksToRun) {
+        // We unpack 'task' directly now, no wrapping object {task, username}
+        // However, we must ensure backward compatibility if the array was {task, username} before.
+        // In prepareTaskBatches above, we pushed raw 'task'. 
+        // So we use 'task' directly.
+        
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
         allTaskPromises.push(
             limit(() => 
-                handleUpdate(task, subTaskId, dependencies, config, username) 
+                handleUpdate(task, subTaskId, dependencies, config) 
                     .then(() => taskCounters.update++)
                     .catch(err => {
                         logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
@@ -112,4 +115,5 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
     );
 }
 
-module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };
+// Note: runUsernameLookups removed from exports
+module.exports = { parseTaskPayload, prepareTaskBatches, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.204",
+  "version": "1.0.206",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [