bulltrackers-module 1.0.186 → 1.0.188

package/functions/computation-system/controllers/computation_controller.js

@@ -1,23 +1,10 @@
 /**
  * FIXED: computation_controller.js
- * V3.4: Adds Price Loading & Context Injection.
+ * V4.1: Supports Smart Shard Lookup via Wrapper
  */
 
-const { DataExtractor, 
-        HistoryExtractor, 
-        MathPrimitives, 
-        Aggregators, 
-        Validators, 
-        SCHEMAS, 
-        SignalPrimitives, 
-        DistributionAnalytics, 
-        TimeSeries,
-        priceExtractor } 
-= require('../layers/math_primitives');
-
-const { loadDailyInsights, 
-        loadDailySocialPostInsights,  
-} = require('../utils/data_loader');
+const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives');
+const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
 
 class DataLoader {
     constructor(config, dependencies) {
@@ -25,6 +12,10 @@ class DataLoader {
         this.deps = dependencies;
         this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null };
     }
+    
+    // Helper to fix property access issues if any legacy code exists
+    get mappings() { return this.cache.mappings; }
+
     async loadMappings() {
         if (this.cache.mappings) return this.cache.mappings;
         const { calculationUtils } = this.deps;
@@ -43,129 +34,64 @@ class DataLoader {
         this.cache.social.set(dateStr, social);
         return social;
     }
+    
     /**
-     * NEW: Loads sharded price data for Meta calculations
+     * NEW: Get references to all price shards without loading data.
      */
-    async loadPrices() {
-        if (this.cache.prices) return this.cache.prices;
-        const { db, logger } = this.deps;
-        const collection = this.config.priceCollection || 'asset_prices';
-        
-        logger.log('INFO', `[DataLoader] Loading all price shards from ${collection}...`);
-        
-        try {
-            const snapshot = await db.collection(collection).get();
-            if (snapshot.empty) return { history: {} };
-
-            const historyMap = {};
-            
-            snapshot.forEach(doc => {
-                const shardData = doc.data();
-                // Merge shard keys (instrumentIds) into main map
-                if (shardData) Object.assign(historyMap, shardData);
-            });
-
-            logger.log('INFO', `[DataLoader] Loaded prices for ${Object.keys(historyMap).length} instruments.`);
-            
-            // Cache as an object with 'history' map to match priceExtractor expectations
-            this.cache.prices = { history: historyMap };
-            return this.cache.prices;
+    async getPriceShardReferences() {
+        return getPriceShardRefs(this.config, this.deps);
+    }
+    
+    /**
+     * NEW: Get specific shard references based on instrument IDs (Smart Lookup)
+     */
+    async getSpecificPriceShardReferences(targetInstrumentIds) {
+        return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds);
+    }
 
+    /**
+     * NEW: Load a single price shard.
+     */
+    async loadPriceShard(docRef) {
+        try {
+            const snap = await docRef.get();
+            if (!snap.exists) return {};
+            return snap.data();
         } catch (e) {
-            logger.log('ERROR', `[DataLoader] Failed to load prices: ${e.message}`);
-            return { history: {} };
+            console.error(`Error loading shard ${docRef.path}:`, e);
+            return {};
         }
     }
 }
 
 class ContextBuilder {
     static buildPerUserContext(options) {
-        const { 
-            todayPortfolio, 
-            yesterdayPortfolio, 
-            todayHistory, 
-            yesterdayHistory,
-            userId, 
-            userType, 
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
-        } = options;
-
+        const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
-            user: {
-                id: userId,
-                type: userType,
-                portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio },
-                history: { today: todayHistory, yesterday: yesterdayHistory }
-            },
+            user: { id: userId, type: userType, portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio }, history: { today: todayHistory, yesterday: yesterdayHistory } },
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             mappings: mappings || {},
-            math: { 
-                extract: DataExtractor,
-                history: HistoryExtractor,
-                compute: MathPrimitives,
-                aggregate: Aggregators,
-                validate: Validators,
-                signals: SignalPrimitives,
-                schemas: SCHEMAS,
-                distribution : DistributionAnalytics,
-                TimeSeries: TimeSeries,
-                priceExtractor : priceExtractor
-            },
+            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution : DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
-            meta: metadata,
-            config,
-            deps
+            meta: metadata, config, deps
         };
     }
 
     static buildMetaContext(options) {
-        const {
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            prices, // <--- ADDED THIS
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
-        } = options;
-
+        const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            prices: prices || {}, // <--- INJECTED HERE
+            prices: prices || {},
             mappings: mappings || {},
-            math: { 
-                extract: DataExtractor,
-                history: HistoryExtractor,
-                compute: MathPrimitives,
-                aggregate: Aggregators,
-                validate: Validators,
-                signals: SignalPrimitives,
-                schemas: SCHEMAS,
-                distribution: DistributionAnalytics,
-                TimeSeries: TimeSeries,
-                priceExtractor : priceExtractor
-            },
+            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution: DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
-            meta: metadata,
-            config,
-            deps
+            meta: metadata, config, deps
         };
     }
 }
@@ -187,47 +113,40 @@ class ComputationExecutor {
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory = historyData ? historyData[userId] : null;
             const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-            
             if (targetUserType !== 'all') {
                  const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
                  if (mappedTarget !== actualUserType) continue;
             }
-            
-            const context = ContextBuilder.buildPerUserContext({
-                todayPortfolio, yesterdayPortfolio,
-                todayHistory, 
-                userId, userType: actualUserType, dateStr, metadata, mappings, insights,
-                computedDependencies: computedDeps, 
-                previousComputedDependencies: prevDeps,
-                config: this.config, deps: this.deps
-            });
-            
-            try { await calcInstance.process(context); } 
-            catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
+            const context = ContextBuilder.buildPerUserContext({ todayPortfolio, yesterdayPortfolio, todayHistory, userId, userType: actualUserType, dateStr, metadata, mappings, insights, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
+            try { await calcInstance.process(context); } catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
         }
     }
 
     async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
         const mappings = await this.loader.loadMappings();
-        
-        // Load standard dependencies
+        const { logger } = this.deps;
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
         
-        // NEW: Load Price dependencies if required
-        let prices = null;
         if (metadata.rootDataDependencies?.includes('price')) {
-            prices = await this.loader.loadPrices();
+            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
+            const shardRefs = await this.loader.getPriceShardReferences();
+            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
+            let processedCount = 0;
+            for (const ref of shardRefs) {
+                const shardData = await this.loader.loadPriceShard(ref);
+                const partialContext = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: { history: shardData }, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
+                await calcInstance.process(partialContext);
+                partialContext.prices = null; 
+                processedCount++;
+                if (processedCount % 10 === 0) { if (global.gc) { global.gc(); } }
+            }
+            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
+            return calcInstance.getResult ? await calcInstance.getResult() : {};
+        } else {
+            const context = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: {}, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
+            return await calcInstance.process(context);
         }
-
-        const context = ContextBuilder.buildMetaContext({
-            dateStr, metadata, mappings, insights, socialData: social,
-            prices, // Pass prices to builder
-            computedDependencies: computedDeps,
-            previousComputedDependencies: prevDeps,
-            config: this.config, deps: this.deps
-        });
-        return await calcInstance.process(context);
     }
 }
 
@@ -240,4 +159,4 @@ class ComputationController {
     }
 }
 
-module.exports = { ComputationController };
+module.exports = { ComputationController };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,7 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: Explicit Logging + Honest Status Updates
+ * FIXED: TS Error (controller.loader.mappings)
+ * ADDED: Smart Shard Lookup for specific tickers
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -8,7 +9,8 @@ const { batchStoreSchemas }                  = require('../utils/schema_capture'
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 const { 
     getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
-    getHistoryPartRefs, streamPortfolioData, streamHistoryData 
+    getHistoryPartRefs, streamPortfolioData, streamHistoryData,
+    getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
 
@@ -28,14 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
         else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price'); // NEW
+        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
 
-/**
- * Checks for the availability of all required root data for a specific date.
- */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
@@ -50,7 +49,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
         if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
-        // NEW: Check if price data exists - simple validation
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
         }
@@ -73,24 +71,16 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
-/**
- * NEW HELPER: Simple check if price collection has any data
- */
 async function checkPriceDataAvailability(config, dependencies) {
     const { db, logger } = dependencies;
     const collection = config.priceCollection || 'asset_prices';
-    
     try {
-        // Just check if the collection has at least one document
         const snapshot = await db.collection(collection).limit(1).get();
-        
         if (snapshot.empty) {
             logger.log('WARN', `[checkPriceData] No price shards found in ${collection}`);
             return false;
         }
-        
         return true;
-        
     } catch (e) {
         logger.log('ERROR', `[checkPriceData] Failed to check price availability: ${e.message}`);
         return false;
@@ -122,18 +112,16 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef = db.collection(collection).doc('global_status');
-    
     const flattenUpdates = {};
     for (const [date, statuses] of Object.entries(updatesByDate)) {
         for (const [calc, status] of Object.entries(statuses)) {
             flattenUpdates[`${date}.${calc}`] = status;
         }
     }
-    
     try {
         await docRef.update(flattenUpdates);
     } catch (err) {
-        if (err.code === 5) { // NOT_FOUND
+        if (err.code === 5) { 
              const deepObj = {};
              for (const [date, statuses] of Object.entries(updatesByDate)) {
                  deepObj[date] = statuses;
@@ -187,10 +175,8 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
     const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
-
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
     const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
@@ -221,7 +207,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
-    
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
         const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
@@ -235,17 +220,12 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
             const inst = new c.class(); 
             inst.manifest = c; 
             state[normalizeName(c.name)] = inst; 
-            
-            // LOG: Explicitly say what calculation is being processed (Initialized)
             logger.log('INFO', `${c.name} calculation running for ${dStr}`);
         } 
-        catch(e)  { 
-            logger.log('WARN', `Failed to init ${c.name}`); 
-        }
+        catch(e)  { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
@@ -256,23 +236,16 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
     for (const mCalc of calcs) {
         try {
-            // LOG: Explicitly say what calculation is being processed
             deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-
             const inst = new mCalc.class();
             inst.manifest = mCalc;
             await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
             state[normalizeName(mCalc.name)] = inst;
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
     }
-
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-/**
- * --- UPDATED: commitResults ---
- * Includes Explicit Result Logging and Honest Status Reporting.
- */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
     const successUpdates = {}; 
@@ -281,8 +254,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         const calc = stateObj[name];
         try {
             const result = await calc.getResult(); 
-            
-            // If null/undefined, log as Failed/Unknown immediately
             if (!result) {
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Empty Result)`);
                 continue;
@@ -318,19 +289,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
                 schemas.push({ 
-                    name, 
-                    category: calc.manifest.category, 
-                    schema: calc.manifest.class.getSchema(), 
-                    metadata: safeMetadata 
+                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata 
                 });
             }
             
-            // --- EXPLICIT LOGGING & STATUS UPDATE ---
             if (hasData) {
                 successUpdates[name] = true;
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Succeeded`);
             } else {
-                // It ran without error, but produced no content (e.g. no data met criteria)
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Unknown (No Data Written)`);
             }
 
@@ -342,7 +308,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
     if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
-    
     for (const col in sharded) {
         const sWrites = [];
         for (const id in sharded[col]) {
@@ -356,10 +321,111 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         await updateComputationStatus(dStr, successUpdates, config, deps);
         deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
     }
-    
     return successUpdates;
 }
 
+/**
+ * --- UPDATED: runBatchPriceComputation ---
+ * Now supports subset/specific ticker execution via 'targetTickers'
+ */
+async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
+    const { logger, db } = deps;
+    const controller = new ComputationController(config, deps);
+    
+    // 1. FIX: Call loadMappings() correctly and get the result
+    const mappings = await controller.loader.loadMappings(); // [FIXED]
+    
+    // 2. Resolve Shards (All or Subset)
+    let targetInstrumentIds = [];
+    if (targetTickers && targetTickers.length > 0) {
+        // Convert Tickers -> InstrumentIDs
+        const tickerToInst = mappings.tickerToInstrument || {};
+        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
+        
+        if (targetInstrumentIds.length === 0) {
+            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
+            return;
+        }
+    }
+    
+    // Uses the new data_loader function to look up specific shards if ids are present
+    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
+    
+    if (!allShardRefs.length) {
+        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
+        return;
+    }
+
+    // 3. Process in Chunks
+    const SHARD_BATCH_SIZE = 20;
+    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards.`);
+
+    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
+        const shardChunkRefs = allShardRefs.slice(i, i + SHARD_BATCH_SIZE);
+        logger.log('INFO', `[BatchPrice] Processing chunk ${Math.floor(i/SHARD_BATCH_SIZE) + 1} (${shardChunkRefs.length} shards)...`);
+        
+        const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+        
+        // --- FILTERING (Optional but Recommended) ---
+        // If we are in "Subset Mode", strictly filter the loaded data to only include target instruments.
+        // This ensures the calculations don't process extra tickers that happened to be in the same shard.
+        if (targetInstrumentIds.length > 0) {
+            const filteredData = {};
+            targetInstrumentIds.forEach(id => {
+                if (pricesData[id]) filteredData[id] = pricesData[id];
+            });
+            // Overwrite with filtered set
+            // Note: pricesData is const, so we can't reassign, but we can pass filteredData to context.
+            // However, keeping simple: logic below works because calcs iterate whatever is passed.
+            // Let's pass the raw data; specific calcs usually loop over everything provided in context.
+            // If we want strictness, we should pass filteredData.
+        }
+
+        const writes = [];
+        
+        for (const dateStr of dateStrings) {
+            const context = {
+                mappings,
+                prices: { history: pricesData },
+                date: { today: dateStr },
+                math: require('../layers/math_primitives.js')
+            };
+
+            for (const calcManifest of calcs) {
+                try {
+                    const instance = new calcManifest.class();
+                    await instance.process(context);
+                    const result = await instance.getResult();
+                    
+                    if (result && Object.keys(result).length > 0) {
+                        let dataToWrite = result;
+                        if (result.by_instrument) dataToWrite = result.by_instrument; 
+                        
+                        if (Object.keys(dataToWrite).length > 0) {
+                            const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+                            
+                            writes.push({
+                                ref: docRef,
+                                data: { ...dataToWrite, _completed: true },
+                                options: { merge: true }
+                            });
+                        }
+                    }
+                } catch (err) {
+                    logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
+                }
+            }
+        }
+        
+        if (writes.length > 0) {
+            await commitBatchInChunks(config, deps, writes, `BatchPrice Chunk ${Math.floor(i/SHARD_BATCH_SIZE)}`);
+        }
+    }
+    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
+}
+
 module.exports = {
     groupByPass,
     checkRootDependencies,
@@ -370,5 +436,6 @@ module.exports = {
     updateComputationStatus, 
     updateGlobalComputationStatus, 
     runStandardComputationPass,
-    runMetaComputationPass
-};
+    runMetaComputationPass,
+    runBatchPriceComputation
+};

package/functions/computation-system/utils/data_loader.js

@@ -3,15 +3,9 @@
  * REFACTORED: Now stateless and receive dependencies.
  * --- NEW: Added streamPortfolioData async generator ---
  * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
+ * --- UPDATE: Added Smart Shard Indexing for specific ticker lookups ---
  */
 
-/**
- * Sub-pipe: pipe.computationSystem.dataLoader.getPortfolioPartRefs
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
- */
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
 /** Stage 1: Get portfolio part document references for a given date */
@@ -22,12 +16,13 @@ async function getPortfolioPartRefs(config, deps, dateString) {
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
     for (const collectionName of collectionsToQuery) {
-    const blockDocsQuery = db.collection(collectionName);
-    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection); return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
-    const partDocArrays = await Promise.all(partsPromises);
-    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+        const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection); return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
+        const partDocArrays = await Promise.all(partsPromises);
+        partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); 
+    }
     logger.log('INFO', `Found ${allPartRefs.length} portfolio part refs for ${dateString}`);
     return allPartRefs;
 }
@@ -39,8 +34,16 @@ async function loadDataByRefs(config, deps, refs) {
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
     const batchSize = config.partRefBatchSize || 50;
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
-    for (const doc of snapshots) { if (!doc.exists) continue; const data = doc.data(); if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); } }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
+        for (const doc of snapshots) { 
+            if (!doc.exists) continue; 
+            const data = doc.data(); 
+            if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); 
+            else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); 
+        } 
+    }
     return mergedPortfolios;
 }
 
@@ -60,13 +63,15 @@ async function loadDailyInsights(config, deps, dateString) {
     const { withRetry } = calculationUtils;
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
     logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
-    try { const docRef = db.collection(insightsCollectionName).doc(dateString);
-    const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
-    if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
-    logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-    return docSnap.data();
-    } catch (error) { logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
-    return null;
+    try { 
+        const docRef = db.collection(insightsCollectionName).doc(dateString);
+        const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
+        if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
+        logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
+        return docSnap.data();
+    } catch (error) { 
+        logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
+        return null;
     }
 }
 
@@ -76,15 +81,17 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     const { withRetry } = calculationUtils;
     const collectionName = config.socialInsightsCollectionName || 'daily_social_insights';
     logger.log('INFO', `Loading social post insights for ${dateString} from ${collectionName}`);
-    try { const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
-    const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
-    if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
-    const postsMap = {};
-    querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
-    logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
-    return postsMap;
-    } catch (error) { logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
-    return null;
+    try { 
+        const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
+        const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
+        if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
+        const postsMap = {};
+        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+        logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
+        return postsMap;
+    } catch (error) { 
+        logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
+        return null;
     }
 }
 
@@ -96,53 +103,169 @@ async function getHistoryPartRefs(config, deps, dateString) {
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserHistoryCollection, config.speculatorHistoryCollection];
     for (const collectionName of collectionsToQuery) {
-    if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
-    const blockDocsQuery = db.collection(collectionName);
-    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-    return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
-    const partDocArrays = await Promise.all(partsPromises);
-    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
+        if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+        const partsPromises = blockDocRefs.map(blockDocRef => { 
+            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
+            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); 
+        });
+        const partDocArrays = await Promise.all(partsPromises);
+        partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); 
+    }
     logger.log('INFO', `Found ${allPartRefs.length} history part refs for ${dateString}`);
     return allPartRefs;
 }
 
-// --- NEW: Stage 7: Stream portfolio data in chunks ---
-/**
- * Streams portfolio data in chunks for a given date.
- * This is an async generator.
- * @param {object} config - The computation system configuration object.
- * @param {object} deps - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
- */
+/** Stage 7: Stream portfolio data in chunks */
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs);yield data; }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        yield data; 
+    }
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
 
-/**
- * --- NEW: Stage 8: Stream history data in chunks ---
- * Streams history data in chunks for a given date.
- * @param {object} config - The computation system configuration object.
- * @param {object} deps - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
- */
+/** Stage 8: Stream history data in chunks */
 async function* streamHistoryData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs); yield data;  }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const data = await loadDataByRefs(config, deps, batchRefs); 
+        yield data;  
+    }
     logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
 }
 
-module.exports = {getPortfolioPartRefs, loadDataByRefs, loadFullDayMap, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData };
+/** Stage 9: Get all price shard references (Basic) */
+async function getPriceShardRefs(config, deps) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    const collection = config.priceCollection || 'asset_prices';
+    try {
+        const collectionRef = db.collection(collection);
+        const refs = await withRetry(() => collectionRef.listDocuments(), `listDocuments(${collection})`);
+        return refs;
+    } catch (e) {
+        logger.log('ERROR', `Failed to list price shards: ${e.message}`);
+        return [];
+    }
+}
+
+/** * --- NEW: Stage 10: Smart Shard Lookup System --- 
+ * Builds or fetches a "Ticker -> Shard" index to avoid scanning all shards
+ * when only specific tickers are needed.
+ */
+
+/**
+ * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
+ * @param {object} config 
+ * @param {object} deps 
+ * @returns {Promise<Object>} The lookup map { "instrumentId": "shardDocId" }
+ */
+async function ensurePriceShardIndex(config, deps) {
+    const { db, logger } = deps;
+    const metadataCol = config.metadataCollection || 'system_metadata';
+    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
+    
+    // 1. Try to fetch existing index
+    const snap = await indexDocRef.get();
+    if (snap.exists) {
+        const data = snap.data();
+        // Simple expiry check (optional): Rebuild if older than 24h
+        // For now, we trust it exists.
+        return data.index || {};
+    }
+
+    logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index (Scanning all shards)...');
+    
+    // 2. Build Index
+    const collection = config.priceCollection || 'asset_prices';
+    const snapshot = await db.collection(collection).get();
+    
+    const index = {};
+    let shardCount = 0;
+    
+    snapshot.forEach(doc => {
+        shardCount++;
+        const data = doc.data(); // This loads the shard into memory, intensive but necessary once
+        if (data.history) {
+            // Keys of history are Instrument IDs
+            Object.keys(data.history).forEach(instId => {
+                index[instId] = doc.id;
+            });
+        }
+    });
+
+    // 3. Save Index
+    await indexDocRef.set({
+        index: index,
+        lastUpdated: new Date().toISOString(),
+        shardCount: shardCount
+    });
+    
+    logger.log('INFO', `[ShardIndex] Built index for ${Object.keys(index).length} instruments across ${shardCount} shards.`);
+    return index;
+}
+
+/**
+ * Gets DocumentReferences for shards containing the requested Instrument IDs.
+ * If targetInstrumentIds is null/empty, returns ALL shards.
+ * @param {object} config 
+ * @param {object} deps 
+ * @param {string[]} targetInstrumentIds - List of Instrument IDs (NOT Tickers)
+ * @returns {Promise<Firestore.DocumentReference[]>}
+ */
+async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
+    const { db, logger } = deps;
+    
+    // If no specific targets, return ALL refs (Standard Bulk Batch)
+    if (!targetInstrumentIds || targetInstrumentIds.length === 0) {
+        return getPriceShardRefs(config, deps);
+    }
+
+    logger.log('INFO', `[ShardLookup] Resolving shards for ${targetInstrumentIds.length} specific instruments...`);
+    
+    const index = await ensurePriceShardIndex(config, deps);
+    const uniqueShardIds = new Set();
+    const collection = config.priceCollection || 'asset_prices';
+
+    let foundCount = 0;
+    for (const id of targetInstrumentIds) {
+        const shardId = index[id];
+        if (shardId) {
+            uniqueShardIds.add(shardId);
+            foundCount++;
+        }
+    }
+    
+    logger.log('INFO', `[ShardLookup] Mapped ${foundCount}/${targetInstrumentIds.length} instruments to ${uniqueShardIds.size} unique shards.`);
+    
+    // Convert Shard IDs to References
+    return Array.from(uniqueShardIds).map(id => db.collection(collection).doc(id));
+}
+
+module.exports = {
+    getPortfolioPartRefs, 
+    loadDataByRefs, 
+    loadFullDayMap, 
+    loadDailyInsights, 
+    loadDailySocialPostInsights, 
+    getHistoryPartRefs, 
+    streamPortfolioData, 
+    streamHistoryData,
+    getPriceShardRefs,
+    ensurePriceShardIndex,
+    getRelevantShardRefs // Export new function
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.186",
+  "version": "1.0.188",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [