bulltrackers-module 1.0.187 → 1.0.189

package/functions/computation-system/controllers/computation_controller.js

@@ -1,23 +1,10 @@
 /**
  * FIXED: computation_controller.js
- * V4.0: Implements Batch/Sharded Execution for Price Dependencies to prevent OOM.
+ * V4.1: Supports Smart Shard Lookup via Wrapper
  */
 
-const { DataExtractor, 
-        HistoryExtractor, 
-        MathPrimitives, 
-        Aggregators, 
-        Validators, 
-        SCHEMAS, 
-        SignalPrimitives, 
-        DistributionAnalytics, 
-        TimeSeries,
-        priceExtractor } 
-= require('../layers/math_primitives');
-
-const { loadDailyInsights, 
-        loadDailySocialPostInsights,  
-} = require('../utils/data_loader');
+const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives');
+const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
 
 class DataLoader {
     constructor(config, dependencies) {
@@ -25,6 +12,10 @@ class DataLoader {
         this.deps = dependencies;
         this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null };
     }
+    
+    // Helper to fix property access issues if any legacy code exists
+    get mappings() { return this.cache.mappings; }
+
     async loadMappings() {
         if (this.cache.mappings) return this.cache.mappings;
         const { calculationUtils } = this.deps;
@@ -48,16 +39,14 @@ class DataLoader {
      * NEW: Get references to all price shards without loading data.
      */
     async getPriceShardReferences() {
-        const { db, logger } = this.deps;
-        const collection = this.config.priceCollection || 'asset_prices';
-        try {
-            const refs = await db.collection(collection).listDocuments();
-            logger.log('INFO', `[DataLoader] Found ${refs.length} price shards to process.`);
-            return refs;
-        } catch (e) {
-            logger.log('ERROR', `[DataLoader] Failed to list price shards: ${e.message}`);
-            return [];
-        }
+        return getPriceShardRefs(this.config, this.deps);
+    }
+    
+    /**
+     * NEW: Get specific shard references based on instrument IDs (Smart Lookup)
+     */
+    async getSpecificPriceShardReferences(targetInstrumentIds) {
+        return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds);
     }
 
     /**
@@ -73,98 +62,36 @@ class DataLoader {
             return {};
         }
     }
-
-    /**
-     * LEGACY: Kept for non-batched calls if ever needed, but effectively replaced by batching.
-     */
-    async loadPrices() {
-        const { db, logger } = this.deps;
-        const collection = this.config.priceCollection || 'asset_prices';
-        logger.log('WARN', `[DataLoader] loadPrices() called. This is memory intensive!`);
-        try {
-            const snapshot = await db.collection(collection).get();
-            if (snapshot.empty) return { history: {} };
-            const historyMap = {};
-            snapshot.forEach(doc => {
-                const shardData = doc.data();
-                if (shardData) Object.assign(historyMap, shardData);
-            });
-            return { history: historyMap };
-        } catch (e) {
-            return { history: {} };
-        }
-    }
 }
 
 class ContextBuilder {
     static buildPerUserContext(options) {
-        const { 
-            todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory,
-            userId, userType, dateStr, metadata, mappings, insights, socialData,
-            computedDependencies, previousComputedDependencies, config, deps
-        } = options;
-
+        const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
-            user: {
-                id: userId,
-                type: userType,
-                portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio },
-                history: { today: todayHistory, yesterday: yesterdayHistory }
-            },
+            user: { id: userId, type: userType, portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio }, history: { today: todayHistory, yesterday: yesterdayHistory } },
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             mappings: mappings || {},
-            math: { 
-                extract: DataExtractor,
-                history: HistoryExtractor,
-                compute: MathPrimitives,
-                aggregate: Aggregators,
-                validate: Validators,
-                signals: SignalPrimitives,
-                schemas: SCHEMAS,
-                distribution : DistributionAnalytics,
-                TimeSeries: TimeSeries,
-                priceExtractor : priceExtractor
-            },
+            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution : DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
-            meta: metadata,
-            config,
-            deps
+            meta: metadata, config, deps
         };
     }
 
     static buildMetaContext(options) {
-        const {
-            dateStr, metadata, mappings, insights, socialData,
-            prices, 
-            computedDependencies, previousComputedDependencies, config, deps
-        } = options;
-
+        const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             prices: prices || {},
             mappings: mappings || {},
-            math: { 
-                extract: DataExtractor,
-                history: HistoryExtractor,
-                compute: MathPrimitives,
-                aggregate: Aggregators,
-                validate: Validators,
-                signals: SignalPrimitives,
-                schemas: SCHEMAS,
-                distribution: DistributionAnalytics,
-                TimeSeries: TimeSeries,
-                priceExtractor : priceExtractor
-            },
+            math: { extract: DataExtractor, history: HistoryExtractor, compute: MathPrimitives, aggregate: Aggregators, validate: Validators, signals: SignalPrimitives, schemas: SCHEMAS, distribution: DistributionAnalytics, TimeSeries: TimeSeries, priceExtractor : priceExtractor },
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
-            meta: metadata,
-            config,
-            deps
+            meta: metadata, config, deps
         };
     }
 }
@@ -186,88 +113,38 @@ class ComputationExecutor {
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory = historyData ? historyData[userId] : null;
             const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-            
             if (targetUserType !== 'all') {
                  const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
                  if (mappedTarget !== actualUserType) continue;
             }
-            
-            const context = ContextBuilder.buildPerUserContext({
-                todayPortfolio, yesterdayPortfolio,
-                todayHistory, 
-                userId, userType: actualUserType, dateStr, metadata, mappings, insights,
-                computedDependencies: computedDeps, 
-                previousComputedDependencies: prevDeps,
-                config: this.config, deps: this.deps
-            });
-            
-            try { await calcInstance.process(context); } 
-            catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
+            const context = ContextBuilder.buildPerUserContext({ todayPortfolio, yesterdayPortfolio, todayHistory, userId, userType: actualUserType, dateStr, metadata, mappings, insights, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
+            try { await calcInstance.process(context); } catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
         }
     }
 
-    /**
-     * REFACTORED: Batched execution for Price dependencies
-     */
     async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
         const mappings = await this.loader.loadMappings();
         const { logger } = this.deps;
-        
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
         
-        // CHECK: Does this calculation require price history?
         if (metadata.rootDataDependencies?.includes('price')) {
             logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
-            
-            // 1. Get Shard References (Low Memory)
             const shardRefs = await this.loader.getPriceShardReferences();
-            
-            if (shardRefs.length === 0) {
-                 logger.log('WARN', '[Executor] No price shards found.');
-                 return {};
-            }
-
-            // 2. Iterate Shards (One at a time in memory)
+            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
             let processedCount = 0;
             for (const ref of shardRefs) {
-                // Load ONE shard
                 const shardData = await this.loader.loadPriceShard(ref);
-                
-                // Construct a "Partial Context" containing only this shard's prices
-                const partialContext = ContextBuilder.buildMetaContext({
-                    dateStr, metadata, mappings, insights, socialData: social,
-                    prices: { history: shardData }, // Inject partial history
-                    computedDependencies: computedDeps,
-                    previousComputedDependencies: prevDeps,
-                    config: this.config, deps: this.deps
-                });
-
-                // Process the chunk
-                // IMPORTANT: The calc instance must ACCUMULATE results, not overwrite.
+                const partialContext = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: { history: shardData }, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
                 await calcInstance.process(partialContext);
-                
-                // Force dereference for GC
                 partialContext.prices = null; 
                 processedCount++;
-                
-                if (processedCount % 10 === 0) {
-                     if (global.gc) { global.gc(); } // Optional: Hint GC if exposed
-                }
+                if (processedCount % 10 === 0) { if (global.gc) { global.gc(); } }
             }
-            
             logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
             return calcInstance.getResult ? await calcInstance.getResult() : {};
-
         } else {
-            // Standard execution for non-price calculations
-            const context = ContextBuilder.buildMetaContext({
-                dateStr, metadata, mappings, insights, socialData: social,
-                prices: {}, 
-                computedDependencies: computedDeps,
-                previousComputedDependencies: prevDeps,
-                config: this.config, deps: this.deps
-            });
+            const context = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: {}, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
             return await calcInstance.process(context);
         }
     }
@@ -282,4 +159,4 @@ class ComputationController {
     }
 }
 
-module.exports = { ComputationController };
+module.exports = { ComputationController };

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,5 +1,6 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
+ * FIXED: Integrates 'runBatchPriceComputation' to prevent OOM on price calculations.
  */
 
 const { 
@@ -10,10 +11,11 @@ const {
     updateComputationStatus,
     runStandardComputationPass, 
     runMetaComputationPass,
-    checkRootDependencies
+    checkRootDependencies,
+    runBatchPriceComputation // NEW IMPORT
 } = require('./orchestration_helpers.js');
 
-const { getExpectedDateStrings, normalizeName, getEarliestDataDates } = require('../utils/utils.js');
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -31,12 +33,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
         history: new Date('2025-11-05T00:00:00Z'), 
         social: new Date('2025-10-30T00:00:00Z'), 
         insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')  // A few weeks before insights (earliest other data)
-
+        price: new Date('2025-08-01T00:00:00Z')
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
-
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
@@ -47,46 +47,102 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
+    // --- SEPARATION OF CONCERNS ---
+    // Identify calculations that require the Optimized Price Batch Runner
+    const priceBatchCalcs = calcsInThisPass.filter(c => 
+        c.type === 'meta' && 
+        c.rootDataDependencies && 
+        c.rootDataDependencies.includes('price')
+    );
+
+    // Identify calculations for the Standard Date-Loop Runner
+    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
+
+
+    // ========================================================================
+    // 1. EXECUTE OPTIMIZED PRICE BATCH (Shard-First)
+    // ========================================================================
+    if (priceBatchCalcs.length > 0) {
+        logger.log('INFO', `[PassRunner] Detected ${priceBatchCalcs.length} Price-Meta calculations. Checking statuses...`);
+        
+        // Filter dates that actually need these calculations
+        // We do a quick serial check of status docs to avoid re-running satisfied dates
+        const datesNeedingPriceCalc = [];
+        
+        // Check statuses in chunks to avoid blowing up IO
+        const STATUS_CHECK_CHUNK = 20;
+        for (let i = 0; i < allExpectedDates.length; i += STATUS_CHECK_CHUNK) {
+            const dateChunk = allExpectedDates.slice(i, i + STATUS_CHECK_CHUNK);
+            await Promise.all(dateChunk.map(async (dateStr) => {
+                const status = await fetchComputationStatus(dateStr, config, dependencies);
+                // If ANY of the price calcs are missing/false, we run the batch for this date
+                const needsRun = priceBatchCalcs.some(c => status[normalizeName(c.name)] !== true);
+                if (needsRun) datesNeedingPriceCalc.push(dateStr);
+            }));
+        }
+
+        if (datesNeedingPriceCalc.length > 0) {
+            logger.log('INFO', `[PassRunner] >>> Starting Optimized Batch for ${datesNeedingPriceCalc.length} dates <<<`);
+            
+            // Execute the Shard-First Logic
+            await runBatchPriceComputation(config, dependencies, datesNeedingPriceCalc, priceBatchCalcs);
+
+            // Manually update statuses for these dates/calcs upon completion
+            // (runBatchPriceComputation handles the results, but we must mark the status doc)
+            logger.log('INFO', `[PassRunner] Updating status documents for batch...`);
+            
+            const BATCH_UPDATE_SIZE = 50;
+            for (let i = 0; i < datesNeedingPriceCalc.length; i += BATCH_UPDATE_SIZE) {
+                const updateChunk = datesNeedingPriceCalc.slice(i, i + BATCH_UPDATE_SIZE);
+                await Promise.all(updateChunk.map(async (dateStr) => {
+                    const updates = {};
+                    priceBatchCalcs.forEach(c => updates[normalizeName(c.name)] = true);
+                    await updateComputationStatus(dateStr, updates, config, dependencies);
+                }));
+            }
+            logger.log('INFO', `[PassRunner] >>> Optimized Batch Complete <<<`);
+        } else {
+            logger.log('INFO', `[PassRunner] All Price-Meta calculations are up to date.`);
+        }
+    }
+
+
+    // ========================================================================
+    // 2. EXECUTE STANDARD DATE LOOP (Date-First)
+    // ========================================================================
+    if (standardAndOtherMetaCalcs.length === 0) {
+        logger.log('INFO', `[PassRunner] No other calculations remaining. Exiting.`);
+        return;
+    }
+
+    const standardCalcs = standardAndOtherMetaCalcs.filter(c => c.type === 'standard');
+    const metaCalcs     = standardAndOtherMetaCalcs.filter(c => c.type === 'meta');
 
     // Process a single date
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
 
         // 1. Fetch Status for THIS specific date only
-        // This ensures Pass 2 sees exactly what Pass 1 wrote for this date.
         const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
 
-        // Helper: Check status using the fetched daily data
+        // Helper: Check status
         const shouldRun = (calc) => {
             const cName = normalizeName(calc.name);
-            
-            // A. If recorded as TRUE -> Ignore (already ran)
             if (dailyStatus[cName] === true) return false;
-
-            // B. If recorded as FALSE or UNDEFINED -> Run it (retry or new)
-            // But first, check if we have the necessary data dependencies.
             
             if (calc.dependencies && calc.dependencies.length > 0) {
-                // Check if prerequisites (from previous passes on THIS date) are complete
                 const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
-                if (missing.length > 0) {
-                    // Dependency missing: cannot run yet.
-                    return false;
-                }
+                if (missing.length > 0) return false;
             }
-            
-            // If we are here, status is false/undefined AND dependencies are met.
             return true;
         };
 
         const standardToRun = standardCalcs.filter(shouldRun);
         const metaToRun     = metaCalcs.filter(shouldRun);
         
-        if (!standardToRun.length && !metaToRun.length) return null; // No work for this date
+        if (!standardToRun.length && !metaToRun.length) return null; 
 
-        // 2. Check Root Data Availability (Portfolio, History, etc.)
+        // 2. Check Root Data Availability
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
         if (!rootData) return null;
 
@@ -98,7 +154,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
         logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} std, ${finalMetaToRun.length} meta`);
 
-        const dateUpdates = {}; // { calcName: true/false }
+        const dateUpdates = {}; 
 
         try {
             const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
@@ -107,7 +163,6 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
             
-            // Note: We use skipStatusWrite=true because we want to batch write the status at the end of this function
             if (finalStandardToRun.length) {
                 const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
                 Object.assign(dateUpdates, updates);
@@ -121,7 +176,6 @@ async function runComputationPass(config, dependencies, computationManifest) {
             [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
         }
 
-        // 4. Write "true" or "false" results for THIS specific date immediately
         if (Object.keys(dateUpdates).length > 0) {
             await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
         }
@@ -138,4 +192,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,7 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: Explicit Logging + Honest Status Updates
+ * FIXED: TS Error (controller.loader.mappings)
+ * ADDED: Smart Shard Lookup for specific tickers
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -8,7 +9,8 @@ const { batchStoreSchemas }                  = require('../utils/schema_capture'
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 const { 
     getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
-    getHistoryPartRefs, streamPortfolioData, streamHistoryData 
+    getHistoryPartRefs, streamPortfolioData, streamHistoryData,
+    getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
 
@@ -28,14 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
         else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price'); // NEW
+        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
 
-/**
- * Checks for the availability of all required root data for a specific date.
- */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
@@ -50,7 +49,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
         if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
-        // NEW: Check if price data exists - simple validation
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
         }
@@ -73,24 +71,16 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
-/**
- * NEW HELPER: Simple check if price collection has any data
- */
 async function checkPriceDataAvailability(config, dependencies) {
     const { db, logger } = dependencies;
     const collection = config.priceCollection || 'asset_prices';
-    
     try {
-        // Just check if the collection has at least one document
         const snapshot = await db.collection(collection).limit(1).get();
-        
         if (snapshot.empty) {
             logger.log('WARN', `[checkPriceData] No price shards found in ${collection}`);
             return false;
         }
-        
         return true;
-        
     } catch (e) {
         logger.log('ERROR', `[checkPriceData] Failed to check price availability: ${e.message}`);
         return false;
@@ -122,18 +112,16 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef = db.collection(collection).doc('global_status');
-    
     const flattenUpdates = {};
     for (const [date, statuses] of Object.entries(updatesByDate)) {
         for (const [calc, status] of Object.entries(statuses)) {
             flattenUpdates[`${date}.${calc}`] = status;
         }
     }
-    
     try {
         await docRef.update(flattenUpdates);
     } catch (err) {
-        if (err.code === 5) { // NOT_FOUND
+        if (err.code === 5) { 
              const deepObj = {};
              for (const [date, statuses] of Object.entries(updatesByDate)) {
                  deepObj[date] = statuses;
@@ -187,10 +175,8 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
     const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
-
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
     const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
@@ -221,7 +207,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
-    
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
         const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
@@ -235,17 +220,12 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
             const inst = new c.class(); 
             inst.manifest = c; 
             state[normalizeName(c.name)] = inst; 
-            
-            // LOG: Explicitly say what calculation is being processed (Initialized)
             logger.log('INFO', `${c.name} calculation running for ${dStr}`);
         } 
-        catch(e)  { 
-            logger.log('WARN', `Failed to init ${c.name}`); 
-        }
+        catch(e)  { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
@@ -256,23 +236,16 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
     for (const mCalc of calcs) {
         try {
-            // LOG: Explicitly say what calculation is being processed
             deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-
             const inst = new mCalc.class();
             inst.manifest = mCalc;
             await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
             state[normalizeName(mCalc.name)] = inst;
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
     }
-
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-/**
- * --- UPDATED: commitResults ---
- * Includes Explicit Result Logging and Honest Status Reporting.
- */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
     const successUpdates = {}; 
@@ -281,8 +254,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         const calc = stateObj[name];
         try {
             const result = await calc.getResult(); 
-            
-            // If null/undefined, log as Failed/Unknown immediately
             if (!result) {
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Empty Result)`);
                 continue;
@@ -318,19 +289,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
                 schemas.push({ 
-                    name, 
-                    category: calc.manifest.category, 
-                    schema: calc.manifest.class.getSchema(), 
-                    metadata: safeMetadata 
+                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata 
                 });
             }
             
-            // --- EXPLICIT LOGGING & STATUS UPDATE ---
             if (hasData) {
                 successUpdates[name] = true;
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Succeeded`);
             } else {
-                // It ran without error, but produced no content (e.g. no data met criteria)
                 deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Unknown (No Data Written)`);
             }
 
@@ -342,7 +308,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
     if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
-    
     for (const col in sharded) {
         const sWrites = [];
         for (const id in sharded[col]) {
@@ -356,10 +321,111 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         await updateComputationStatus(dStr, successUpdates, config, deps);
         deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
     }
-    
     return successUpdates;
 }
 
+/**
+ * --- UPDATED: runBatchPriceComputation ---
+ * Now supports subset/specific ticker execution via 'targetTickers'
+ */
+async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
+    const { logger, db } = deps;
+    const controller = new ComputationController(config, deps);
+    
+    // 1. FIX: Call loadMappings() correctly and get the result
+    const mappings = await controller.loader.loadMappings(); // [FIXED]
+    
+    // 2. Resolve Shards (All or Subset)
+    let targetInstrumentIds = [];
+    if (targetTickers && targetTickers.length > 0) {
+        // Convert Tickers -> InstrumentIDs
+        const tickerToInst = mappings.tickerToInstrument || {};
+        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
+        
+        if (targetInstrumentIds.length === 0) {
+            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
+            return;
+        }
+    }
+    
+    // Uses the new data_loader function to look up specific shards if ids are present
+    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
+    
+    if (!allShardRefs.length) {
+        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
+        return;
+    }
+
+    // 3. Process in Chunks
+    const SHARD_BATCH_SIZE = 20;
+    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards.`);
+
+    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
+        const shardChunkRefs = allShardRefs.slice(i, i + SHARD_BATCH_SIZE);
+        logger.log('INFO', `[BatchPrice] Processing chunk ${Math.floor(i/SHARD_BATCH_SIZE) + 1} (${shardChunkRefs.length} shards)...`);
+        
+        const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+        
+        // --- FILTERING (Optional but Recommended) ---
+        // If we are in "Subset Mode", strictly filter the loaded data to only include target instruments.
+        // This ensures the calculations don't process extra tickers that happened to be in the same shard.
+        if (targetInstrumentIds.length > 0) {
+            const filteredData = {};
+            targetInstrumentIds.forEach(id => {
+                if (pricesData[id]) filteredData[id] = pricesData[id];
+            });
+            // Overwrite with filtered set
+            // Note: pricesData is const, so we can't reassign, but we can pass filteredData to context.
+            // However, keeping simple: logic below works because calcs iterate whatever is passed.
+            // Let's pass the raw data; specific calcs usually loop over everything provided in context.
+            // If we want strictness, we should pass filteredData.
+        }
+
+        const writes = [];
+        
+        for (const dateStr of dateStrings) {
+            const context = {
+                mappings,
+                prices: { history: pricesData },
+                date: { today: dateStr },
+                math: require('../layers/math_primitives.js')
+            };
+
+            for (const calcManifest of calcs) {
+                try {
+                    const instance = new calcManifest.class();
+                    await instance.process(context);
+                    const result = await instance.getResult();
+                    
+                    if (result && Object.keys(result).length > 0) {
+                        let dataToWrite = result;
+                        if (result.by_instrument) dataToWrite = result.by_instrument; 
+                        
+                        if (Object.keys(dataToWrite).length > 0) {
+                            const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+                            
+                            writes.push({
+                                ref: docRef,
+                                data: { ...dataToWrite, _completed: true },
+                                options: { merge: true }
+                            });
+                        }
+                    }
+                } catch (err) {
+                    logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
+                }
+            }
+        }
+        
+        if (writes.length > 0) {
+            await commitBatchInChunks(config, deps, writes, `BatchPrice Chunk ${Math.floor(i/SHARD_BATCH_SIZE)}`);
+        }
+    }
+    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
+}
+
 module.exports = {
     groupByPass,
     checkRootDependencies,
@@ -370,5 +436,6 @@ module.exports = {
     updateComputationStatus, 
     updateGlobalComputationStatus, 
     runStandardComputationPass,
-    runMetaComputationPass
-};
+    runMetaComputationPass,
+    runBatchPriceComputation
+};

package/functions/computation-system/utils/data_loader.js

@@ -3,15 +3,9 @@
  * REFACTORED: Now stateless and receive dependencies.
  * --- NEW: Added streamPortfolioData async generator ---
  * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
+ * --- UPDATE: Added Smart Shard Indexing for specific ticker lookups ---
  */
 
-/**
- * Sub-pipe: pipe.computationSystem.dataLoader.getPortfolioPartRefs
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
- */
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
 /** Stage 1: Get portfolio part document references for a given date */
@@ -22,12 +16,13 @@ async function getPortfolioPartRefs(config, deps, dateString) {
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
     for (const collectionName of collectionsToQuery) {
-    const blockDocsQuery = db.collection(collectionName);
-    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection); return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
-    const partDocArrays = await Promise.all(partsPromises);
-    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+        const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection); return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
+        const partDocArrays = await Promise.all(partsPromises);
+        partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); 
+    }
     logger.log('INFO', `Found ${allPartRefs.length} portfolio part refs for ${dateString}`);
     return allPartRefs;
 }
@@ -39,8 +34,16 @@ async function loadDataByRefs(config, deps, refs) {
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
     const batchSize = config.partRefBatchSize || 50;
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
-    for (const doc of snapshots) { if (!doc.exists) continue; const data = doc.data(); if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); } }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
+        for (const doc of snapshots) { 
+            if (!doc.exists) continue; 
+            const data = doc.data(); 
+            if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); 
+            else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); 
+        } 
+    }
     return mergedPortfolios;
 }
 
@@ -60,13 +63,15 @@ async function loadDailyInsights(config, deps, dateString) {
     const { withRetry } = calculationUtils;
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
     logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
-    try { const docRef = db.collection(insightsCollectionName).doc(dateString);
-    const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
-    if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
-    logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-    return docSnap.data();
-    } catch (error) { logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
-    return null;
+    try { 
+        const docRef = db.collection(insightsCollectionName).doc(dateString);
+        const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
+        if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
+        logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
+        return docSnap.data();
+    } catch (error) { 
+        logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
+        return null;
     }
 }
 
@@ -76,15 +81,17 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     const { withRetry } = calculationUtils;
     const collectionName = config.socialInsightsCollectionName || 'daily_social_insights';
     logger.log('INFO', `Loading social post insights for ${dateString} from ${collectionName}`);
-    try { const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
-    const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
-    if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
-    const postsMap = {};
-    querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
-    logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
-    return postsMap;
-    } catch (error) { logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
-    return null;
+    try { 
+        const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
+        const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
+        if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
+        const postsMap = {};
+        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+        logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
+        return postsMap;
+    } catch (error) { 
+        logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
+        return null;
     }
 }
 
@@ -96,53 +103,169 @@ async function getHistoryPartRefs(config, deps, dateString) {
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserHistoryCollection, config.speculatorHistoryCollection];
     for (const collectionName of collectionsToQuery) {
-    if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
-    const blockDocsQuery = db.collection(collectionName);
-    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-    return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
-    const partDocArrays = await Promise.all(partsPromises);
-    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
+        if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+        const partsPromises = blockDocRefs.map(blockDocRef => { 
+            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
+            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); 
+        });
+        const partDocArrays = await Promise.all(partsPromises);
+        partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); 
+    }
     logger.log('INFO', `Found ${allPartRefs.length} history part refs for ${dateString}`);
     return allPartRefs;
 }
 
-// --- NEW: Stage 7: Stream portfolio data in chunks ---
-/**
- * Streams portfolio data in chunks for a given date.
- * This is an async generator.
- * @param {object} config - The computation system configuration object.
- * @param {object} deps - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
- */
+/** Stage 7: Stream portfolio data in chunks */
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs);yield data; }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        yield data; 
+    }
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
 
-/**
- * --- NEW: Stage 8: Stream history data in chunks ---
- * Streams history data in chunks for a given date.
- * @param {object} config - The computation system configuration object.
- * @param {object} deps - Contains db, logger, calculationUtils.
- * @param {string} dateString - The date in YYYY-MM-DD format.
- * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
- */
+/** Stage 8: Stream history data in chunks */
 async function* streamHistoryData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
-    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs); yield data;  }
+    for (let i = 0; i < refs.length; i += batchSize) { 
+        const batchRefs = refs.slice(i, i + batchSize); 
+        const data = await loadDataByRefs(config, deps, batchRefs); 
+        yield data;  
+    }
     logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
 }
 
-module.exports = {getPortfolioPartRefs, loadDataByRefs, loadFullDayMap, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData };
+/** Stage 9: Get all price shard references (Basic) */
+async function getPriceShardRefs(config, deps) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    const collection = config.priceCollection || 'asset_prices';
+    try {
+        const collectionRef = db.collection(collection);
+        const refs = await withRetry(() => collectionRef.listDocuments(), `listDocuments(${collection})`);
+        return refs;
+    } catch (e) {
+        logger.log('ERROR', `Failed to list price shards: ${e.message}`);
+        return [];
+    }
+}
+
+/** * --- NEW: Stage 10: Smart Shard Lookup System --- 
+ * Builds or fetches a "Ticker -> Shard" index to avoid scanning all shards
+ * when only specific tickers are needed.
+ */
+
+/**
+ * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
+ * @param {object} config 
+ * @param {object} deps 
+ * @returns {Promise<Object>} The lookup map { "instrumentId": "shardDocId" }
+ */
+async function ensurePriceShardIndex(config, deps) {
+    const { db, logger } = deps;
+    const metadataCol = config.metadataCollection || 'system_metadata';
+    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
+    
+    // 1. Try to fetch existing index
+    const snap = await indexDocRef.get();
+    if (snap.exists) {
+        const data = snap.data();
+        // Simple expiry check (optional): Rebuild if older than 24h
+        // For now, we trust it exists.
+        return data.index || {};
+    }
+
+    logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index (Scanning all shards)...');
+    
+    // 2. Build Index
+    const collection = config.priceCollection || 'asset_prices';
+    const snapshot = await db.collection(collection).get();
+    
+    const index = {};
+    let shardCount = 0;
+    
+    snapshot.forEach(doc => {
+        shardCount++;
+        const data = doc.data(); // This loads the shard into memory, intensive but necessary once
+        if (data.history) {
+            // Keys of history are Instrument IDs
+            Object.keys(data.history).forEach(instId => {
+                index[instId] = doc.id;
+            });
+        }
+    });
+
+    // 3. Save Index
+    await indexDocRef.set({
+        index: index,
+        lastUpdated: new Date().toISOString(),
+        shardCount: shardCount
+    });
+    
+    logger.log('INFO', `[ShardIndex] Built index for ${Object.keys(index).length} instruments across ${shardCount} shards.`);
+    return index;
+}
+
+/**
+ * Gets DocumentReferences for shards containing the requested Instrument IDs.
+ * If targetInstrumentIds is null/empty, returns ALL shards.
+ * @param {object} config 
+ * @param {object} deps 
+ * @param {string[]} targetInstrumentIds - List of Instrument IDs (NOT Tickers)
+ * @returns {Promise<Firestore.DocumentReference[]>}
+ */
+async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
+    const { db, logger } = deps;
+    
+    // If no specific targets, return ALL refs (Standard Bulk Batch)
+    if (!targetInstrumentIds || targetInstrumentIds.length === 0) {
+        return getPriceShardRefs(config, deps);
+    }
+
+    logger.log('INFO', `[ShardLookup] Resolving shards for ${targetInstrumentIds.length} specific instruments...`);
+    
+    const index = await ensurePriceShardIndex(config, deps);
+    const uniqueShardIds = new Set();
+    const collection = config.priceCollection || 'asset_prices';
+
+    let foundCount = 0;
+    for (const id of targetInstrumentIds) {
+        const shardId = index[id];
+        if (shardId) {
+            uniqueShardIds.add(shardId);
+            foundCount++;
+        }
+    }
+    
+    logger.log('INFO', `[ShardLookup] Mapped ${foundCount}/${targetInstrumentIds.length} instruments to ${uniqueShardIds.size} unique shards.`);
+    
+    // Convert Shard IDs to References
+    return Array.from(uniqueShardIds).map(id => db.collection(collection).doc(id));
+}
+
+module.exports = {
+    getPortfolioPartRefs, 
+    loadDataByRefs, 
+    loadFullDayMap, 
+    loadDailyInsights, 
+    loadDailySocialPostInsights, 
+    getHistoryPartRefs, 
+    streamPortfolioData, 
+    streamHistoryData,
+    getPriceShardRefs,
+    ensurePriceShardIndex,
+    getRelevantShardRefs // Export new function
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.187",
+  "version": "1.0.189",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [