bulltrackers-module 1.0.195 → 1.0.197

package/functions/computation-system/controllers/computation_controller.js

@@ -3,8 +3,8 @@
  * V4.1: Supports Smart Shard Lookup via Wrapper
  */
 
-const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives');
-const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
+const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives'); // TODO, SURELY THIS COULD BE DYNAMIC, DETECT WHAT IS EXPORTED? 
+const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');                                                                     // THE IDEA IS THAT WE CAN PRODUCE NEW COMPUTATIONS IN MATH PRIMITIVES, AND NOT NEED TO UPDATE ANY OTHER CODE FILE
 
 class DataLoader {
     constructor(config, dependencies) {
@@ -64,7 +64,7 @@ class DataLoader {
     }
 }
 
-class ContextBuilder {
+class ContextBuilder { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
     static buildPerUserContext(options) {
         const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
@@ -80,7 +80,7 @@ class ContextBuilder {
         };
     }
 
-    static buildMetaContext(options) {
+    static buildMetaContext(options) { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
         const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
             date: { today: dateStr },

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -13,7 +13,7 @@ const {
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies,
-    runBatchPriceComputation // NEW IMPORT
+    runBatchPriceComputation 
 } = require('./orchestration_helpers.js');
 
 const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
@@ -31,10 +31,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
     // Hardcoded earliest dates
     const earliestDates = { 
         portfolio: new Date('2025-09-25T00:00:00Z'), 
-        history: new Date('2025-11-05T00:00:00Z'), 
-        social: new Date('2025-10-30T00:00:00Z'), 
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'), 
+        social:    new Date('2025-10-30T00:00:00Z'), 
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z') // This is slightly arbitrary, the true earliest date is 1 year ago, but there is no need to backfill that far so we just set it to be slightly earlier than the 2nd earliest computation.
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
@@ -169,12 +169,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
             
-            // Changed skipstatus write to false to ensure updates are recorded
+            // Changed skipstatus write to false to ensure updates are recorded, allowing for proper tracking and avoiding re-computation in future passes. NOTE : Writing true here introduces significant bugs and should be avoided.
             if (finalStandardToRun.length) {
                 const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false); 
                 Object.assign(dateUpdates, updates);
             }
-            if (finalMetaToRun.length) {
+            if (finalMetaToRun.length) { // Again, writing true here introduces significant bugs and should be avoided.
                 const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false); 
                 Object.assign(dateUpdates, updates);
             }

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -13,7 +13,8 @@ const {
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
-const pLimit = require('p-limit'); // Ensure p-limit is required
+
+const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -25,7 +26,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
  * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
  * Logs warnings but DOES NOT block the commit.
  */
-function validateResultPatterns(logger, calcName, results, category) {
+function validateResultPatterns(logger, calcName, results, category) { // TODO, THIS COULD BE MUCH MORE SOPHISTICATED, WE WILL NEVER FORCE FAIL A COMPUTATION REGARDLESS, BUT IT COULD BE A GREAT WARNING SYSTEM, USE GLOUD LOG SINKS TO DETECT PROBLEMS BASED ON THESE LOGS
     // 1. Skip Speculators (Too sparse, nulls are expected)
     if (category === 'speculator' || category === 'speculators') return;
 
@@ -90,16 +91,16 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
+    let portfolioRefs   = [], historyRefs = [];
+    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData    = null, socialData = null;
 
     try {
         const tasks = [];
         if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));

package/functions/computation-system/layers/math_primitives.js

@@ -732,7 +732,13 @@ class DistributionAnalytics {
 }
 
 
-// This block is dynamically generated. Do not edit manually.
+// TODO, THIS EXPORT CAN SURELY BE DYNAMICALLY PRODUCED? TAKE WHATEVER CLASSES ARE DEFINED? THE IDEA IS THAT WE CAN SIMPLY PRODUCE NEW CLASSES AND NOT NEED TO MAKE ANY FURTHER CHANGES SO :
+// 1. DYNAMICALLY FINDS THE CLASS, EXPORTS HERE
+// 2. DYNAMICALLY IMPORTS INTO COMPUTATION CONTROLLER
+// 3. DYNAMICALLY POPULATES THE MATH VALUE OF THE COMPUTATION CONTROLLER FOR STANDARD AND META COMPUTATION CONTEXTBUILDERS
+// 4. GO AHEAD AND WRITE YOUR COMPUTATION FILES USING THE NEW MATH PRIMITIVES DEFINED AS NEEDED.
+// THIS THEN MEANS THE MATH PRIMTIVIES CAN BE UPDATED EASILY WITHOUT FURTHER CHANGES TO OTHER SYSTEMS AND AVOID COMPLICATED DEBUGGING OR FORGETFUL INTEGRATIONS.
+
 module.exports = {
     Aggregators,
     DataExtractor,

package/functions/computation-system/utils/data_loader.js

@@ -177,7 +177,7 @@ async function getPriceShardRefs(config, deps) {
 async function ensurePriceShardIndex(config, deps) {
     const { db, logger } = deps;
     const metadataCol = config.metadataCollection || 'system_metadata';
-    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
+    const indexDocRef = db.collection(metadataCol).doc('price_shard_index'); // TODO. TEST THIS SHARD INDEX SYSTEM, CURRENTLY UNUSED IN COMPUTATIONS BUT IS EXTREMELY EFFICIENT AND GREAT FOR COST REDUCTION
     
     // 1. Try to fetch existing index
     const snap = await indexDocRef.get();

package/functions/computation-system/utils/schema_capture.js

@@ -6,6 +6,7 @@
 /**
  * Batch store schemas for multiple computations.
  * This function now expects a fully-formed schema, not sample output.
+ * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
  *
  * @param {object} dependencies - Contains db, logger
  * @param {object} config - Configuration object
@@ -13,16 +14,49 @@
  */
 async function batchStoreSchemas(dependencies, config, schemas) {
     const { db, logger } = dependencies;
-    if (config.captureSchemas === false)  { logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.'); return; }
+    
+    if (config.captureSchemas === false) {
+        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
+        return;
+    }
+
     const batch = db.batch();
     const schemaCollection = config.schemaCollection || 'computation_schemas';
+    let validCount = 0;
+
     for (const item of schemas) {
-    try { if (!item.schema) { logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`); continue; }
-    const docRef = db.collection(schemaCollection).doc(item.name);
-    batch.set(docRef, { computationName: item.name, category: item.category, schema: item.schema, metadata: item.metadata || {}, lastUpdated: new Date() }, { merge: true });
-    } catch (error) { logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message }); } }
-    try { await batch.commit(); logger.log('INFO', `[SchemaCapture] Batch stored ${schemas.length} computation schemas`);
-    } catch (error) { logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message }); }
+        try {
+            if (!item.schema) {
+                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
+                continue;
+            }
+
+            const docRef = db.collection(schemaCollection).doc(item.name);
+            
+            // Critical: Always overwrite 'lastUpdated' to now
+            batch.set(docRef, {
+                computationName: item.name,
+                category: item.category,
+                schema: item.schema,
+                metadata: item.metadata || {},
+                lastUpdated: new Date() 
+            }, { merge: true });
+            
+            validCount++;
+
+        } catch (error) {
+            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
+        }
+    }
+
+    if (validCount > 0) {
+        try {
+            await batch.commit();
+            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
+        } catch (error) {
+            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
+        }
+    }
 }
 
 module.exports = {

package/functions/computation-system/utils/utils.js

@@ -83,7 +83,7 @@ async function getEarliestDataDates(config, deps) {
         speculatorHistoryDate, 
         insightsDate, 
         socialDate,
-        priceDate  // NEW
+        priceDate  
     ] = await Promise.all([
         getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
@@ -91,7 +91,7 @@ async function getEarliestDataDates(config, deps) {
         getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
         getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName),
-        getFirstDateFromPriceCollection(config, deps)  // NEW
+        getFirstDateFromPriceCollection(config, deps)  //TODO, Why no config.pricecollectionname here, looks ugly.
     ]);
     
     const getMinDate = (...dates) => { 
@@ -101,35 +101,35 @@ async function getEarliestDataDates(config, deps) {
     };
     
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate = getMinDate(insightsDate);
-    const earliestSocialDate = getMinDate(socialDate); 
-    const earliestPriceDate = getMinDate(priceDate);  // NEW
-    const absoluteEarliest = getMinDate(
+    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate  = getMinDate(insightsDate);
+    const earliestSocialDate    = getMinDate(socialDate); 
+    const earliestPriceDate     = getMinDate(priceDate);  
+    const absoluteEarliest      = getMinDate(
         earliestPortfolioDate, 
         earliestHistoryDate, 
         earliestInsightsDate, 
         earliestSocialDate,
-        earliestPriceDate  // NEW
+        earliestPriceDate  
     );
     
     const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
     
     const result = { 
-        portfolio: earliestPortfolioDate || new Date('2999-12-31'), 
-        history: earliestHistoryDate || new Date('2999-12-31'), 
-        insights: earliestInsightsDate || new Date('2999-12-31'), 
-        social: earliestSocialDate || new Date('2999-12-31'),
-        price: earliestPriceDate || new Date('2999-12-31'),  // NEW
-        absoluteEarliest: absoluteEarliest || fallbackDate 
+        portfolio: earliestPortfolioDate   || new Date('2999-12-31T00:00:00Z'), 
+        history: earliestHistoryDate       || new Date('2999-12-31T00:00:00Z'), 
+        insights: earliestInsightsDate     || new Date('2999-12-31T00:00:00Z'), 
+        social: earliestSocialDate         || new Date('2999-12-31T00:00:00Z'),
+        price: earliestPriceDate           || new Date('2999-12-31T00:00:00Z'),  
+        absoluteEarliest: absoluteEarliest || fallbackDate
     };
     
     logger.log('INFO', 'Earliest data availability map built:', { 
-        portfolio: result.portfolio.toISOString().slice(0, 10), 
-        history: result.history.toISOString().slice(0, 10), 
-        insights: result.insights.toISOString().slice(0, 10), 
-        social: result.social.toISOString().slice(0, 10),
-        price: result.price.toISOString().slice(0, 10),  // NEW
+        portfolio:        result.portfolio.toISOString().slice(0, 10), 
+        history:          result.history.toISOString().slice(0, 10), 
+        insights:         result.insights.toISOString().slice(0, 10), 
+        social:           result.social.toISOString().slice(0, 10),
+        price:            result.price.toISOString().slice(0, 10),  
         absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) 
     });
     
@@ -143,7 +143,7 @@ async function getEarliestDataDates(config, deps) {
 async function getFirstDateFromPriceCollection(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collection = config.priceCollection || 'asset_prices';
+    const collection = config.priceCollection || 'asset_prices'; // TODO  This hardcode is right, but we should really be passing the config directly as other data sources do.
     
     try {
         logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
@@ -185,7 +185,7 @@ async function getFirstDateFromPriceCollection(config, deps) {
         });
         
         if (earliestDate) {
-            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
+            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`); // TODO, WTF IS THIS TS ERROR. Property 'toISOString' does not exist on type 'never'.ts(2339)
         }
         
         return earliestDate;

package/functions/generic-api/helpers/api_helpers.js

@@ -2,7 +2,7 @@
  * @fileoverview API sub-pipes.
  * REFACTORED: Now stateless and receive dependencies.
  * NEW: getDynamicSchema now reads static schema.
- * NEW: createManifestHandler added to serve all schemas.
+ * NEW: createManifestHandler filters out STALE schemas (>7 days old).
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
@@ -15,13 +15,18 @@ const validateRequest = (query, config) => {
     if (!query.computations) return "Missing 'computations' parameter.";
     if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
     if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
+    
     const start = new Date(query.startDate);
     const end = new Date(query.endDate);
+    
     if (end < start) return "'endDate' must be after 'startDate'.";
+    
     const maxDateRange = config.maxDateRange || 100;
     const diffTime = Math.abs(end - start);
     const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
+    
     if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
+    
     return null;
 };
 
@@ -34,8 +39,19 @@ const validateRequest = (query, config) => {
 const buildCalculationMap = (unifiedCalculations) => {
     const calcMap = {};
     for (const category in unifiedCalculations) {
-        for (const subKey in unifiedCalculations[category]) { const item = unifiedCalculations[category][subKey]; if (subKey === 'historical' && typeof item === 'object') { for (const calcName in item) { calcMap[calcName] = {  category: category, class: item[calcName]  }; } }  
-        else if (typeof item === 'function') { const calcName = subKey; calcMap[calcName] = {  category: category, class: item  }; }}} return calcMap;
+        for (const subKey in unifiedCalculations[category]) {
+            const item = unifiedCalculations[category][subKey];
+            if (subKey === 'historical' && typeof item === 'object') {
+                for (const calcName in item) {
+                    calcMap[calcName] = { category: category, class: item[calcName] };
+                }
+            } else if (typeof item === 'function') {
+                const calcName = subKey;
+                calcMap[calcName] = { category: category, class: item };
+            }
+        }
+    }
+    return calcMap;
 };
 
 /**
@@ -45,7 +61,10 @@ const getDateStringsInRange = (startDate, endDate) => {
     const dates = [];
     const current = new Date(startDate + 'T00:00:00Z');
     const end = new Date(endDate + 'T00:00:00Z');
-    while (current <= end) { dates.push(current.toISOString().slice(0, 10)); current.setUTCDate(current.getUTCDate() + 1); }
+    while (current <= end) {
+        dates.push(current.toISOString().slice(0, 10));
+        current.setUTCDate(current.getUTCDate() + 1);
+    }
     return dates;
 };
 
@@ -58,19 +77,42 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
     const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
     const resultsSub = config.resultsSubcollection || 'results';
     const compsSub = config.computationsSubcollection || 'computations';
+    
     try {
         for (const date of dateStrings) {
             response[date] = {};
             const docRefs = [];
             const keyPaths = [];
+            
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
-                if (pathInfo) { const docRef = db.collection(insightsCollection).doc(date) .collection(resultsSub).doc(pathInfo.category) .collection(compsSub).doc(key); docRefs.push(docRef); keyPaths.push(key);
-                } else { logger.log('WARN', `[${date}] No path info found for computation key: ${key}`); } }
+                if (pathInfo) {
+                    const docRef = db.collection(insightsCollection).doc(date)
+                        .collection(resultsSub).doc(pathInfo.category)
+                        .collection(compsSub).doc(key);
+                    docRefs.push(docRef);
+                    keyPaths.push(key);
+                } else {
+                    logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
+                }
+            }
+            
             if (docRefs.length === 0) continue;
+            
             const snapshots = await db.getAll(...docRefs);
-            snapshots.forEach((doc, i) => { const key = keyPaths[i]; if (doc.exists) { response[date][key] = doc.data(); } else { response[date][key] = null; } }); }
-    } catch (error) { logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message }); throw new Error('Failed to retrieve computation data.'); }
+            snapshots.forEach((doc, i) => {
+                const key = keyPaths[i];
+                if (doc.exists) {
+                    response[date][key] = doc.data();
+                } else {
+                    response[date][key] = null;
+                }
+            });
+        }
+    } catch (error) {
+        logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
+        throw new Error('Failed to retrieve computation data.');
+    }
     return response;
 };
 
@@ -79,14 +121,30 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
  */
 const createApiHandler = (config, dependencies, calcMap) => {
     const { logger } = dependencies; 
-    return async (req, res) => { const validationError = validateRequest(req.query, config);
-        if (validationError) { logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });  return res.status(400).send({ status: 'error', message: validationError }); }
+    return async (req, res) => {
+        const validationError = validateRequest(req.query, config);
+        if (validationError) {
+            logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
+            return res.status(400).send({ status: 'error', message: validationError });
+        }
         try {
             const computationKeys = req.query.computations.split(',');
             const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
-            res.status(200).send({ status: 'success', metadata: { computations: computationKeys, startDate: req.query.startDate, endDate: req.query.endDate, }, data, });
-        } catch (error) { logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack }); res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
+            res.status(200).send({
+                status: 'success',
+                metadata: {
+                    computations: computationKeys,
+                    startDate: req.query.startDate,
+                    endDate: req.query.endDate,
+                },
+                data,
+            });
+        } catch (error) {
+            logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
 };
 
 /**
@@ -99,21 +157,29 @@ function createStructureSnippet(data, maxKeys = 20) {
         if (typeof data === 'boolean') return true;
         return data;
     }
-    if (Array.isArray(data)) { if (data.length === 0) return "<empty array>"; return [ createStructureSnippet(data[0], maxKeys) ]; }
+    if (Array.isArray(data)) {
+        if (data.length === 0) return "<empty array>";
+        return [ createStructureSnippet(data[0], maxKeys) ];
+    }
     const newObj = {};
-    const keys = Object.keys(data)
+    const keys = Object.keys(data);
+    
     if (keys.length > 0 && keys.every(k => k.match(/^[A-Z.]+$/) || k.includes('_') || k.match(/^[0-9]+$/))) {
         const exampleKey = keys[0];
         newObj[exampleKey] = createStructureSnippet(data[exampleKey], maxKeys);
         newObj["... (more items)"] = "...";
         return newObj;
     }
+    
     if (keys.length > maxKeys) {
         const firstKey = keys[0] || "example_key";
         newObj[firstKey] = createStructureSnippet(data[firstKey], maxKeys);
         newObj[`... (${keys.length - 1} more keys)`] = "<object>";
     } else {
-        for (const key of keys) { newObj[key] = createStructureSnippet(data[key], maxKeys); } }
+        for (const key of keys) {
+            newObj[key] = createStructureSnippet(data[key], maxKeys);
+        }
+    }
     return newObj;
 }
 
@@ -130,11 +196,16 @@ async function getComputationStructure(computationName, calcMap, config, depende
         const resultsSub = config.resultsSubcollection || 'results';
         const compsSub = config.computationsSubcollection || 'computations';
         const computationQueryPath = `${category}.${computationName}`;
-        const dateQuery = db.collection(insightsCollection) .where(computationQueryPath, '==', true) .orderBy(FieldPath.documentId(), 'desc') .limit(1);
+        const dateQuery = db.collection(insightsCollection)
+            .where(computationQueryPath, '==', true)
+            .orderBy(FieldPath.documentId(), 'desc')
+            .limit(1);
         const dateSnapshot = await dateQuery.get();
         if (dateSnapshot.empty) { return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` }; }
         const latestStoredDate = dateSnapshot.docs[0].id;
-        const docRef = db.collection(insightsCollection).doc(latestStoredDate) .collection(resultsSub).doc(category) .collection(compsSub).doc(computationName);
+        const docRef = db.collection(insightsCollection).doc(latestStoredDate)
+            .collection(resultsSub).doc(category)
+            .collection(compsSub).doc(computationName);
         const doc = await docRef.get();
         if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
         const fullData = doc.data();
@@ -152,30 +223,104 @@ async function getComputationStructure(computationName, calcMap, config, depende
  */
 async function getDynamicSchema(CalcClass, calcName) {
     if (CalcClass && typeof CalcClass.getSchema === 'function') {
-        try { return CalcClass.getSchema();
-        } catch (e) { console.error(`Error running static getSchema() for ${calcName}: ${e.message}`); return { "ERROR": `Failed to get static schema: ${e.message}` };}
-    } else { return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` }; }
+        try {
+            return CalcClass.getSchema();
+        } catch (e) {
+            console.error(`Error running static getSchema() for ${calcName}: ${e.message}`);
+            return { "ERROR": `Failed to get static schema: ${e.message}` };
+        }
+    } else {
+        return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` };
+    }
 }
 
 
 /**
- * --- NEW: MANIFEST API HANDLER ---
+ * --- NEW: MANIFEST API HANDLER (With Filtering) ---
  */
 const createManifestHandler = (config, dependencies, calcMap) => {
     const { db, logger } = dependencies;
     const schemaCollection = config.schemaCollection || 'computation_schemas';
+    
     return async (req, res) => {
         try {
             logger.log('INFO', '[API /manifest] Fetching all computation schemas...');
             const snapshot = await db.collection(schemaCollection).get();
-            if (snapshot.empty) { logger.log('WARN', '[API /manifest] No schemas found in collection.'); return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' }); }
+            
+            if (snapshot.empty) {
+                logger.log('WARN', '[API /manifest] No schemas found in collection.');
+                return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' });
+            }
+
+            // --- FILTERING LOGIC ---
             const manifest = {};
-            snapshot.forEach(doc => { const data = doc.data(); manifest[doc.id] = { category: data.category, structure: data.schema,  metadata: data.metadata, lastUpdated: data.lastUpdated }; });
-            res.status(200).send({ status: 'success', summary: { source: 'firestore_computation_schemas', totalComputations: snapshot.size, schemasAvailable: snapshot.size, schemasFailed: 0, lastUpdated: Math.max(...Object.values(manifest).map(m =>  m.lastUpdated ? m.lastUpdated.toMillis() : 0 )) }, manifest: manifest });
+            const now = Date.now();
+            const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+            
+            let activeCount = 0;
+            let staleCount = 0;
+
+            snapshot.forEach(doc => {
+                const data = doc.data();
+                
+                // Safe Timestamp conversion
+                let lastUpdatedMs = 0;
+                if (data.lastUpdated && typeof data.lastUpdated.toMillis === 'function') {
+                    lastUpdatedMs = data.lastUpdated.toMillis();
+                } else if (data.lastUpdated instanceof Date) {
+                    lastUpdatedMs = data.lastUpdated.getTime();
+                } else {
+                    // Fallback for very old records without a timestamp
+                    lastUpdatedMs = 0; 
+                }
+
+                // Exclude stale records
+                if ((now - lastUpdatedMs) < MAX_AGE_MS) {
+                    manifest[doc.id] = {
+                        category: data.category,
+                        structure: data.schema,
+                        metadata: data.metadata,
+                        lastUpdated: data.lastUpdated
+                    };
+                    activeCount++;
+                } else {
+                    staleCount++;
+                }
+            });
+            
+            // Log filtering results
+            if (staleCount > 0) {
+                logger.log('INFO', `[API /manifest] Filtered out ${staleCount} stale schemas (older than 7 days). Returning ${activeCount} active.`);
+            }
+
+            res.status(200).send({
+                status: 'success',
+                summary: {
+                    source: 'firestore_computation_schemas',
+                    totalComputations: snapshot.size,
+                    schemasAvailable: activeCount,
+                    schemasFiltered: staleCount,
+                    lastUpdated: Math.max(...Object.values(manifest).map(m => 
+                         (m.lastUpdated && m.lastUpdated.toMillis) ? m.lastUpdated.toMillis() : 0 
+                    ), 0)
+                },
+                manifest: manifest
+            });
+
         } catch (error) {
             logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message, stack: error.stack });
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
 };
 
 
-module.exports = { validateRequest, buildCalculationMap, fetchUnifiedData, createApiHandler, getComputationStructure, getDynamicSchema, createManifestHandler  };
+module.exports = {
+    validateRequest,
+    buildCalculationMap,
+    fetchUnifiedData,
+    createApiHandler,
+    getComputationStructure,
+    getDynamicSchema,
+    createManifestHandler 
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.195",
+  "version": "1.0.197",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [