bulltrackers-module 1.0.147 → 1.0.149

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -2,6 +2,8 @@ const { FieldPath } = require('@google-cloud/firestore');
 // --- MODIFIED: Import streamPortfolioData ---
 const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
+// --- CHANGED: We only need batchStoreSchemas now ---
+const { batchStoreSchemas } = require('../utils/schema_capture');
 
 /** Stage 1: Group manifest by pass number */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
@@ -20,7 +22,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
     }
     return { canRun: missing.length === 0, missing };
-}
+}   
 
 /** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
  * Stage 3: Check root data availability for a date
@@ -33,24 +35,17 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio)
-            {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
-        if (dateToProcess >= earliestDates.insights) {
-            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
-        if (dateToProcess >= earliestDates.social) {
-            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
-        if (dateToProcess >= earliestDates.history) {
-            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
+        if (dateToProcess >= earliestDates.portfolio) {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
+        if (dateToProcess >= earliestDates.insights)  {tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
+        if (dateToProcess >= earliestDates.social)    {tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
+        if (dateToProcess >= earliestDates.history)   {tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
         await Promise.all(tasks);
-        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
-
+        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${insightsData_}, S:${socialData_}, H:${hasHistory}`);
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
         return { portfolioRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, historyRefs, status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
 
-
-
 /** --- MODIFIED: Stage 4: Fetch ALL existing computed results for the pass ---
  * This function now checks for *all* calcs in the pass, not just meta-dependencies,
  * to enable skipping completed work.
@@ -58,22 +53,18 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const allCalcsInPass = new Set(calcsInPass.map(c => normalizeName(c.name)));
-    if (!allCalcsInPass.size) return {};
-    logger.log('INFO', `[PassRunner] Checking for ${allCalcsInPass.size} existing results for ${dateStr}...`);
+    if (!allCalcsInPass.size) return {}; logger.log('INFO', `[PassRunner] Checking for ${allCalcsInPass.size} existing results for ${dateStr}...`);
     const docRefs = [], depNames = [];
-    for (const calcName of allCalcsInPass) {
-        const calcManifest = manifestMap.get(calcName);
+    for (const calcName of allCalcsInPass) { const calcManifest = manifestMap.get(calcName);
         if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for ${calcName}`); continue; }
         docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category||'unknown').collection(config.computationsSubcollection).doc(calcName));
         depNames.push(calcName);
     }
-    const fetched = {};
-    if (docRefs.length) (await db.getAll(...docRefs)).forEach((doc,i)=>fetched[depNames[i]]=doc.exists?doc.data():null);
-    return fetched;
+    const fetched = {}; if (docRefs.length) (await db.getAll(...docRefs)).forEach((doc,i)=>fetched[depNames[i]]=doc.exists?doc.data():null); return fetched;
 }
 
 /**
- * --- ENTIRELY REWRITTEN: Stage 5: Filter calculations ---
+ * --- Stage 5: Filter calculations ---
  * This function now implements your "even better design".
  * It calculates the *true earliest run date* for every calculation
  * and filters them out *before* the "Running..." log ever appears.
@@ -95,13 +86,10 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
     };
     const filterCalc = (calc) => {
         if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
-
         const earliestRunDate = getTrueEarliestRunDate(calc);
         if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
-        
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
-        
         if (calc.type === 'meta') { const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]); if (missingDeps.length > 0) { logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); skipped.add(calc.name); return false;} }
         return true;
     };
@@ -110,7 +98,6 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
-
 /** Stage 6: Initialize calculator instances */
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 
@@ -120,16 +107,10 @@ function initializeCalculators(calcs, logger) { const state = {}; for (const c o
 async function loadHistoricalData(date, calcs, config, deps, rootData) { 
     const { logger }              = deps;
     const updated                 = {...rootData};
-    const tasks = [];
-    
-    // --- REMOVED: needsYesterdayPortfolio ---
-    // --- REMOVED: needsTodayHistory ---
-    // --- REMOVED: needsYesterdayHistory ---
+    const tasks                   = [];
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
 
-    // --- REMOVED: All async tasks for portfolio and history data ---
-
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
@@ -140,7 +121,7 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
     await Promise.all(tasks); 
-    return updated; // This no longer contains the large data maps
+    return updated; 
 }
 
 /**
@@ -153,22 +134,17 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
-    
-    // --- MODIFIED: yesterdayPortfolios & todayHistoryData are no longer in rootData ---
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
-    
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
     let firstUser=true; 
 
-    // --- (Non-streaming (insights/social) calculation logic remains unchanged) ---
     for(const name in state){ 
         const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
         const cat=calc.manifest.category;
         if(cat==='socialPosts'||cat==='insights') {
             if (firstUser) {
                 logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
-                // (Using 'null' for hT and hY as they aren't relevant for these calcs)
                 let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
                 if(calc.manifest.isHistorical) {
                     args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
@@ -177,7 +153,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             }
         }
     }
-    // --- (End of non-streaming calc logic) ---
 
     if (calcsThatStreamPortfolio.length === 0) {
         logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
@@ -186,7 +161,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 
     logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs...`);
 
-    // --- NEW: Prepare iterators and maps for parallel streaming ---
     const prevDate = new Date(dateStr + 'T00:00:00Z');
     prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
@@ -194,15 +168,12 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
     const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
 
-    // Get the async iterators
     const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr) : null;
     const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr) : null;
 
-    // These maps will accumulate data chunk-by-chunk
     let yesterdayPortfolios = {};
     let todayHistoryData = {};
 
-    // Load the FIRST chunk of historical data before the loop starts
     if (yP_iterator) { 
         Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
         logger.log('INFO', `[${passName}] Loaded first chunk of yesterday's portfolios.`);
@@ -211,12 +182,9 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
         logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`);
     }
-    
-    // --- MODIFIED: Main streaming loop (driven by TODAY's portfolio stream) ---
+
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
         
-        // --- NEW: Load the NEXT chunk of historical data ---
-        // This keeps the historical maps populated as the main stream progresses
         if (yP_iterator) { 
             Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
         }
@@ -224,12 +192,11 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
         }
 
-        for(const uid in chunk){ // Iterate through today's portfolio chunk
+        for(const uid in chunk){ 
             const p = chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
             context.userType=userType; 
 
-            // --- NEW: Look up corresponding historical data for THIS user ---
             const pY = yesterdayPortfolios[uid] || null; // Yesterday's Portfolio
             const hT = todayHistoryData[uid] || null;    // Today's History
             // (Note: yesterdayHistoryData (hY) would require another stream if needed)
@@ -240,11 +207,9 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
                 
                 if(isSocialOrInsights) continue; // Skip non-streaming calcs
 
-                // --- MODIFIED: Arguments now use streamed historical data (hT) ---
                 let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
                 
                 if(isHistorical){ 
-                    // pY is now the streamed yesterday's portfolio for this user
                     if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
                     args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
                 } 
@@ -255,7 +220,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             } 
             firstUser=false; 
 
-            // --- NEW: Clear this user from historical maps to free memory ---
             if (pY) { delete yesterdayPortfolios[uid]; }
             if (hT) { delete todayHistoryData[uid]; }
         } 
@@ -268,24 +232,26 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
-        return; }
+        return;
+    }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
-    
-    // ===================================================================
-    // === UPDATED CODE BLOCK 1 (runStandardComputationPass) START ===
-    // ===================================================================
-    
+        
     let success = 0;
-    const failedCalcs = []; // <-- ADDED: Initialize failure tracking
+    const failedCalcs = [];
     const standardWrites = [];
     const shardedWrites = {}; 
+    
+    // === NEW: Collect schemas ===
+    const schemasToStore = [];
+    
     for (const name in state) {
         const calc = state[name];
         if (!calc || typeof calc.getResult !== 'function') continue;
-        try {const result = await Promise.resolve(calc.getResult());
+        try {
+            const result = await Promise.resolve(calc.getResult());
             if (result && Object.keys(result).length > 0) {
                 const standardResult = {};
                 for (const key in result) {
@@ -303,22 +269,60 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(calc.manifest.category)
                         .collection(config.computationsSubcollection).doc(name);
-                    console.log('Writing standard result for', name, 'on', dStr);
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
+                
+                // === CHANGED: Capture static schema ===
+                const calcClass = calc.manifest.class;
+                let staticSchema = null;
+                if (calcClass && typeof calcClass.getSchema === 'function') {
+                    try {
+                        staticSchema = calcClass.getSchema();
+                    } catch (e) {
+                        logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message });
+                    }
+                } else {
+                    logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`);
+                }
+
+                if (staticSchema) {
+                    schemasToStore.push({
+                        name,
+                        category: calc.manifest.category,
+                        schema: staticSchema, // <-- Use the static schema
+                        metadata: {
+                            isHistorical: calc.manifest.isHistorical || false,
+                            dependencies: calc.manifest.dependencies || [],
+                            rootDataDependencies: calc.manifest.rootDataDependencies || [],
+                            pass: calc.manifest.pass,
+                            type: calc.manifest.type || 'standard'
+                        }
+                    });
+                }
+                // === END CHANGED SECTION ===
+                
                 success++;
             }
         } catch (e) {
             logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push(name); // <-- ADDED: Track failure
+            failedCalcs.push(name);
         }
     }
     
-    // ( ... The sharded write logic remains unchanged ... )
-    
+    // === NEW: Store schemas asynchronously (don't block computation) ===
+    if (schemasToStore.length > 0) {
+        // This function is now imported from the simplified schema_capture.js
+        batchStoreSchemas(deps, config, schemasToStore).catch(err => {
+            logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', {
+                errorMessage: err.message
+            });
+        });
+    }
+        
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
     }
+    
     for (const docPath in shardedWrites) { 
         const docData = shardedWrites[docPath]; 
         const shardedDocWrites = [];
@@ -341,7 +345,6 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         }
     }
     
-    // --- MODIFIED: Add failed computations to the final log ---
     const logMetadata = {};
     if (failedCalcs.length > 0) {
         logMetadata.failedComputations = failedCalcs;
@@ -349,15 +352,13 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     logger.log(
         success === calcs.length ? 'SUCCESS' : 'WARN', 
         `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`,
-        logMetadata // <-- MODIFIED: Add metadata to log
+        logMetadata
     );
-    
-    // ===================================================================
-    // === UPDATED CODE BLOCK 1 (runStandardComputationPass) END ===
-    // ===================================================================
 }
 
-/** Stage 10: Run meta computations */
+/**
+ * Modified runMetaComputationPass with schema capture Stage 10
+ */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     if (calcs.length === 0) {
@@ -366,28 +367,34 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
-    
-    // ===================================================================
-    // === UPDATED CODE BLOCK 2 (runMetaComputationPass) START ===
-    // ===================================================================
-    
+        
     let success = 0;
-    const failedCalcs = []; // <-- ADDED: Initialize failure tracking
+    const failedCalcs = [];
     const standardWrites = [];
     const shardedWrites = {}; 
+    
+    // === NEW: Collect schemas ===
+    const schemasToStore = [];
+    
     for (const mCalc of calcs) {
         const name = normalizeName(mCalc.name), Cl = mCalc.class;
         if (typeof Cl !== 'function') {
             logger.log('ERROR', `Invalid class ${name}`);
-            failedCalcs.push(name); // <-- ADDED: Track failure (invalid class)
+            failedCalcs.push(name);
             continue;
         }
         const inst = new Cl();
         try {
             const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
-            if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
-                    if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 
-                        {if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};Object.assign(shardedWrites[collectionName], shardedData[collectionName]);}
+            if (result && Object.keys(result).length > 0) {
+                const standardResult = {};
+                for (const key in result) {
+                    if (key.startsWith('sharded_')) {
+                        const shardedData = result[key];
+                        for (const collectionName in shardedData) {
+                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
+                        }
                     } else {
                         standardResult[key] = result[key];
                     }
@@ -396,22 +403,60 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(mCalc.category)
                         .collection(config.computationsSubcollection).doc(name);
-                    console.log('Writing standard result for', name, 'on', dStr);
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
+                
+                // === CHANGED: Capture static schema ===
+                const calcClass = mCalc.class;
+                let staticSchema = null;
+                if (calcClass && typeof calcClass.getSchema === 'function') {
+                    try {
+                        staticSchema = calcClass.getSchema();
+                    } catch (e) {
+                        logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message });
+                    }
+                } else {
+                    logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`);
+                }
+
+                if (staticSchema) {
+                    schemasToStore.push({
+                        name,
+                        category: mCalc.category,
+                        schema: staticSchema, // <-- Use the static schema
+                        metadata: {
+                            isHistorical: mCalc.isHistorical || false,
+                            dependencies: mCalc.dependencies || [],
+                            rootDataDependencies: mCalc.rootDataDependencies || [],
+                            pass: mCalc.pass,
+                            type: 'meta'
+                        }
+                    });
+                }
+                // === END CHANGED SECTION ===
+                
                 success++;
             }
         } catch (e) {
             logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push(name); // <-- ADDED: Track failure
+            failedCalcs.push(name);
         }
     }
     
-    // ( ... The sharded write logic remains unchanged ... )
+    // === NEW: Store schemas asynchronously ===
+    if (schemasToStore.length > 0) {
+        // This function is now imported from the simplified schema_capture.js
+        batchStoreSchemas(deps, config, schemasToStore).catch(err => {
+            logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', {
+                errorMessage: err.message
+            });
+        });
+    }
     
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
     }
+    
     for (const collectionName in shardedWrites) {
         const docs = shardedWrites[collectionName];
         const shardedDocWrites = [];
@@ -425,21 +470,10 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
         }
     }    
-    
-    // --- MODIFIED: Add failed computations to the final log ---
+
     const logMetadata = {};
-    if (failedCalcs.length > 0) {
-        logMetadata.failedComputations = failedCalcs;
-    }
-    logger.log(
-        success === calcs.length ? 'SUCCESS' : 'WARN', 
-        `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`,
-        logMetadata // <-- MODIFIED: Add metadata to log
-    );
-    
-    // ===================================================================
-    // === UPDATED CODE BLOCK 2 (runMetaComputationPass) END ===
-    // ===================================================================
+    if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }
+    logger.log( success === calcs.length ? 'SUCCESS' : 'WARN',  `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`, logMetadata );
 }
 
 module.exports = { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/functions/computation-system/utils/schema_capture.js

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview Schema capture utility for computation outputs
+ * This module batches and stores pre-defined static schemas in Firestore.
+ */
+
+/**
+ * Batch store schemas for multiple computations.
+ * This function now expects a fully-formed schema, not sample output.
+ *
+ * @param {object} dependencies - Contains db, logger
+ * @param {object} config - Configuration object
+ * @param {Array} schemas - Array of {name, category, schema, metadata} objects
+ */
+async function batchStoreSchemas(dependencies, config, schemas) {
+    const { db, logger } = dependencies;
+    
+    // Check if schema capture is disabled
+    if (config.captureSchemas === false) {
+        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
+        return;
+    }
+    
+    const batch = db.batch();
+    const schemaCollection = config.schemaCollection || 'computation_schemas';
+    
+    for (const item of schemas) {
+        try {
+            // 'item.schema' is now the pre-built schema from static getSchema()
+            if (!item.schema) {
+                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
+                continue;
+            }
+            
+            const docRef = db.collection(schemaCollection).doc(item.name);
+            
+            batch.set(docRef, {
+                computationName: item.name,
+                category: item.category,
+                schema: item.schema, // Use the provided schema directly
+                metadata: item.metadata || {},
+                lastUpdated: new Date()
+                // 'sampleKeys' is removed as we no longer have the output
+            }, { merge: true });
+        } catch (error) {
+            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, {
+                errorMessage: error.message
+            });
+        }
+    }
+    
+    try {
+        await batch.commit();
+        logger.log('INFO', `[SchemaCapture] Batch stored ${schemas.length} computation schemas`);
+    } catch (error) {
+        logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', {
+            errorMessage: error.message
+        });
+    }
+}
+
+module.exports = {
+    // generateSchema and storeComputationSchema are now obsolete
+    batchStoreSchemas
+};

package/functions/generic-api/helpers/api_helpers.js

@@ -1,63 +1,13 @@
 /**
  * @fileoverview API sub-pipes.
  * REFACTORED: Now stateless and receive dependencies.
- * NEW: Added getDynamicSchema to "test run" calculations
- * by mocking async dependencies.
+ * NEW: getDynamicSchema now reads static schema.
+ * NEW: createManifestHandler added to serve all schemas.
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
-// --- NEW: Import calculation utils for mocking ---
-// We import 'aiden-shared-calculations-unified' to access its 'utils'
-const { utils } = require('aiden-shared-calculations-unified');
-
-// --- NEW: Store original utils ---
-const originalLoadMappings = utils.loadInstrumentMappings;
-const originalLoadPrices = utils.loadAllPriceData;
-const originalGetSectorMap = utils.getInstrumentSectorMap;
-
-// --- NEW: Define Mocks ---
-// This mock data will be "injected" into the calculations during the test run
-const mockMappings = { instrumentToTicker: { 1: 'TEST_TICKER', 2: 'ANOTHER' }, instrumentToSector: { 1: 'Test Sector', 2: 'Other' } };
-const mockPrices = { 1: { '2025-01-01': 100, '2025-01-02': 102 } };
-
-const mockPos = { InstrumentID: 1, NetProfit: 0.05, InvestedAmount: 50, Amount: 1000, Value: 55, Direction: 'Buy', IsBuy: true, PositionID: 123, OpenRate: 100, StopLossRate: 90, TakeProfitRate: 120, Leverage: 1, IsTslEnabled: false, OpenDateTime: '2025-01-01T12:00:00Z', CurrentRate: 105 };
-const mockToday = { AggregatedPositions: [mockPos, { ...mockPos, InstrumentID: 2 }], PublicPositions: [mockPos, { ...mockPos, InstrumentID: 2 }], PortfolioValue: 110 };
-const mockYesterday = { AggregatedPositions: [mockPos], PublicPositions: [mockPos], PortfolioValue: 100 };
-const mockInsights = { insights: [{ instrumentId: 1, total: 100, buy: 50, sell: 50 }] };
-const mockSocial = { 'post1': { tickers: ['TEST_TICKER'], sentiment: { overallSentiment: 'Bullish', topics: ['AI'] }, likeCount: 5, commentCount: 2, fullText: 'TEST_TICKER AI' } };
-
-// A mock context that's passed to process()
-const mockContext = { 
-    instrumentMappings: mockMappings.instrumentToTicker, 
-    sectorMapping: mockMappings.instrumentToSector,
-    todayDateStr: '2025-01-02',
-    yesterdayDateStr: '2025-01-01',
-    dependencies: { // For meta-calcs that read other calc results
-        db: { // Mock the DB to return fake data
-            collection: function() { return this; },
-            doc: function() { return this; },
-            get: async () => ({ 
-                exists: true, 
-                data: () => ({ /* mock data for meta-calc deps */ 
-                    'asset-crowd-flow': { 'TEST_TICKER': { net_crowd_flow_pct: 1.5 } },
-                    'social_sentiment_aggregation': { 'tickerSentiment': { 'TEST_TICKER': { sentimentRatio: 80 } } },
-                    'daily_investor_scores': { 'user123': 8.5 }
-                }) 
-            }),
-            getAll: async (...refs) => refs.map(ref => ({ 
-                exists: true, 
-                data: () => ({ /* mock data for meta-calc deps */ 
-                    'asset-crowd-flow': { 'TEST_TICKER': { net_crowd_flow_pct: 1.5 } },
-                    'social_sentiment_aggregation': { 'tickerSentiment': { 'TEST_TICKER': { sentimentRatio: 80 } } }
-                }) 
-            }))
-        },
-        logger: { log: () => {} } // Suppress logs during test run
-    },
-    config: {} // For meta-calcs
-};
-// --- END NEW MOCKS ---
 
+// --- All Mocks are REMOVED ---
 
 /**
  * Sub-pipe: pipe.api.helpers.validateRequest
@@ -81,6 +31,9 @@ const validateRequest = (query, config) => {
 
 /**
  * Sub-pipe: pipe.api.helpers.buildCalculationMap
+ * --- CRITICAL UPDATE ---
+ * This function now stores the class itself in the map,
+ * which is required by the /manifest/generate endpoint.
  */
 const buildCalculationMap = (unifiedCalculations) => {
     const calcMap = {};
@@ -91,13 +44,19 @@ const buildCalculationMap = (unifiedCalculations) => {
             // Handle historical subdirectory
             if (subKey === 'historical' && typeof item === 'object') {
                 for (const calcName in item) {
-                    calcMap[calcName] = { category: category };
+                    calcMap[calcName] = { 
+                        category: category,
+                        class: item[calcName] // <-- Store the class
+                    };
                 }
             } 
             // Handle regular daily/meta/social calc
             else if (typeof item === 'function') {
                 const calcName = subKey;
-                calcMap[calcName] = { category: category };
+                calcMap[calcName] = { 
+                    category: category,
+                    class: item // <-- Store the class
+                };
             }
         }
     }
@@ -121,12 +80,6 @@ const getDateStringsInRange = (startDate, endDate) => {
 
 /**
  * Sub-pipe: pipe.api.helpers.fetchData
- * @param {object} config - The Generic API V2 configuration object.
- * @param {object} dependencies - Contains db, logger.
- * @param {string[]} calcKeys - Array of computation keys to fetch.
- * @param {string[]} dateStrings - Array of dates to fetch for.
- * @param {Object} calcMap - The pre-built calculation lookup map.
- * @returns {Promise<Object>} A nested object of [date][computationKey] = data.
  */
 const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
     const { db, logger } = dependencies;
@@ -144,21 +97,16 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
                 if (pathInfo) {
-                    // Use db from dependencies
                     const docRef = db.collection(insightsCollection).doc(date)
                         .collection(resultsSub).doc(pathInfo.category)
                         .collection(compsSub).doc(key);
-
                     docRefs.push(docRef);
                     keyPaths.push(key);
                 } else {
                     logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
                 }
             }
-
             if (docRefs.length === 0) continue;
-
-            // Use db from dependencies
             const snapshots = await db.getAll(...docRefs);
             snapshots.forEach((doc, i) => {
                 const key = keyPaths[i];
@@ -178,13 +126,9 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
 
 /**
  * Factory for the main API handler.
- * @param {object} config - The Generic API V2 configuration object.
- * @param {object} dependencies - Contains db, logger.
- * @param {Object} calcMap - The pre-built calculation lookup map.
- * @returns {Function} An async Express request handler.
  */
 const createApiHandler = (config, dependencies, calcMap) => {
-    const { logger } = dependencies; // db is in dependencies
+    const { logger } = dependencies; 
     
     return async (req, res) => {
         const validationError = validateRequest(req.query, config);
@@ -192,14 +136,10 @@ const createApiHandler = (config, dependencies, calcMap) => {
             logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
             return res.status(400).send({ status: 'error', message: validationError });
         }
-
         try {
             const computationKeys = req.query.computations.split(',');
             const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
-
-            // Pass dependencies to sub-pipe
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
-
             res.status(200).send({
                 status: 'success',
                 metadata: {
@@ -221,31 +161,24 @@ const createApiHandler = (config, dependencies, calcMap) => {
  */
 function createStructureSnippet(data, maxKeys = 20) {
     if (data === null || typeof data !== 'object') {
-        // Handle primitive types
         if (typeof data === 'number') return 0;
         if (typeof data === 'string') return "string";
         if (typeof data === 'boolean') return true;
         return data;
     }
     if (Array.isArray(data)) {
-        if (data.length === 0) {
-            return "<empty array>";
-        }
-        // Generalize array contents to just the first element's structure
+        if (data.length === 0) return "<empty array>";
         return [ createStructureSnippet(data[0], maxKeys) ];
     }
     const newObj = {};
     const keys = Object.keys(data);
     
-    // Check if it's an "example" object (like { "AAPL": {...} })
-    // This heuristic identifies keys that are all-caps or look like example tickers
     if (keys.length > 0 && keys.every(k => k.match(/^[A-Z.]+$/) || k.includes('_') || k.match(/^[0-9]+$/))) {
         const exampleKey = keys[0];
         newObj[exampleKey] = createStructureSnippet(data[exampleKey], maxKeys);
         newObj["... (more items)"] = "...";
         return newObj;
     }
-
     if (keys.length > maxKeys) {
         const firstKey = keys[0] || "example_key";
         newObj[firstKey] = createStructureSnippet(data[firstKey], maxKeys);
@@ -260,7 +193,6 @@ function createStructureSnippet(data, maxKeys = 20) {
 
 /**
  * Sub-pipe: pipe.api.helpers.getComputationStructure
- * (This is now a debug tool to check *live* data)
  */
 async function getComputationStructure(computationName, calcMap, config, dependencies) {
     const { db, logger } = dependencies;
@@ -270,40 +202,29 @@ async function getComputationStructure(computationName, calcMap, config, depende
             return { status: 'error', computation: computationName, message: `Computation not found in calculation map.` };
         }
         const { category } = pathInfo;
-
         const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
         const resultsSub = config.resultsSubcollection || 'results';
         const compsSub = config.computationsSubcollection || 'computations';
-
         const computationQueryPath = `${category}.${computationName}`;
-        // Use db from dependencies
+        
         const dateQuery = db.collection(insightsCollection)
             .where(computationQueryPath, '==', true)
             .orderBy(FieldPath.documentId(), 'desc')
             .limit(1);
-            
         const dateSnapshot = await dateQuery.get();
-
         if (dateSnapshot.empty) {
             return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` };
         }
-        
         const latestStoredDate = dateSnapshot.docs[0].id;
-
-        // Use db from dependencies
         const docRef = db.collection(insightsCollection).doc(latestStoredDate)
             .collection(resultsSub).doc(category)
             .collection(compsSub).doc(computationName);
-
         const doc = await docRef.get();
-
         if (!doc.exists) {
             return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` };
         }
-
         const fullData = doc.data();
         const structureSnippet = createStructureSnippet(fullData);
-
         return {
             status: 'success',
             computation: computationName,
@@ -311,7 +232,6 @@ async function getComputationStructure(computationName, calcMap, config, depende
             latestStoredDate: latestStoredDate,
             structureSnippet: structureSnippet,
         };
-
     } catch (error) {
         logger.log('ERROR', `API /structure/${computationName} helper failed.`, { errorMessage: error.message });
         return { status: 'error', computation: computationName, message: error.message };
@@ -320,54 +240,73 @@ async function getComputationStructure(computationName, calcMap, config, depende
 
 
 /**
- * --- NEW: DYNAMIC SCHEMA GENERATION HARNESS ---
- * @param {class} CalcClass The calculation class to test.
- * @param {string} calcName The name of the calculation for logging.
- * @returns {Promise<object>} A snippet of the output structure.
+ * --- UPDATED: DYNAMIC SCHEMA GENERATION HARNESS ---
  */
 async function getDynamicSchema(CalcClass, calcName) {
-    // 1. Apply Mocks (Monkey-Patching)
-    utils.loadInstrumentMappings = async () => mockMappings;
-    utils.loadAllPriceData = async () => mockPrices;
-    utils.getInstrumentSectorMap = async () => mockMappings.instrumentToSector;
-
-    let result = {};
-    const calc = new CalcClass();
-
-    try {
-        // 2. Check for Meta-Calculation signature: process(dateStr, dependencies, config)
-        const processStr = calc.process.toString();
-        if (processStr.includes('dateStr') && processStr.includes('dependencies')) {
-            // It's a meta-calc. Run its process() with mock dependencies
-            result = await calc.process('2025-01-02', mockContext.dependencies, mockContext.config);
-        } else {
-            // It's a standard calculation. Run process() + getResult()
-            await calc.process(
-                mockToday, 
-                mockYesterday, 
-                'test-user-123', 
-                mockContext, 
-                mockInsights, // todayInsights
-                mockInsights, // yesterdayInsights
-                mockSocial,   // todaySocial
-                mockSocial    // yesterdaySocial
-            );
-            result = await calc.getResult();
+    if (CalcClass && typeof CalcClass.getSchema === 'function') {
+        try {
+            return CalcClass.getSchema();
+        } catch (e) {
+            console.error(`Error running static getSchema() for ${calcName}: ${e.message}`);
+            return { "ERROR": `Failed to get static schema: ${e.message}` };
         }
-    } catch (e) {
-        console.error(`Error running schema test for ${calcName}: ${e.message}`);
-        result = { "ERROR": `Failed to generate schema: ${e.message}` };
-    } finally {
-        // 3. Restore Original Functions
-        utils.loadInstrumentMappings = originalLoadMappings;
-        utils.loadAllPriceData = originalLoadPrices;
-        utils.getInstrumentSectorMap = originalGetSectorMap;
+    } else {
+        return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` };
     }
-
-    // 4. Sanitize the result to just a "structure"
-    return createStructureSnippet(result);
 }
-// --- END NEW HARNESS ---
+// --- END UPDATED HARNESS ---
+
+
+/**
+ * --- NEW: MANIFEST API HANDLER ---
+ */
+const createManifestHandler = (config, dependencies, calcMap) => {
+    const { db, logger } = dependencies;
+    const schemaCollection = config.schemaCollection || 'computation_schemas';
+    
+    return async (req, res) => {
+        try {
+            logger.log('INFO', '[API /manifest] Fetching all computation schemas...');
+            const snapshot = await db.collection(schemaCollection).get();
+            if (snapshot.empty) {
+                logger.log('WARN', '[API /manifest] No schemas found in collection.');
+                return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' });
+            }
+            
+            const manifest = {};
+            snapshot.forEach(doc => {
+                const data = doc.data();
+                manifest[doc.id] = {
+                    // --- CHANGED: Return the structure consistent with your file ---
+                    category: data.category,
+                    structure: data.schema, // Use 'structure' key
+                    metadata: data.metadata,
+                    lastUpdated: data.lastUpdated
+                };
+            });
+            
+            res.status(200).send({
+                status: 'success',
+                // --- CHANGED: Use the structure from your file ---
+                summary: {
+                    source: 'firestore_computation_schemas',
+                    totalComputations: snapshot.size,
+                    schemasAvailable: snapshot.size,
+                    schemasFailed: 0,
+                    lastUpdated: Math.max(...Object.values(manifest).map(m => 
+                        m.lastUpdated ? m.lastUpdated.toMillis() : 0
+                    ))
+                },
+                manifest: manifest
+            });
+
+        } catch (error) {
+            logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message, stack: error.stack });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
+};
+// --- END NEW HANDLER ---
 
 
 module.exports = {
@@ -376,5 +315,6 @@ module.exports = {
     fetchUnifiedData,
     createApiHandler,
     getComputationStructure,
-    getDynamicSchema // <-- EXPORT NEW HELPER
+    getDynamicSchema,
+    createManifestHandler // <-- EXPORT NEW HANDLER
 };

package/functions/generic-api/index.js

@@ -1,8 +1,8 @@
 /**
  * @fileoverview Main entry point for the Generic API module.
  * Exports the 'createApiApp' main pipe function.
- * REFACTORED: /manifest endpoint now uses a dynamic "test harness"
- * to generate the schema without modifying calculation files.
+ * REFACTORED: /manifest endpoint now reads static schemas from Firestore.
+ * REFACTORED: /manifest/generate endpoint now reads static schema from class.
  */
 
 const express = require('express');
@@ -12,7 +12,8 @@ const {
     buildCalculationMap, 
     createApiHandler, 
     getComputationStructure,
-    getDynamicSchema // <-- IMPORT NEW HELPER
+    createManifestHandler, // <-- IMPORT NEW HANDLER
+    getDynamicSchema      // <-- This helper's behavior has changed
 } = require('./helpers/api_helpers.js');
 
 /**
@@ -25,9 +26,9 @@ const {
  */
 function createApiApp(config, dependencies, unifiedCalculations) {
     const app = express();
-    const { logger } = dependencies; // db is in dependencies
+    const { logger, db } = dependencies; 
 
-    // --- Pre-compute Calculation Map ---
+    // --- Pre-compute Calculation Map (now includes classes) ---
     const calcMap = buildCalculationMap(unifiedCalculations);
 
     // --- Middleware ---
@@ -60,90 +61,148 @@ function createApiApp(config, dependencies, unifiedCalculations) {
     // --- Debug Endpoint to get *stored* structure from Firestore ---
     app.get('/structure/:computationName', async (req, res) => {
         const { computationName } = req.params;
-        
-        // Call sub-pipe, passing dependencies
         const result = await getComputationStructure(computationName, calcMap, config, dependencies);
-
         if (result.status === 'error') {
             const statusCode = result.message.includes('not found') ? 404 : 500;
             return res.status(statusCode).send(result);
         }
-        
         res.status(200).send(result);
     });
 
-    // --- NEW: Fully Refactored Manifest Endpoint (Dynamic Test Run) ---
-    app.get('/manifest', async (req, res) => {
-        logger.log('INFO', 'API /manifest dynamic generation starting...');
+    /**
+     * ---!!!--- REPLACED SECTION ---!!!---
+     * This route now uses the createManifestHandler to serve the
+     * pre-generated schemas from Firestore.
+     */
+    app.get('/manifest', createManifestHandler(config, dependencies, calcMap));
+    
+    
+    /**
+     * ---!!!--- REPLACED SECTION ---!!!---
+     * This endpoint now reads the *static* getSchema() method from a class
+     * and stores that in Firestore. It no longer does runtime inference.
+     */
+    app.post('/manifest/generate/:computationName', async (req, res) => {
+        const { computationName } = req.params;
+        logger.log('INFO', `Manual static schema generation requested for: ${computationName}`);
+        
         try {
-            const manifest = {};
-            let successCount = 0;
-            const errors = [];
+            // 1. Find the calculation class from the calcMap
+            const calcInfo = calcMap[computationName];
             
-            // This logic iterates through the calculation module structure
-            for (const category in unifiedCalculations) {
-                for (const subKey in unifiedCalculations[category]) {
-                    const item = unifiedCalculations[category][subKey];
-                    let calcName = null;
-                    let CalcClass = null;
-
-                    // Handle nested 'historical' directory
-                    if (subKey === 'historical' && typeof item === 'object') {
-                        for (const name in item) {
-                            calcName = name;
-                            CalcClass = item[name];
-                            if (CalcClass && typeof CalcClass === 'function') {
-                                try {
-                                    manifest[calcName] = {
-                                        category: category,
-                                        structure: await getDynamicSchema(CalcClass, calcName) // <-- DYNAMIC CALL
-                                    };
-                                    successCount++;
-                                } catch (e) { errors.push(`${category}/${calcName}: ${e.message}`); }
-                            }
-                        }
-                    } 
-                    // Handle regular calc at root of category
-                    else if (typeof item === 'function') {
-                        calcName = subKey;
-                        CalcClass = item;
-                        if (CalcClass && typeof CalcClass === 'function') {
-                             try {
-                                manifest[calcName] = {
-                                    category: category,
-                                    structure: await getDynamicSchema(CalcClass, calcName) // <-- DYNAMIC CALL
-                                };
-                                successCount++;
-                            } catch (e) { errors.push(`${category}/${calcName}: ${e.message}`); }
-                        }
-                    }
-                }
+            if (!calcInfo || !calcInfo.class) {
+                return res.status(404).send({
+                    status: 'error',
+                    message: `Computation '${computationName}' not found or has no class in calculation map.`
+                });
             }
+            
+            const targetCalcClass = calcInfo.class;
+            const targetCategory = calcInfo.category;
 
-            const totalComputations = Object.keys(calcMap).length;
-            logger.log('INFO', `API /manifest complete. Generated schema for ${successCount}/${totalComputations} computations.`);
+            // 2. Use the getDynamicSchema helper (which now just reads the static method)
+            const schemaStructure = await getDynamicSchema(targetCalcClass, computationName);
+            
+            if (schemaStructure.ERROR) {
+                return res.status(400).send({
+                    status: 'error',
+                    message: `Failed to get static schema: ${schemaStructure.ERROR}`
+                });
+            }
 
+            // 3. Import the new batchStoreSchemas utility
+            const { batchStoreSchemas } = require('../computation-system/utils/schema_capture');
+
+            // 4. Get metadata (as much as we can from the class)
+            const metadata = {
+                isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')),
+                dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [],
+                rootDataDependencies: [], // Cannot be known here
+                pass: 'unknown', // Cannot be known here
+                type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard',
+                note: "Manually generated via API"
+            };
+
+            // 5. Store the schema in Firestore
+            await batchStoreSchemas(
+                dependencies,
+                config,
+                [{
+                    name: computationName,
+                    category: targetCategory,
+                    schema: schemaStructure, // Pass the static schema
+                    metadata: metadata
+                }]
+            );
+            
+            // 6. Respond with the schema
             res.status(200).send({
                 status: 'success',
-                summary: {
-                    source: 'computation_module_dynamic_test',
-                    totalComputations: totalComputations,
-                    schemasGenerated: successCount,
-                    schemasFailed: errors.length,
-                },
-                manifest: manifest,
-                errors: errors.length > 0 ? errors : undefined,
+                message: `Static schema read and stored for ${computationName}`,
+                computation: computationName,
+                category: targetCategory,
+                schema: schemaStructure
+            });
+            
+        } catch (error) {
+            logger.log('ERROR', `Failed to generate schema for ${computationName}`, {
+                errorMessage: error.message,
+                stack: error.stack
             });
+            res.status(5.00).send({
+                status: 'error',
+                message: `Failed to generate/store schema: ${error.message}`
+            });
+        }
+    });
 
+    /**
+     * This endpoint is fine as-is. It reads from the Firestore
+     * collection that the /manifest and /manifest/generate routes populate.
+     */
+    app.get('/manifest/:computationName', async (req, res) => {
+        const { computationName } = req.params;
+        
+        try {
+            const schemaCollection = config.schemaCollection || 'computation_schemas';
+            const schemaDoc = await db.collection(schemaCollection).doc(computationName).get();
+            
+            if (!schemaDoc.exists) {
+                return res.status(404).send({
+                    status: 'error',
+                    message: `Schema not found for computation: ${computationName}`,
+                    hint: 'Try running the computation system or use POST /manifest/generate/:computationName'
+                });
+            }
+            
+            const data = schemaDoc.data();
+            
+            // --- Use the same response structure as /manifest ---
+            res.status(200).send({
+                status: 'success',
+                computation: computationName,
+                category: data.category,
+                structure: data.schema,
+                metadata: data.metadata || {},
+                lastUpdated: data.lastUpdated
+            });
+            
         } catch (error) {
-            logger.log('ERROR', `API /manifest failed unexpectedly.`, { errorMessage: error.message, stack: error.stack });
-            res.status(500).send({ status: 'error', message: `An internal error occurred while building the manifest.` });
+            logger.log('ERROR', `Failed to fetch schema for ${computationName}`, {
+                errorMessage: error.message
+            });
+            res.status(500).send({
+                status: 'error',
+                message: 'An internal error occurred.'
+            });
         }
     });
 
+    // --- THIS MUST BE THE LAST LINE OF THE FUNCTION ---
     return app;
 }
 
+
 module.exports = {
     createApiApp,
     // Exporting helpers so they can be part of the pipe.api.helpers object

package/functions/task-engine/helpers/update_helpers.js

@@ -56,15 +56,18 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     
     let wasHistorySuccess = false, isPrivate = false;
     
+    // --- FIX: 'fetchHistory' MUST be declared here, outside the 'try' block ---
+    let fetchHistory = false;
+    
     try {
         // Fetch history ONCE per user
         const promisesToRun = [];
-        let fetchHistory = false;
+        // --- 'fetchHistory' is no longer declared here ---
         
         if (!historyFetchedForUser.has(userId)) {
             historyHeader = await headerManager.selectHeader();
             if (historyHeader) {
-                fetchHistory = true;
+                fetchHistory = true; // This now sets the outer variable
                 historyFetchedForUser.add(userId);
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.147",
+  "version": "1.0.149",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [