bulltrackers-module 1.0.231 → 1.0.233

package/functions/generic-api/helpers/api_helpers.js

@@ -1,53 +1,144 @@
 /**
  * @fileoverview API sub-pipes.
- * REFACTORED: Now stateless and receive dependencies.
- * NEW: getDynamicSchema now reads static schema.
- * NEW: createManifestHandler filters out STALE schemas (>7 days old).
+ * REFACTORED: Fixed Category Resolution to match ManifestBuilder logic.
+ * Implements Status-Based Availability Caching and Smart Date Resolution.
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
 
+// --- AVAILABILITY CACHE ---
+class AvailabilityCache {
+    constructor(db, logger, ttlMs = 5 * 60 * 1000) { // 5 Minute TTL
+        this.db = db;
+        this.logger = logger;
+        this.ttlMs = ttlMs;
+        this.cache = null; 
+        this.lastFetched = 0;
+        this.statusCollection = 'computation_status';
+    }
+
+    async getMap() {
+        const now = Date.now();
+        if (this.cache && (now - this.lastFetched < this.ttlMs)) {
+            return this.cache;
+        }
+
+        this.logger.log('INFO', '[AvailabilityCache] Refreshing availability map from Firestore...');
+        
+        // Fetch recent status to build the map (Limit 400 days)
+        const snapshot = await this.db.collection(this.statusCollection)
+            .orderBy(FieldPath.documentId(), 'desc')
+            .limit(400)
+            .get();
+
+        const newMap = {};
+
+        snapshot.forEach(doc => {
+            const dateStr = doc.id;
+            const statusData = doc.data();
+            
+            if (!/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return;
+
+            for (const [calcName, status] of Object.entries(statusData)) {
+                if (status && status !== 'IMPOSSIBLE') {
+                    if (!newMap[calcName]) newMap[calcName] = [];
+                    newMap[calcName].push(dateStr);
+                }
+            }
+        });
+
+        this.cache = newMap;
+        this.lastFetched = now;
+        this.logger.log('INFO', `[AvailabilityCache] Refreshed. Tracked ${Object.keys(newMap).length} computations.`);
+        return this.cache;
+    }
+}
 
 /**
- * Sub-pipe: pipe.api.helpers.validateRequest
+ * Helper: Resolve which dates to fetch based on mode and availability.
  */
-const validateRequest = (query, config) => {
-    if (!query.computations) return "Missing 'computations' parameter.";
-    if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
-    if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
-    
-    const start = new Date(query.startDate);
-    const end = new Date(query.endDate);
-    
-    if (end < start) return "'endDate' must be after 'startDate'.";
+async function resolveTargetDates(availabilityCache, computationKeys, mode, limit) {
+    const map = await availabilityCache.getMap();
     
-    const maxDateRange = config.maxDateRange || 100;
-    const diffTime = Math.abs(end - start);
-    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
+    const relevantDatesSet = new Set();
+    computationKeys.forEach(key => {
+        const dates = map[key] || [];
+        dates.forEach(d => relevantDatesSet.add(d));
+    });
+
+    const sortedDates = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
+
+    if (sortedDates.length === 0) return [];
+
+    if (mode === 'latest') {
+        return [sortedDates[0]];
+    } 
     
-    if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
+    if (mode === 'series') {
+        return sortedDates.slice(0, limit);
+    }
+
+    return [];
+}
+
+/**
+ * Sub-pipe: pipe.api.helpers.validateRequest
+ */
+const validateRequest = (query) => {
+    if (!query.computations) return "Missing 'computations' parameter.";
     
+    const allowedModes = ['latest', 'series'];
+    if (query.mode && !allowedModes.includes(query.mode)) {
+        return "Invalid 'mode'. Must be 'latest' or 'series'.";
+    }
+
+    if (query.mode === 'series') {
+        const limit = parseInt(query.limit);
+        if (query.limit && (isNaN(limit) || limit < 1 || limit > 365)) {
+            return "Invalid 'limit'. Must be between 1 and 365.";
+        }
+    }
+
     return null;
 };
 
 /**
  * Sub-pipe: pipe.api.helpers.buildCalculationMap
- * --- CRITICAL UPDATE ---
- * This function now stores the class itself in the map,
- * which is required by the /manifest/generate endpoint.
+ * FIX APPLIED: Checks class metadata for category override (mirroring ManifestBuilder).
  */
 const buildCalculationMap = (unifiedCalculations) => {
     const calcMap = {};
+
+    // Helper to resolve category exactly like ManifestBuilder.js does
+    const resolveCategory = (cls, folderName) => {
+        if (typeof cls.getMetadata === 'function') {
+            const meta = cls.getMetadata();
+            // If folder is 'core', allow metadata to override
+            if (folderName === 'core' && meta.category) {
+                return meta.category;
+            }
+        }
+        return folderName;
+    };
+
     for (const category in unifiedCalculations) {
         for (const subKey in unifiedCalculations[category]) {
             const item = unifiedCalculations[category][subKey];
+            
             if (subKey === 'historical' && typeof item === 'object') {
                 for (const calcName in item) {
-                    calcMap[calcName] = { category: category, class: item[calcName] };
+                    const cls = item[calcName];
+                    calcMap[calcName] = { 
+                        category: resolveCategory(cls, category), 
+                        class: cls 
+                    };
                 }
             } else if (typeof item === 'function') {
                 const calcName = subKey;
-                calcMap[calcName] = { category: category, class: item };
+                calcMap[calcName] = { 
+                    category: resolveCategory(item, category), 
+                    class: item 
+                };
             }
         }
     }
@@ -55,21 +146,7 @@ const buildCalculationMap = (unifiedCalculations) => {
 };
 
 /**
- * Internal helper for date strings.
- */
-const getDateStringsInRange = (startDate, endDate) => {
-    const dates = [];
-    const current = new Date(startDate + 'T00:00:00Z');
-    const end = new Date(endDate + 'T00:00:00Z');
-    while (current <= end) {
-        dates.push(current.toISOString().slice(0, 10));
-        current.setUTCDate(current.getUTCDate() + 1);
-    }
-    return dates;
-};
-
-/**
- * Sub-pipe: pipe.api.helpers.fetchData
+ * Sub-pipe: pipe.api.helpers.fetchUnifiedData
  */
 const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
     const { db, logger } = dependencies;
@@ -78,11 +155,13 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
     const resultsSub = config.resultsSubcollection || 'results';
     const compsSub = config.computationsSubcollection || 'computations';
     
+    if (dateStrings.length === 0) return {};
+
     try {
+        const readPromises = [];
+
         for (const date of dateStrings) {
-            response[date] = {};
-            const docRefs = [];
-            const keyPaths = [];
+            response[date] = {}; 
             
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
@@ -90,18 +169,27 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
                     const docRef = db.collection(insightsCollection).doc(date)
                         .collection(resultsSub).doc(pathInfo.category)
                         .collection(compsSub).doc(key);
-                    docRefs.push(docRef);
-                    keyPaths.push(key);
+                    
+                    readPromises.push({ date, key, ref: docRef });
                 } else {
-                    logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
+                    // If no path info, we can't fetch.
+                    response[date][key] = null;
                 }
             }
+        }
+        
+        if (readPromises.length === 0) return response;
+
+        // Batch reads
+        const CHUNK_SIZE = 100;
+        for (let i = 0; i < readPromises.length; i += CHUNK_SIZE) {
+            const chunk = readPromises.slice(i, i + CHUNK_SIZE);
+            const refs = chunk.map(item => item.ref);
             
-            if (docRefs.length === 0) continue;
+            const snapshots = await db.getAll(...refs);
             
-            const snapshots = await db.getAll(...docRefs);
-            snapshots.forEach((doc, i) => {
-                const key = keyPaths[i];
+            snapshots.forEach((doc, idx) => {
+                const { date, key } = chunk[idx];
                 if (doc.exists) {
                     response[date][key] = doc.data();
                 } else {
@@ -109,6 +197,7 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
                 }
             });
         }
+
     } catch (error) {
         logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
         throw new Error('Failed to retrieve computation data.');
@@ -120,24 +209,53 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
  * Factory for the main API handler.
  */
 const createApiHandler = (config, dependencies, calcMap) => {
-    const { logger } = dependencies; 
+    const { logger, db } = dependencies;
+    
+    // Singleton Cache
+    const availabilityCache = new AvailabilityCache(db, logger);
+
     return async (req, res) => {
-        const validationError = validateRequest(req.query, config);
+        const validationError = validateRequest(req.query);
         if (validationError) {
             logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
             return res.status(400).send({ status: 'error', message: validationError });
         }
+        
         try {
             const computationKeys = req.query.computations.split(',');
-            const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
+            const mode = req.query.mode || 'latest'; 
+            const limit = parseInt(req.query.limit) || 30;
+
+            // 1. Resolve Dates
+            const dateStrings = await resolveTargetDates(availabilityCache, computationKeys, mode, limit);
+
+            if (dateStrings.length === 0) {
+                 return res.status(200).send({
+                    status: 'success',
+                    metadata: {
+                        computations: computationKeys,
+                        mode,
+                        count: 0,
+                        dates: []
+                    },
+                    data: {}
+                });
+            }
+
+            // 2. Fetch Data
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
+            
             res.set('Cache-Control', 'public, max-age=300, s-maxage=3600');
             res.status(200).send({
                 status: 'success',
                 metadata: {
                     computations: computationKeys,
-                    startDate: req.query.startDate,
-                    endDate: req.query.endDate,
+                    mode,
+                    limit: mode === 'series' ? limit : 1,
+                    dateRange: {
+                        start: dateStrings[dateStrings.length - 1], 
+                        end: dateStrings[0]
+                    }
                 },
                 data,
             });
@@ -148,9 +266,6 @@ const createApiHandler = (config, dependencies, calcMap) => {
     };
 };
 
-/**
- * Internal helper for snippet generation.
- */
 function createStructureSnippet(data, maxKeys = 20) {
     if (data === null || typeof data !== 'object') {
         if (typeof data === 'number') return 0;
@@ -184,9 +299,6 @@ function createStructureSnippet(data, maxKeys = 20) {
     return newObj;
 }
 
-/**
- * Sub-pipe: pipe.api.helpers.getComputationStructure
- */
 async function getComputationStructure(computationName, calcMap, config, dependencies) {
     const { db, logger } = dependencies;
     try {
@@ -218,10 +330,6 @@ async function getComputationStructure(computationName, calcMap, config, depende
     }
 }
 
-
-/**
- * --- UPDATED: DYNAMIC SCHEMA GENERATION HARNESS ---
- */
 async function getDynamicSchema(CalcClass, calcName) {
     if (CalcClass && typeof CalcClass.getSchema === 'function') {
         try {
@@ -235,47 +343,31 @@ async function getDynamicSchema(CalcClass, calcName) {
     }
 }
 
-
-/**
- * --- NEW: MANIFEST API HANDLER (With Filtering) ---
- */
 const createManifestHandler = (config, dependencies, calcMap) => {
     const { db, logger } = dependencies;
     const schemaCollection = config.schemaCollection || 'computation_schemas';
     
     return async (req, res) => {
         try {
-            logger.log('INFO', '[API /manifest] Fetching all computation schemas...');
             const snapshot = await db.collection(schemaCollection).get();
-            
-            if (snapshot.empty) {
-                logger.log('WARN', '[API /manifest] No schemas found in collection.');
-                return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' });
-            }
+            if (snapshot.empty) { return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' }); }
 
-            // --- FILTERING LOGIC ---
             const manifest = {};
             const now = Date.now();
-            const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+            const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; 
             
             let activeCount = 0;
             let staleCount = 0;
 
             snapshot.forEach(doc => {
                 const data = doc.data();
-                
-                // Safe Timestamp conversion
                 let lastUpdatedMs = 0;
                 if (data.lastUpdated && typeof data.lastUpdated.toMillis === 'function') {
                     lastUpdatedMs = data.lastUpdated.toMillis();
                 } else if (data.lastUpdated instanceof Date) {
                     lastUpdatedMs = data.lastUpdated.getTime();
-                } else {
-                    // Fallback for very old records without a timestamp
-                    lastUpdatedMs = 0; 
                 }
 
-                // Exclude stale records
                 if ((now - lastUpdatedMs) < MAX_AGE_MS) {
                     manifest[doc.id] = {
                         category: data.category,
@@ -284,16 +376,9 @@ const createManifestHandler = (config, dependencies, calcMap) => {
                         lastUpdated: data.lastUpdated
                     };
                     activeCount++;
-                } else {
-                    staleCount++;
-                }
+                } else { staleCount++; }
             });
             
-            // Log filtering results
-            if (staleCount > 0) {
-                logger.log('INFO', `[API /manifest] Filtered out ${staleCount} stale schemas (older than 7 days). Returning ${activeCount} active.`);
-            }
-
             res.status(200).send({
                 status: 'success',
                 summary: {
@@ -301,21 +386,18 @@ const createManifestHandler = (config, dependencies, calcMap) => {
                     totalComputations: snapshot.size,
                     schemasAvailable: activeCount,
                     schemasFiltered: staleCount,
-                    lastUpdated: Math.max(...Object.values(manifest).map(m => 
-                         (m.lastUpdated && m.lastUpdated.toMillis) ? m.lastUpdated.toMillis() : 0 
-                    ), 0)
+                    lastUpdated: Date.now()
                 },
                 manifest: manifest
             });
 
         } catch (error) {
-            logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message, stack: error.stack });
+            logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message });
             res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
         }
     };
 };
 
-
 module.exports = {
     validateRequest,
     buildCalculationMap,

package/functions/generic-api/index.js

@@ -1,142 +1,190 @@
 /**
  * @fileoverview Main entry point for the Generic API module.
- * Exports the 'createApiApp' main pipe function.
- * REFACTORED: /manifest endpoint now reads static schemas from Firestore.
- * REFACTORED: /manifest/generate endpoint now reads static schema from class.
- *
- * --- MODIFIED: Added in-memory cache wrapper for the main API handler ---
+ * Export the 'createApiApp' main pipe function.
+ * REFACTORED: API V3 - Status-Aware Data Fetching.
  */
 
 const express = require('express');
 const cors = require('cors');
-const { FieldPath } = require('@google-cloud/firestore'); 
-const { buildCalculationMap, createApiHandler, getComputationStructure,createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
+const { buildCalculationMap, createApiHandler, getComputationStructure, createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
 
 /**
- * --- NEW: In-Memory Cache Handler ---
- * A wrapper function that adds a time-to-live (TTL) in-memory cache
- * to any Express request handler.
- * @param {function} handler - The original (req, res) handler to wrap.
- * @param {object} dependencies - { logger }
- * @returns {function} The new (req, res) handler with caching logic.
+ * In-Memory Cache Handler
+ * Wrapper that adds TTL cache to GET requests.
  */
 const createCacheHandler = (handler, { logger }) => {
-    // 1. Cache
     const CACHE = {};
-    const CACHE_TTL_MS = 10 * 60 * 1000; 
-    // 2. Return the new handler
-    return async (req, res) => { const cacheKey = req.url;  const now = Date.now();
-    // 3. --- Cache HIT ---
-    if (CACHE[cacheKey] && (now - CACHE[cacheKey].timestamp) < CACHE_TTL_MS) {
-    logger.log('INFO', `[API] Cache HIT for ${cacheKey}`);
-    return res.status(CACHE[cacheKey].status).send(CACHE[cacheKey].data); }
-    // 4. --- Cache MISS ---
-    logger.log('INFO', `[API] Cache MISS for ${cacheKey}`);
-    const originalSend = res.send;
-    const originalStatus = res.status;
-    let capturedData = null;
-    let capturedStatus = 200;
-    res.status = (statusCode) => { capturedStatus = statusCode; return originalStatus.call(res, statusCode); };
-    res.send = (data) => { capturedData = data; return originalSend.call(res, data); };
-    // 5. Call the original handler (which will now use our patched functions)
-    await handler(req, res);
-    // 6. If the response was successful, cache it
-    if (capturedStatus === 200 && capturedData) {
-    logger.log('INFO', `[API] Caching new entry for ${cacheKey}`);
-    CACHE[cacheKey] = { data: capturedData, status: capturedStatus, timestamp: now }; } };
+    const CACHE_TTL_MS = 10 * 60 * 1000; // 10 Minutes
+
+    return async (req, res) => {
+        // Cache Key now includes mode and limit
+        const cacheKey = req.url; 
+        const now = Date.now();
+
+        if (CACHE[cacheKey] && (now - CACHE[cacheKey].timestamp) < CACHE_TTL_MS) {
+            logger.log('INFO', `[API] Cache HIT for ${cacheKey}`);
+            return res.status(CACHE[cacheKey].status).send(CACHE[cacheKey].data);
+        }
+
+        logger.log('INFO', `[API] Cache MISS for ${cacheKey}`);
+        
+        const originalSend = res.send;
+        const originalStatus = res.status;
+        let capturedData = null;
+        let capturedStatus = 200;
+
+        res.status = (statusCode) => {
+            capturedStatus = statusCode;
+            return originalStatus.call(res, statusCode);
+        };
+        
+        res.send = (data) => {
+            capturedData = data;
+            return originalSend.call(res, data);
+        };
+
+        await handler(req, res);
+
+        if (capturedStatus === 200 && capturedData) {
+            logger.log('INFO', `[API] Caching new entry for ${cacheKey}`);
+            CACHE[cacheKey] = {
+                data: capturedData,
+                status: capturedStatus,
+                timestamp: now
+            };
+        }
+    };
 };
 
 
 /**
  * Main pipe: pipe.api.createApiApp
- * Creates and configures the Express app for the Generic API.
- * @param {object} config - The Generic API V2 configuration object.
- * @param {object} dependencies - Shared dependencies { db, logger }.
- * @param {Object} unifiedCalculations - The calculations manifest from 'aiden-shared-calculations-unified'.
- * @returns {express.Application} The configured Express app.
  */
 function createApiApp(config, dependencies, unifiedCalculations) {
     const app = express();
     const { logger, db } = dependencies;
     
-    // --- Pre-compute Calculation Map (now includes classes) ---
+    // Build Calc Map once
     const calcMap = buildCalculationMap(unifiedCalculations);
 
-    // --- Middleware ---
+    // Middleware
     app.use(cors({ origin: true }));
     app.use(express.json());
 
-    // --- Main API Endpoint ---
+    // --- Main API V3 Endpoint ---
+    // createApiHandler now initializes the AvailabilityCache internally
     const originalApiHandler = createApiHandler(config, dependencies, calcMap);
     const cachedApiHandler = createCacheHandler(originalApiHandler, dependencies);
+    
+    // This handler now supports ?mode=latest and ?mode=series&limit=X
     app.get('/', cachedApiHandler); 
 
-    // --- Health Check Endpoint ---
+    // Health Check
     app.get('/health', (req, res) => { res.status(200).send('OK'); });
 
-    // --- Debug Endpoint to list all computation keys ---
+    // Debug: List keys
     app.get('/list-computations', (req, res) => {
-        try { const computationKeys = Object.keys(calcMap); 
-        res.status(200).send({ status: 'success', count: computationKeys.length, computations: computationKeys.sort(), });
-        } catch (error) { logger.log('ERROR', 'API /list-computations failed.', { errorMessage: error.message });
-        res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } });
+        try {
+            const computationKeys = Object.keys(calcMap); 
+            res.status(200).send({ 
+                status: 'success', 
+                count: computationKeys.length, 
+                computations: computationKeys.sort(), 
+            });
+        } catch (error) { 
+            logger.log('ERROR', 'API /list-computations failed.', { errorMessage: error.message });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
+        } 
+    });
 
-    // --- Debug Endpoint to get *stored* structure from Firestore ---
+    // Structure Inspection
     app.get('/structure/:computationName', async (req, res) => {
         const { computationName } = req.params;
         const result = await getComputationStructure(computationName, calcMap, config, dependencies);
-        if (result.status === 'error') { const statusCode = result.message.includes('not found') ? 404 : 500;
-        return res.status(statusCode).send(result); }
-        res.status(200).send(result); });
+        if (result.status === 'error') { 
+            const statusCode = result.message.includes('not found') ? 404 : 500;
+            return res.status(statusCode).send(result); 
+        }
+        res.status(200).send(result); 
+    });
+
+    // Manifests (Schema Generation)
     app.get('/manifest', createManifestHandler(config, dependencies, calcMap));
+    
+    // Manual Schema Gen Trigger
     app.post('/manifest/generate/:computationName', async (req, res) => {
         const { computationName } = req.params;
         logger.log('INFO', `Manual static schema generation requested for: ${computationName}`);
 
         try {
-            // 1. Find the calculation class from the calcMap
             const calcInfo = calcMap[computationName];
-            if (!calcInfo || !calcInfo.class) { return res.status(404).send({ status: 'error', message: `Computation '${computationName}' not found or has no class in calculation map.` }); }
+            if (!calcInfo || !calcInfo.class) { 
+                return res.status(404).send({ status: 'error', message: `Computation '${computationName}' not found.` }); 
+            }
             const targetCalcClass = calcInfo.class;
             const targetCategory = calcInfo.category;
 
-            // 2. Use the getDynamicSchema helper (which now just reads the static method)
             const schemaStructure = await getDynamicSchema(targetCalcClass, computationName);
-            if (schemaStructure.ERROR) { return res.status(400).send({ status: 'error', message: `Failed to get static schema: ${schemaStructure.ERROR}` }); }
+            if (schemaStructure.ERROR) { 
+                return res.status(400).send({ status: 'error', message: `Failed to get static schema: ${schemaStructure.ERROR}` }); 
+            }
             
-            // 3. Import the new batchStoreSchemas utility
             const { batchStoreSchemas } = require('../computation-system/utils/schema_capture.js');
 
-            // 4. Get metadata (as much as we can from the class)
-            const metadata = { isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')), dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [], rootDataDependencies: [],  pass: 'unknown',  type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard', note: "Manually generated via API" };
-
-            // 5. Store the schema in Firestore
-            await batchStoreSchemas( dependencies, config, [{ name: computationName, category: targetCategory, schema: schemaStructure, metadata: metadata }] );
-
-            // 6. Respond with the schema
-            res.status(200).send({ status: 'success', message: `Static schema read and stored for ${computationName}`, computation: computationName, category: targetCategory, schema: schemaStructure });
+            const metadata = { 
+                isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')), 
+                dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [], 
+                rootDataDependencies: [],  
+                type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard', 
+                note: "Manually generated via API" 
+            };
+
+            await batchStoreSchemas(dependencies, config, [{ 
+                name: computationName, 
+                category: targetCategory, 
+                schema: schemaStructure, 
+                metadata: metadata 
+            }]);
+
+            res.status(200).send({ 
+                status: 'success', 
+                message: `Static schema read and stored for ${computationName}`, 
+                computation: computationName, 
+                category: targetCategory, 
+                schema: schemaStructure 
+            });
 
         } catch (error) {
-            logger.log('ERROR', `Failed to generate schema for ${computationName}`, { errorMessage: error.message, stack: error.stack });
-            res.status(5.00).send({ status: 'error', message: `Failed to generate/store schema: ${error.message}` });
+            logger.log('ERROR', `Failed to generate schema for ${computationName}`, { errorMessage: error.message });
+            res.status(500).send({ status: 'error', message: `Failed: ${error.message}` });
         }
     });
 
-    /**
-     * This endpoint is fine as-is. It reads from the Firestore
-     * collection that the /manifest and /manifest/generate routes populate.
-     */
+    // Single Manifest Get
     app.get('/manifest/:computationName', async (req, res) => {
         const { computationName } = req.params;
         try {
             const schemaCollection = config.schemaCollection || 'computation_schemas';
             const schemaDoc = await db.collection(schemaCollection).doc(computationName).get();
-            if (!schemaDoc.exists) { return res.status(4404).send({ status: 'error', message: `Schema not found for computation: ${computationName}`, hint: 'Try running the computation system or use POST /manifest/generate/:computationName' }); }
+            if (!schemaDoc.exists) { 
+                return res.status(404).send({ status: 'error', message: `Schema not found for ${computationName}` }); 
+            }
             const data = schemaDoc.data();            
-            res.status(200).send({ status: 'success', computation: computationName, category: data.category, structure: data.schema, metadata: data.metadata || {}, lastUpdated: data.lastUpdated });
-        } catch (error) { logger.log('ERROR', `Failed to fetch schema for ${computationName}`, { errorMessage: error.message }); res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); }});
+            res.status(200).send({ 
+                status: 'success', 
+                computation: computationName, 
+                category: data.category, 
+                structure: data.schema, 
+                metadata: data.metadata || {}, 
+                lastUpdated: data.lastUpdated 
+            });
+        } catch (error) { 
+            logger.log('ERROR', `Failed to fetch schema for ${computationName}`, { errorMessage: error.message }); 
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
+        }
+    });
+    
     return app;
 }
 
-
 module.exports = { createApiApp, helpers: require('./helpers/api_helpers.js') };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.231",
+  "version": "1.0.233",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system/controllers/computation_controller.js

@@ -1,146 +0,0 @@
-/**
- * FIXED: computation_controller.js
- * V5.1: Exports LEGACY_MAPPING for Manifest Builder
- */
-
-// Load all layers dynamically from the index
-const mathLayer                                                                                   = require('../layers/index');
-const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
-
-// Legacy Keys Mapping (Ensures backward compatibility with existing Calculations)
-// Maps the new modular class names to the property names expected by existing code (e.g. math.extract)
-const LEGACY_MAPPING = {
-    DataExtractor: 'extract',
-    HistoryExtractor: 'history',
-    MathPrimitives: 'compute',
-    Aggregators: 'aggregate',
-    Validators: 'validate',
-    SignalPrimitives: 'signals',
-    SCHEMAS: 'schemas',
-    DistributionAnalytics: 'distribution',
-    TimeSeries: 'TimeSeries',
-    priceExtractor: 'priceExtractor',
-    InsightsExtractor: 'insights',
-    UserClassifier: 'classifier',
-    Psychometrics: 'psychometrics',
-    CognitiveBiases: 'bias',
-    SkillAttribution: 'skill',
-    ExecutionAnalytics: 'execution',
-    AdaptiveAnalytics: 'adaptive'
-};
-
-class DataLoader {
-    constructor(config, dependencies)     { this.config = config; this.deps = dependencies; this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null }; }
-    get mappings()                        { return this.cache.mappings; }
-    async loadMappings()                  { if (this.cache.mappings) return this.cache.mappings; const { calculationUtils } = this.deps; this.cache.mappings = await calculationUtils.loadInstrumentMappings(); return this.cache.mappings; }
-    async loadInsights(dateStr)           { if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); const insights = await loadDailyInsights(this.config, this.deps, dateStr); this.cache.insights.set(dateStr, insights); return insights; }
-    async loadSocial(dateStr)             { if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); const social = await loadDailySocialPostInsights(this.config, this.deps, dateStr); this.cache.social.set(dateStr, social); return social; }
-    async getPriceShardReferences()       { return getPriceShardRefs(this.config, this.deps); }
-    async getSpecificPriceShardReferences (targetInstrumentIds) { return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); }
-    async loadPriceShard(docRef)          { try { const snap = await docRef.get(); if (!snap.exists) return {}; return snap.data(); } catch (e) { console.error(`Error loading shard ${docRef.path}:`, e); return {}; } }
-}
-
-class ContextBuilder { 
-    static buildMathContext() {
-        const mathContext = {};
-        for (const [key, value] of Object.entries(mathLayer)) { mathContext[key] = value; const legacyKey = LEGACY_MAPPING[key]; if (legacyKey) { mathContext[legacyKey] = value; } }
-        return mathContext;
-    }
-    static buildPerUserContext(options) {
-        const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
-        return {
-            user: { id: userId, type: userType, portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio }, history: { today: todayHistory, yesterday: yesterdayHistory } },
-            date: { today: dateStr },
-            insights: { today: insights?.today, yesterday: insights?.yesterday },
-            social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            mappings: mappings || {},
-            math: ContextBuilder.buildMathContext(), 
-            computed: computedDependencies || {},
-            previousComputed: previousComputedDependencies || {},
-            meta: metadata, config, deps
-        };
-    }
-
-    static buildMetaContext(options) {
-        const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
-        return {
-            date: { today: dateStr },
-            insights: { today: insights?.today, yesterday: insights?.yesterday },
-            social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            prices: prices || {},
-            mappings: mappings || {},
-            math: ContextBuilder.buildMathContext(), 
-            computed: computedDependencies || {},
-            previousComputed: previousComputedDependencies || {},
-            meta: metadata, config, deps
-        };
-    }
-}
-
-class ComputationExecutor {
-    constructor(config, dependencies, dataLoader) {
-        this.config = config;
-        this.deps   = dependencies;
-        this.loader = dataLoader;
-    }
-
-    async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps) {
-        const { logger }     = this.deps;
-        const targetUserType = metadata.userType; 
-        const mappings       = await this.loader.loadMappings();
-        const insights       = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
-        
-        // Access SCHEMAS dynamically from the loaded layer
-        const SCHEMAS = mathLayer.SCHEMAS;
-
-        for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
-            const todayHistory       = historyData ? historyData[userId] : null;
-            const actualUserType     = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-            if (targetUserType !== 'all') {
-                 const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-                 if (mappedTarget !== actualUserType) continue;
-            }
-            const context = ContextBuilder.buildPerUserContext({ todayPortfolio, yesterdayPortfolio, todayHistory, userId, userType: actualUserType, dateStr, metadata, mappings, insights, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-            try { await calcInstance.process(context); } catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
-        }
-    }
-
-    async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
-        const mappings   = await this.loader.loadMappings();
-        const { logger } = this.deps;
-        const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
-        const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await this.loader.loadSocial(dateStr)   } : null;
-        
-        if (metadata.rootDataDependencies?.includes('price')) {
-            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
-            const shardRefs = await this.loader.getPriceShardReferences();
-            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
-            let processedCount = 0;
-            for (const ref of shardRefs) {
-                const shardData      = await this.loader.loadPriceShard(ref);
-                const partialContext = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: { history: shardData }, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-                await calcInstance.process(partialContext);
-                partialContext.prices = null; 
-                processedCount++;
-                if (processedCount % 10 === 0) { if (global.gc) { global.gc(); } }
-            }
-            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
-            return calcInstance.getResult ? await calcInstance.getResult() : {};
-        } else {
-            const context = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: {}, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-            return await calcInstance.process(context);
-        }
-    }
-}
-
-class ComputationController {
-    constructor(config, dependencies) {
-        this.config   = config;
-        this.deps     = dependencies;
-        this.loader   = new DataLoader(config, dependencies);
-        this.executor = new ComputationExecutor(config, dependencies, this.loader);
-    }
-}
-
-module.exports = { ComputationController, LEGACY_MAPPING };