bulltrackers-module 1.0.230 → 1.0.232

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Implements 'IMPOSSIBLE' state logic for missing root data on historical dates.
+ * UPDATED: Removed legacy boolean 'true' logic.
  */
 const { normalizeName }                                    = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -10,7 +10,6 @@ const { StandardExecutor }                                 = require('./executor
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
 const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
-// New Status Constant
 const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
 
 function groupByPass(manifest) { 
@@ -20,21 +19,14 @@ function groupByPass(manifest) {
     }, {}); 
 }
 
-/**
- * Performs strict analysis of what can run.
- * IMPOSSIBLE LOGIC:
- * 1. If Root Data is missing AND Date != Today -> IMPOSSIBLE.
- * 2. If Dependency is IMPOSSIBLE -> IMPOSSIBLE.
- * 3. IMPOSSIBLE items are written to DB to prevent future retries.
- */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
     const report = {
         runnable: [],
-        blocked: [],          // Missing Root Data (Today - Retriable)
-        impossible: [],       // Missing Root Data (Historical) or Dependency Impossible
-        failedDependency: [], // Missing/Stale Dependency (Transient)
-        reRuns: [],           // Hash Mismatch
-        skipped: []           // Already done & valid
+        blocked: [],          
+        impossible: [],       
+        failedDependency: [], 
+        reRuns: [],           
+        skipped: []           
     };
 
     const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
@@ -44,9 +36,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedDepHash = dailyStatus[norm];
         const depManifest = manifestMap.get(norm);
 
-        // Check 1: Is dependency IMPOSSIBLE? (Logic handled in main loop, but safe to check here)
         if (storedDepHash === STATUS_IMPOSSIBLE) return false; 
-
         if (!storedDepHash) return false; 
         if (!depManifest) return false;
         if (storedDepHash !== depManifest.hash) return false;
@@ -59,7 +49,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedHash = dailyStatus[cName];
         const currentHash = calc.hash;
 
-        // 1. Check if ALREADY marked IMPOSSIBLE
+        // 1. Check Impossible
         if (storedHash === STATUS_IMPOSSIBLE) {
             report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
             continue;
@@ -78,7 +68,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         }
 
         if (missingRoots.length > 0) {
-            // LOGIC: If date is NOT today, missing root data is fatal and permanent.
             if (!isTargetToday) {
                 report.impossible.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Historical)` });
             } else {
@@ -95,10 +84,8 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             for (const dep of calc.dependencies) {
                 const normDep = normalizeName(dep);
                 
-                // Check if the dependency is marked IMPOSSIBLE in the DB
                 if (dailyStatus[normDep] === STATUS_IMPOSSIBLE) {
                     dependencyIsImpossible = true;
-                    // We can break early, if one input is impossible, the result is impossible.
                     break; 
                 }
 
@@ -109,7 +96,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         }
 
         if (dependencyIsImpossible) {
-            // Propagate the Impossible Status
             report.impossible.push({ name: cName, reason: 'Dependency is Impossible' });
             continue;
         }
@@ -119,14 +105,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             continue;
         }
 
-        // 4. Hash / State Check
-        if (!storedHash || storedHash === false) { // false indicates previous transient failure
+        // 4. Hash / State Check (Legacy 'true' logic removed)
+        if (!storedHash || storedHash === false) { 
             report.runnable.push(calc);
         } else if (storedHash !== currentHash) {
             report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
-        } else if (storedHash === true) {
-            report.reRuns.push({ name: cName, reason: 'Legacy Upgrade' });
         } else {
+            // Stored Hash === Current Hash
             report.skipped.push({ name: cName });
         }
     }
@@ -162,7 +147,6 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     if (logger && typeof logger.logDateAnalysis === 'function') {
         logger.logDateAnalysis(dateStr, analysisReport);
     } else {
-        // Safe fallback
         const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | Impossible: ${analysisReport.impossible.length}`;
         if (logger && logger.info) logger.info(logMsg);
         else console.log(logMsg);
@@ -170,12 +154,8 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
 
     // 5. UPDATE STATUS FOR NON-RUNNABLE ITEMS
     const statusUpdates = {};
-
-    // A. Mark BLOCKED as 'false' (Transient Failure)
     analysisReport.blocked.forEach(item => statusUpdates[item.name] = false);
     analysisReport.failedDependency.forEach(item => statusUpdates[item.name] = false);
-
-    // B. Mark IMPOSSIBLE as 'IMPOSSIBLE' (Permanent Failure - Overwrites existing status)
     analysisReport.impossible.forEach(item => statusUpdates[item.name] = STATUS_IMPOSSIBLE);
     
     if (Object.keys(statusUpdates).length > 0) {

package/functions/computation-system/persistence/ResultCommitter.js

@@ -45,12 +45,19 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 await commitBatchInChunks(config, deps, updates, `${name} Results`);
                 
                 // Structured Storage Log
-                logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                if (logger && logger.logStorage) {
+                    logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                }
                 
-                successUpdates[name] = calc.manifest.hash || true;
+                // Update success tracking
+                if (calc.manifest.hash) {
+                    successUpdates[name] = calc.manifest.hash;
+                }
             }
         } catch (e) {
-            logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
+            if (logger && logger.log) {
+                 logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
+            }
         }
     }
 
@@ -62,10 +69,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     return successUpdates;
 }
 
-// ... rest of file (calculateFirestoreBytes, prepareAutoShardedWrites) remains same ...
-// Just ensure prepareAutoShardedWrites uses the provided logger if it logs internal warnings.
 function calculateFirestoreBytes(value) {
-    // ... same as before
     if (value === null) return 1;
     if (value === undefined) return 0;
     if (typeof value === 'boolean') return 1;
@@ -79,8 +83,6 @@ function calculateFirestoreBytes(value) {
 }
 
 async function prepareAutoShardedWrites(result, docRef, logger) {
-    // ... same logic, just ensure existing logs inside here use the logger properly if needed
-    // Copied from previous logic, essentially checks size > 900KB and splits
     const SAFETY_THRESHOLD_BYTES = 1000 * 1024; 
     const OVERHEAD_ALLOWANCE     = 20 * 1024;
     const CHUNK_LIMIT            = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
@@ -94,8 +96,6 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
 
     if ((totalSize + docPathSize) < CHUNK_LIMIT) {  const data = { ...result, _completed: true, _sharded: false };  return [{ ref: docRef, data, options: { merge: true } }];  }
 
-    // Note: We don't log "Sharding..." here anymore because we log the structured event in commitResults
-    
     for (const [key, value] of Object.entries(result)) { 
         if (key.startsWith('_')) continue;
         const keySize   = Buffer.byteLength(key, 'utf8') + 1;         

package/functions/generic-api/helpers/api_helpers.js

@@ -1,40 +1,121 @@
 /**
  * @fileoverview API sub-pipes.
- * REFACTORED: Now stateless and receive dependencies.
- * NEW: getDynamicSchema now reads static schema.
- * NEW: createManifestHandler filters out STALE schemas (>7 days old).
+ * REFACTORED: Implements Status-Based Availability Caching and Smart Date Resolution.
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
 
+// --- AVAILABILITY CACHE ---
+// Maintains a map of which computations are available on which dates.
+class AvailabilityCache {
+    constructor(db, logger, ttlMs = 5 * 60 * 1000) { // 5 Minute TTL
+        this.db = db;
+        this.logger = logger;
+        this.ttlMs = ttlMs;
+        this.cache = null; // { "calcName": ["2023-10-01", "2023-09-30"] } (Sorted DESC)
+        this.lastFetched = 0;
+        this.statusCollection = 'computation_status';
+    }
+
+    async getMap() {
+        const now = Date.now();
+        if (this.cache && (now - this.lastFetched < this.ttlMs)) {
+            return this.cache;
+        }
+
+        this.logger.log('INFO', '[AvailabilityCache] Refreshing availability map from Firestore...');
+        
+        // Fetch last 400 days of status to build the map
+        // We only fetch keys and small status objects, so this is relatively cheap.
+        const snapshot = await this.db.collection(this.statusCollection)
+            .orderBy(FieldPath.documentId(), 'desc')
+            .limit(400)
+            .get();
+
+        const newMap = {};
+
+        snapshot.forEach(doc => {
+            const dateStr = doc.id;
+            const statusData = doc.data();
+            
+            // Regex to validate date format YYYY-MM-DD
+            if (!/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return;
+
+            for (const [calcName, status] of Object.entries(statusData)) {
+                // We consider it available if status is truthy and NOT 'IMPOSSIBLE'
+                if (status && status !== 'IMPOSSIBLE') {
+                    if (!newMap[calcName]) newMap[calcName] = [];
+                    newMap[calcName].push(dateStr);
+                }
+            }
+        });
+
+        this.cache = newMap;
+        this.lastFetched = now;
+        this.logger.log('INFO', `[AvailabilityCache] Refreshed. Tracked ${Object.keys(newMap).length} computations.`);
+        return this.cache;
+    }
+}
 
 /**
- * Sub-pipe: pipe.api.helpers.validateRequest
+ * Helper: Resolve which dates to fetch based on mode and availability.
  */
-const validateRequest = (query, config) => {
-    if (!query.computations) return "Missing 'computations' parameter.";
-    if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
-    if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
-    
-    const start = new Date(query.startDate);
-    const end = new Date(query.endDate);
-    
-    if (end < start) return "'endDate' must be after 'startDate'.";
+async function resolveTargetDates(availabilityCache, computationKeys, mode, limit) {
+    const map = await availabilityCache.getMap();
+    const datesToFetch = new Set();
+
+    // 1. Identify all available dates for the requested computations
+    // We union the dates: if ANY requested calc is available on a date, we consider that date "relevant".
+    // (Alternatively, we could intersect, but union allows sparse data return).
+    const relevantDatesSet = new Set();
     
-    const maxDateRange = config.maxDateRange || 100;
-    const diffTime = Math.abs(end - start);
-    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
+    computationKeys.forEach(key => {
+        const dates = map[key] || [];
+        dates.forEach(d => relevantDatesSet.add(d));
+    });
+
+    // Sort descending
+    const sortedDates = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
+
+    if (sortedDates.length === 0) return [];
+
+    if (mode === 'latest') {
+        // Return only the most recent date found
+        return [sortedDates[0]];
+    } 
     
-    if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
+    if (mode === 'series') {
+        // Return the last N available dates
+        return sortedDates.slice(0, limit);
+    }
+
+    return [];
+}
+
+/**
+ * Sub-pipe: pipe.api.helpers.validateRequest
+ */
+const validateRequest = (query) => {
+    if (!query.computations) return "Missing 'computations' parameter.";
     
+    // New optional params, but computations is mandatory
+    const allowedModes = ['latest', 'series'];
+    if (query.mode && !allowedModes.includes(query.mode)) {
+        return "Invalid 'mode'. Must be 'latest' or 'series'.";
+    }
+
+    if (query.mode === 'series') {
+        const limit = parseInt(query.limit);
+        if (query.limit && (isNaN(limit) || limit < 1 || limit > 365)) {
+            return "Invalid 'limit'. Must be between 1 and 365.";
+        }
+    }
+
     return null;
 };
 
 /**
  * Sub-pipe: pipe.api.helpers.buildCalculationMap
- * --- CRITICAL UPDATE ---
- * This function now stores the class itself in the map,
- * which is required by the /manifest/generate endpoint.
  */
 const buildCalculationMap = (unifiedCalculations) => {
     const calcMap = {};
@@ -55,21 +136,8 @@ const buildCalculationMap = (unifiedCalculations) => {
 };
 
 /**
- * Internal helper for date strings.
- */
-const getDateStringsInRange = (startDate, endDate) => {
-    const dates = [];
-    const current = new Date(startDate + 'T00:00:00Z');
-    const end = new Date(endDate + 'T00:00:00Z');
-    while (current <= end) {
-        dates.push(current.toISOString().slice(0, 10));
-        current.setUTCDate(current.getUTCDate() + 1);
-    }
-    return dates;
-};
-
-/**
- * Sub-pipe: pipe.api.helpers.fetchData
+ * Sub-pipe: pipe.api.helpers.fetchUnifiedData
+ * UPDATED: Uses specific date list derived from availability.
  */
 const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
     const { db, logger } = dependencies;
@@ -78,11 +146,14 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
     const resultsSub = config.resultsSubcollection || 'results';
     const compsSub = config.computationsSubcollection || 'computations';
     
+    if (dateStrings.length === 0) return {};
+
     try {
+        const readPromises = [];
+
+        // Prepare all reads
         for (const date of dateStrings) {
-            response[date] = {};
-            const docRefs = [];
-            const keyPaths = [];
+            response[date] = {}; // Init structure
             
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
@@ -90,25 +161,34 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
                     const docRef = db.collection(insightsCollection).doc(date)
                         .collection(resultsSub).doc(pathInfo.category)
                         .collection(compsSub).doc(key);
-                    docRefs.push(docRef);
-                    keyPaths.push(key);
-                } else {
-                    logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
+                    
+                    readPromises.push({ date, key, ref: docRef });
                 }
             }
+        }
+        
+        if (readPromises.length === 0) return response;
+
+        // Batch reads in chunks (Firestore getAll limit is usually high, but let's be safe)
+        const CHUNK_SIZE = 100;
+        for (let i = 0; i < readPromises.length; i += CHUNK_SIZE) {
+            const chunk = readPromises.slice(i, i + CHUNK_SIZE);
+            const refs = chunk.map(item => item.ref);
             
-            if (docRefs.length === 0) continue;
+            const snapshots = await db.getAll(...refs);
             
-            const snapshots = await db.getAll(...docRefs);
-            snapshots.forEach((doc, i) => {
-                const key = keyPaths[i];
+            snapshots.forEach((doc, idx) => {
+                const { date, key } = chunk[idx];
                 if (doc.exists) {
                     response[date][key] = doc.data();
                 } else {
+                    // Start sparse: don't populate nulls to keep payload small?
+                    // Or populate null to indicate "checked but missing"
                     response[date][key] = null;
                 }
             });
         }
+
     } catch (error) {
         logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
         throw new Error('Failed to retrieve computation data.');
@@ -118,26 +198,59 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
 
 /**
  * Factory for the main API handler.
+ * UPDATED: Uses AvailabilityCache to determine dates.
  */
 const createApiHandler = (config, dependencies, calcMap) => {
-    const { logger } = dependencies; 
+    const { logger, db } = dependencies;
+    
+    // Instantiate Cache (Singleton scope within the closure)
+    const availabilityCache = new AvailabilityCache(db, logger);
+
     return async (req, res) => {
-        const validationError = validateRequest(req.query, config);
+        const validationError = validateRequest(req.query);
         if (validationError) {
             logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
             return res.status(400).send({ status: 'error', message: validationError });
         }
+        
         try {
             const computationKeys = req.query.computations.split(',');
-            const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
+            const mode = req.query.mode || 'latest'; // Default to latest
+            const limit = parseInt(req.query.limit) || 30; // Default 30 days for series
+
+            // 1. Resolve Dates based on Availability
+            const dateStrings = await resolveTargetDates(availabilityCache, computationKeys, mode, limit);
+
+            if (dateStrings.length === 0) {
+                 return res.status(200).send({
+                    status: 'success',
+                    metadata: {
+                        computations: computationKeys,
+                        mode,
+                        count: 0,
+                        dates: []
+                    },
+                    data: {}
+                });
+            }
+
+            // 2. Fetch Data for these dates
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
+            
+            // 3. Cleanup sparse dates (optional: remove dates where all requested keys are null)
+            // For now, we return what was fetched.
+
             res.set('Cache-Control', 'public, max-age=300, s-maxage=3600');
             res.status(200).send({
                 status: 'success',
                 metadata: {
                     computations: computationKeys,
-                    startDate: req.query.startDate,
-                    endDate: req.query.endDate,
+                    mode,
+                    limit: mode === 'series' ? limit : 1,
+                    dateRange: {
+                        start: dateStrings[dateStrings.length - 1], // Oldest
+                        end: dateStrings[0] // Newest
+                    }
                 },
                 data,
             });
@@ -148,6 +261,7 @@ const createApiHandler = (config, dependencies, calcMap) => {
     };
 };
 
+// ... (Previous Helper Functions: getComputationStructure, getDynamicSchema, createManifestHandler stay the same) ...
 /**
  * Internal helper for snippet generation.
  */
@@ -315,7 +429,6 @@ const createManifestHandler = (config, dependencies, calcMap) => {
     };
 };
 
-
 module.exports = {
     validateRequest,
     buildCalculationMap,

package/functions/generic-api/index.js

@@ -1,142 +1,190 @@
 /**
  * @fileoverview Main entry point for the Generic API module.
- * Exports the 'createApiApp' main pipe function.
- * REFACTORED: /manifest endpoint now reads static schemas from Firestore.
- * REFACTORED: /manifest/generate endpoint now reads static schema from class.
- *
- * --- MODIFIED: Added in-memory cache wrapper for the main API handler ---
+ * Export the 'createApiApp' main pipe function.
+ * REFACTORED: API V3 - Status-Aware Data Fetching.
  */
 
 const express = require('express');
 const cors = require('cors');
-const { FieldPath } = require('@google-cloud/firestore'); 
-const { buildCalculationMap, createApiHandler, getComputationStructure,createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
+const { buildCalculationMap, createApiHandler, getComputationStructure, createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
 
 /**
- * --- NEW: In-Memory Cache Handler ---
- * A wrapper function that adds a time-to-live (TTL) in-memory cache
- * to any Express request handler.
- * @param {function} handler - The original (req, res) handler to wrap.
- * @param {object} dependencies - { logger }
- * @returns {function} The new (req, res) handler with caching logic.
+ * In-Memory Cache Handler
+ * Wrapper that adds TTL cache to GET requests.
  */
 const createCacheHandler = (handler, { logger }) => {
-    // 1. Cache
     const CACHE = {};
-    const CACHE_TTL_MS = 10 * 60 * 1000; 
-    // 2. Return the new handler
-    return async (req, res) => { const cacheKey = req.url;  const now = Date.now();
-    // 3. --- Cache HIT ---
-    if (CACHE[cacheKey] && (now - CACHE[cacheKey].timestamp) < CACHE_TTL_MS) {
-    logger.log('INFO', `[API] Cache HIT for ${cacheKey}`);
-    return res.status(CACHE[cacheKey].status).send(CACHE[cacheKey].data); }
-    // 4. --- Cache MISS ---
-    logger.log('INFO', `[API] Cache MISS for ${cacheKey}`);
-    const originalSend = res.send;
-    const originalStatus = res.status;
-    let capturedData = null;
-    let capturedStatus = 200;
-    res.status = (statusCode) => { capturedStatus = statusCode; return originalStatus.call(res, statusCode); };
-    res.send = (data) => { capturedData = data; return originalSend.call(res, data); };
-    // 5. Call the original handler (which will now use our patched functions)
-    await handler(req, res);
-    // 6. If the response was successful, cache it
-    if (capturedStatus === 200 && capturedData) {
-    logger.log('INFO', `[API] Caching new entry for ${cacheKey}`);
-    CACHE[cacheKey] = { data: capturedData, status: capturedStatus, timestamp: now }; } };
+    const CACHE_TTL_MS = 10 * 60 * 1000; // 10 Minutes
+
+    return async (req, res) => {
+        // Cache Key now includes mode and limit
+        const cacheKey = req.url; 
+        const now = Date.now();
+
+        if (CACHE[cacheKey] && (now - CACHE[cacheKey].timestamp) < CACHE_TTL_MS) {
+            logger.log('INFO', `[API] Cache HIT for ${cacheKey}`);
+            return res.status(CACHE[cacheKey].status).send(CACHE[cacheKey].data);
+        }
+
+        logger.log('INFO', `[API] Cache MISS for ${cacheKey}`);
+        
+        const originalSend = res.send;
+        const originalStatus = res.status;
+        let capturedData = null;
+        let capturedStatus = 200;
+
+        res.status = (statusCode) => {
+            capturedStatus = statusCode;
+            return originalStatus.call(res, statusCode);
+        };
+        
+        res.send = (data) => {
+            capturedData = data;
+            return originalSend.call(res, data);
+        };
+
+        await handler(req, res);
+
+        if (capturedStatus === 200 && capturedData) {
+            logger.log('INFO', `[API] Caching new entry for ${cacheKey}`);
+            CACHE[cacheKey] = {
+                data: capturedData,
+                status: capturedStatus,
+                timestamp: now
+            };
+        }
+    };
 };
 
 
 /**
  * Main pipe: pipe.api.createApiApp
- * Creates and configures the Express app for the Generic API.
- * @param {object} config - The Generic API V2 configuration object.
- * @param {object} dependencies - Shared dependencies { db, logger }.
- * @param {Object} unifiedCalculations - The calculations manifest from 'aiden-shared-calculations-unified'.
- * @returns {express.Application} The configured Express app.
  */
 function createApiApp(config, dependencies, unifiedCalculations) {
     const app = express();
     const { logger, db } = dependencies;
     
-    // --- Pre-compute Calculation Map (now includes classes) ---
+    // Build Calc Map once
     const calcMap = buildCalculationMap(unifiedCalculations);
 
-    // --- Middleware ---
+    // Middleware
     app.use(cors({ origin: true }));
     app.use(express.json());
 
-    // --- Main API Endpoint ---
+    // --- Main API V3 Endpoint ---
+    // createApiHandler now initializes the AvailabilityCache internally
     const originalApiHandler = createApiHandler(config, dependencies, calcMap);
     const cachedApiHandler = createCacheHandler(originalApiHandler, dependencies);
+    
+    // This handler now supports ?mode=latest and ?mode=series&limit=X
     app.get('/', cachedApiHandler); 
 
-    // --- Health Check Endpoint ---
+    // Health Check
     app.get('/health', (req, res) => { res.status(200).send('OK'); });
 
-    // --- Debug Endpoint to list all computation keys ---
+    // Debug: List keys
     app.get('/list-computations', (req, res) => {
-        try { const computationKeys = Object.keys(calcMap); 
-        res.status(200).send({ status: 'success', count: computationKeys.length, computations: computationKeys.sort(), });
-        } catch (error) { logger.log('ERROR', 'API /list-computations failed.', { errorMessage: error.message });
-        res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } });
+        try {
+            const computationKeys = Object.keys(calcMap); 
+            res.status(200).send({ 
+                status: 'success', 
+                count: computationKeys.length, 
+                computations: computationKeys.sort(), 
+            });
+        } catch (error) { 
+            logger.log('ERROR', 'API /list-computations failed.', { errorMessage: error.message });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
+        } 
+    });
 
-    // --- Debug Endpoint to get *stored* structure from Firestore ---
+    // Structure Inspection
     app.get('/structure/:computationName', async (req, res) => {
         const { computationName } = req.params;
         const result = await getComputationStructure(computationName, calcMap, config, dependencies);
-        if (result.status === 'error') { const statusCode = result.message.includes('not found') ? 404 : 500;
-        return res.status(statusCode).send(result); }
-        res.status(200).send(result); });
+        if (result.status === 'error') { 
+            const statusCode = result.message.includes('not found') ? 404 : 500;
+            return res.status(statusCode).send(result); 
+        }
+        res.status(200).send(result); 
+    });
+
+    // Manifests (Schema Generation)
     app.get('/manifest', createManifestHandler(config, dependencies, calcMap));
+    
+    // Manual Schema Gen Trigger
     app.post('/manifest/generate/:computationName', async (req, res) => {
         const { computationName } = req.params;
         logger.log('INFO', `Manual static schema generation requested for: ${computationName}`);
 
         try {
-            // 1. Find the calculation class from the calcMap
             const calcInfo = calcMap[computationName];
-            if (!calcInfo || !calcInfo.class) { return res.status(404).send({ status: 'error', message: `Computation '${computationName}' not found or has no class in calculation map.` }); }
+            if (!calcInfo || !calcInfo.class) { 
+                return res.status(404).send({ status: 'error', message: `Computation '${computationName}' not found.` }); 
+            }
             const targetCalcClass = calcInfo.class;
             const targetCategory = calcInfo.category;
 
-            // 2. Use the getDynamicSchema helper (which now just reads the static method)
             const schemaStructure = await getDynamicSchema(targetCalcClass, computationName);
-            if (schemaStructure.ERROR) { return res.status(400).send({ status: 'error', message: `Failed to get static schema: ${schemaStructure.ERROR}` }); }
+            if (schemaStructure.ERROR) { 
+                return res.status(400).send({ status: 'error', message: `Failed to get static schema: ${schemaStructure.ERROR}` }); 
+            }
             
-            // 3. Import the new batchStoreSchemas utility
             const { batchStoreSchemas } = require('../computation-system/utils/schema_capture.js');
 
-            // 4. Get metadata (as much as we can from the class)
-            const metadata = { isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')), dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [], rootDataDependencies: [],  pass: 'unknown',  type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard', note: "Manually generated via API" };
-
-            // 5. Store the schema in Firestore
-            await batchStoreSchemas( dependencies, config, [{ name: computationName, category: targetCategory, schema: schemaStructure, metadata: metadata }] );
-
-            // 6. Respond with the schema
-            res.status(200).send({ status: 'success', message: `Static schema read and stored for ${computationName}`, computation: computationName, category: targetCategory, schema: schemaStructure });
+            const metadata = { 
+                isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')), 
+                dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [], 
+                rootDataDependencies: [],  
+                type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard', 
+                note: "Manually generated via API" 
+            };
+
+            await batchStoreSchemas(dependencies, config, [{ 
+                name: computationName, 
+                category: targetCategory, 
+                schema: schemaStructure, 
+                metadata: metadata 
+            }]);
+
+            res.status(200).send({ 
+                status: 'success', 
+                message: `Static schema read and stored for ${computationName}`, 
+                computation: computationName, 
+                category: targetCategory, 
+                schema: schemaStructure 
+            });
 
         } catch (error) {
-            logger.log('ERROR', `Failed to generate schema for ${computationName}`, { errorMessage: error.message, stack: error.stack });
-            res.status(5.00).send({ status: 'error', message: `Failed to generate/store schema: ${error.message}` });
+            logger.log('ERROR', `Failed to generate schema for ${computationName}`, { errorMessage: error.message });
+            res.status(500).send({ status: 'error', message: `Failed: ${error.message}` });
         }
     });
 
-    /**
-     * This endpoint is fine as-is. It reads from the Firestore
-     * collection that the /manifest and /manifest/generate routes populate.
-     */
+    // Single Manifest Get
     app.get('/manifest/:computationName', async (req, res) => {
         const { computationName } = req.params;
         try {
             const schemaCollection = config.schemaCollection || 'computation_schemas';
             const schemaDoc = await db.collection(schemaCollection).doc(computationName).get();
-            if (!schemaDoc.exists) { return res.status(4404).send({ status: 'error', message: `Schema not found for computation: ${computationName}`, hint: 'Try running the computation system or use POST /manifest/generate/:computationName' }); }
+            if (!schemaDoc.exists) { 
+                return res.status(404).send({ status: 'error', message: `Schema not found for ${computationName}` }); 
+            }
             const data = schemaDoc.data();            
-            res.status(200).send({ status: 'success', computation: computationName, category: data.category, structure: data.schema, metadata: data.metadata || {}, lastUpdated: data.lastUpdated });
-        } catch (error) { logger.log('ERROR', `Failed to fetch schema for ${computationName}`, { errorMessage: error.message }); res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); }});
+            res.status(200).send({ 
+                status: 'success', 
+                computation: computationName, 
+                category: data.category, 
+                structure: data.schema, 
+                metadata: data.metadata || {}, 
+                lastUpdated: data.lastUpdated 
+            });
+        } catch (error) { 
+            logger.log('ERROR', `Failed to fetch schema for ${computationName}`, { errorMessage: error.message }); 
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
+        }
+    });
+    
     return app;
 }
 
-
 module.exports = { createApiApp, helpers: require('./helpers/api_helpers.js') };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.230",
+  "version": "1.0.232",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system/controllers/computation_controller.js

@@ -1,146 +0,0 @@
-/**
- * FIXED: computation_controller.js
- * V5.1: Exports LEGACY_MAPPING for Manifest Builder
- */
-
-// Load all layers dynamically from the index
-const mathLayer                                                                                   = require('../layers/index');
-const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
-
-// Legacy Keys Mapping (Ensures backward compatibility with existing Calculations)
-// Maps the new modular class names to the property names expected by existing code (e.g. math.extract)
-const LEGACY_MAPPING = {
-    DataExtractor: 'extract',
-    HistoryExtractor: 'history',
-    MathPrimitives: 'compute',
-    Aggregators: 'aggregate',
-    Validators: 'validate',
-    SignalPrimitives: 'signals',
-    SCHEMAS: 'schemas',
-    DistributionAnalytics: 'distribution',
-    TimeSeries: 'TimeSeries',
-    priceExtractor: 'priceExtractor',
-    InsightsExtractor: 'insights',
-    UserClassifier: 'classifier',
-    Psychometrics: 'psychometrics',
-    CognitiveBiases: 'bias',
-    SkillAttribution: 'skill',
-    ExecutionAnalytics: 'execution',
-    AdaptiveAnalytics: 'adaptive'
-};
-
-class DataLoader {
-    constructor(config, dependencies)     { this.config = config; this.deps = dependencies; this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null }; }
-    get mappings()                        { return this.cache.mappings; }
-    async loadMappings()                  { if (this.cache.mappings) return this.cache.mappings; const { calculationUtils } = this.deps; this.cache.mappings = await calculationUtils.loadInstrumentMappings(); return this.cache.mappings; }
-    async loadInsights(dateStr)           { if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); const insights = await loadDailyInsights(this.config, this.deps, dateStr); this.cache.insights.set(dateStr, insights); return insights; }
-    async loadSocial(dateStr)             { if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); const social = await loadDailySocialPostInsights(this.config, this.deps, dateStr); this.cache.social.set(dateStr, social); return social; }
-    async getPriceShardReferences()       { return getPriceShardRefs(this.config, this.deps); }
-    async getSpecificPriceShardReferences (targetInstrumentIds) { return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); }
-    async loadPriceShard(docRef)          { try { const snap = await docRef.get(); if (!snap.exists) return {}; return snap.data(); } catch (e) { console.error(`Error loading shard ${docRef.path}:`, e); return {}; } }
-}
-
-class ContextBuilder { 
-    static buildMathContext() {
-        const mathContext = {};
-        for (const [key, value] of Object.entries(mathLayer)) { mathContext[key] = value; const legacyKey = LEGACY_MAPPING[key]; if (legacyKey) { mathContext[legacyKey] = value; } }
-        return mathContext;
-    }
-    static buildPerUserContext(options) {
-        const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
-        return {
-            user: { id: userId, type: userType, portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio }, history: { today: todayHistory, yesterday: yesterdayHistory } },
-            date: { today: dateStr },
-            insights: { today: insights?.today, yesterday: insights?.yesterday },
-            social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            mappings: mappings || {},
-            math: ContextBuilder.buildMathContext(), 
-            computed: computedDependencies || {},
-            previousComputed: previousComputedDependencies || {},
-            meta: metadata, config, deps
-        };
-    }
-
-    static buildMetaContext(options) {
-        const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
-        return {
-            date: { today: dateStr },
-            insights: { today: insights?.today, yesterday: insights?.yesterday },
-            social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            prices: prices || {},
-            mappings: mappings || {},
-            math: ContextBuilder.buildMathContext(), 
-            computed: computedDependencies || {},
-            previousComputed: previousComputedDependencies || {},
-            meta: metadata, config, deps
-        };
-    }
-}
-
-class ComputationExecutor {
-    constructor(config, dependencies, dataLoader) {
-        this.config = config;
-        this.deps   = dependencies;
-        this.loader = dataLoader;
-    }
-
-    async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps) {
-        const { logger }     = this.deps;
-        const targetUserType = metadata.userType; 
-        const mappings       = await this.loader.loadMappings();
-        const insights       = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
-        
-        // Access SCHEMAS dynamically from the loaded layer
-        const SCHEMAS = mathLayer.SCHEMAS;
-
-        for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
-            const todayHistory       = historyData ? historyData[userId] : null;
-            const actualUserType     = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-            if (targetUserType !== 'all') {
-                 const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-                 if (mappedTarget !== actualUserType) continue;
-            }
-            const context = ContextBuilder.buildPerUserContext({ todayPortfolio, yesterdayPortfolio, todayHistory, userId, userType: actualUserType, dateStr, metadata, mappings, insights, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-            try { await calcInstance.process(context); } catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
-        }
-    }
-
-    async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
-        const mappings   = await this.loader.loadMappings();
-        const { logger } = this.deps;
-        const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
-        const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await this.loader.loadSocial(dateStr)   } : null;
-        
-        if (metadata.rootDataDependencies?.includes('price')) {
-            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
-            const shardRefs = await this.loader.getPriceShardReferences();
-            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
-            let processedCount = 0;
-            for (const ref of shardRefs) {
-                const shardData      = await this.loader.loadPriceShard(ref);
-                const partialContext = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: { history: shardData }, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-                await calcInstance.process(partialContext);
-                partialContext.prices = null; 
-                processedCount++;
-                if (processedCount % 10 === 0) { if (global.gc) { global.gc(); } }
-            }
-            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
-            return calcInstance.getResult ? await calcInstance.getResult() : {};
-        } else {
-            const context = ContextBuilder.buildMetaContext({ dateStr, metadata, mappings, insights, socialData: social, prices: {}, computedDependencies: computedDeps, previousComputedDependencies: prevDeps, config: this.config, deps: this.deps });
-            return await calcInstance.process(context);
-        }
-    }
-}
-
-class ComputationController {
-    constructor(config, dependencies) {
-        this.config   = config;
-        this.deps     = dependencies;
-        this.loader   = new DataLoader(config, dependencies);
-        this.executor = new ComputationExecutor(config, dependencies, this.loader);
-    }
-}
-
-module.exports = { ComputationController, LEGACY_MAPPING };