bulltrackers-module 1.0.629 → 1.0.631

package/functions/old-generic-api/helpers/api_helpers.js

@@ -1,457 +0,0 @@
-/**
- * @fileoverview API sub-pipes.
- * REFACTORED: API V3 - Status-Aware Data Fetching.
- * UPDATED: Added GZIP Decompression support for fetching compressed results.
- */
-
-const { FieldPath } = require('@google-cloud/firestore');
-const zlib = require('zlib'); // [NEW] Required for decompression
-
-// --- HELPER: DECOMPRESSION ---
-/**
- * Checks if data is compressed and inflates it if necessary.
- * @param {object} data - The raw Firestore document data.
- * @returns {object} The original (decompressed) JSON object.
- */
-function tryDecompress(data) {
-    if (data && data._compressed === true && data.payload) {
-        try {
-            // Firestore returns Buffers automatically for Blob types
-            return JSON.parse(zlib.gunzipSync(data.payload).toString());
-        } catch (e) {
-            console.error('[API] Decompression failed:', e);
-            // Return empty object or original data on failure to avoid crashing response
-            return {};
-        }
-    }
-    return data;
-}
-
-// --- AVAILABILITY CACHE ---
-class AvailabilityCache {
-    constructor(db, logger, ttlMs = 5 * 60 * 1000) { // 5 Minute TTL
-        this.db = db;
-        this.logger = logger;
-        this.ttlMs = ttlMs;
-        this.cache = null; 
-        this.lastFetched = 0;
-        this.statusCollection = 'computation_status';
-    }
-
-    async getMap() {
-        const now = Date.now();
-        if (this.cache && (now - this.lastFetched < this.ttlMs)) {
-            return this.cache;
-        }
-
-        this.logger.log('INFO', '[AvailabilityCache] Refreshing availability map from Firestore...');
-        
-        // Fetch recent status to build the map (Limit 400 days)
-        const snapshot = await this.db.collection(this.statusCollection)
-            .orderBy(FieldPath.documentId(), 'desc')
-            .limit(400)
-            .get();
-
-        const newMap = {};
-
-        snapshot.forEach(doc => {
-            const dateStr = doc.id;
-            const statusData = doc.data();
-            
-            if (!/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return;
-
-            for (const [calcName, status] of Object.entries(statusData)) {
-                if (status && status !== 'IMPOSSIBLE') {
-                    if (!newMap[calcName]) newMap[calcName] = [];
-                    newMap[calcName].push(dateStr);
-                }
-            }
-        });
-
-        this.cache = newMap;
-        this.lastFetched = now;
-        this.logger.log('INFO', `[AvailabilityCache] Refreshed. Tracked ${Object.keys(newMap).length} computations.`);
-        return this.cache;
-    }
-}
-
-/**
- * Helper: Resolve which dates to fetch based on mode and availability.
- * UPDATED: Added "Inferred Availability" for backfilled computations (user-history-reconstructor).
- */
-async function resolveTargetDates(availabilityCache, computationKeys, mode, limit) {
-    const map = await availabilityCache.getMap();
-    const relevantDatesSet = new Set();
-    
-    // [NEW] Special Handling for Backfilled Computations
-    // These computations write to past dates but only update status for the current run date.
-    // If we see a recent status, we infer that history exists backward from that point.
-    const BACKFILL_COMPUTATIONS = ['user-history-reconstructor'];
-    const hasBackfillCalc = computationKeys.some(k => BACKFILL_COMPUTATIONS.includes(k));
-
-    computationKeys.forEach(key => {
-        const dates = map[key] || [];
-        dates.forEach(d => relevantDatesSet.add(d));
-    });
-
-    // If we are requesting a backfill-capable computation and have at least one valid date
-    if (hasBackfillCalc && relevantDatesSet.size > 0 && mode === 'series') {
-        const sortedExisting = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
-        const anchorDate = sortedExisting[0]; // Latest available date acts as the anchor
-
-        // Generate 'limit' days backwards from the anchor
-        const anchorTime = new Date(anchorDate).getTime();
-        for (let i = 0; i < limit; i++) {
-            const d = new Date(anchorTime - (i * 86400000));
-            const dateStr = d.toISOString().slice(0, 10);
-            relevantDatesSet.add(dateStr);
-        }
-    }
-
-    // Standard sorting and slicing
-    const sortedDates = Array.from(relevantDatesSet).sort((a, b) => b.localeCompare(a));
-
-    if (sortedDates.length === 0) return [];
-
-    if (mode === 'latest') {
-        return [sortedDates[0]];
-    } 
-    
-    if (mode === 'series') {
-        return sortedDates.slice(0, limit);
-    }
-
-    return [];
-}
-
-/**
- * Sub-pipe: pipe.api.helpers.validateRequest
- */
-const validateRequest = (query) => {
-    if (!query.computations) return "Missing 'computations' parameter.";
-    
-    const allowedModes = ['latest', 'series'];
-    if (query.mode && !allowedModes.includes(query.mode)) {
-        return "Invalid 'mode'. Must be 'latest' or 'series'.";
-    }
-
-    if (query.mode === 'series') {
-        const limit = parseInt(query.limit);
-        if (query.limit && (isNaN(limit) || limit < 1 || limit > 365)) {
-            return "Invalid 'limit'. Must be between 1 and 365.";
-        }
-    }
-
-    return null;
-};
-
-/**
- * Sub-pipe: pipe.api.helpers.buildCalculationMap
- * FIX APPLIED: Checks class metadata for category override (mirroring ManifestBuilder).
- */
-const buildCalculationMap = (unifiedCalculations) => {
-    const calcMap = {};
-
-    // Helper to resolve category exactly like ManifestBuilder.js does
-    const resolveCategory = (cls, folderName) => {
-        if (typeof cls.getMetadata === 'function') {
-            const meta = cls.getMetadata();
-            // If folder is 'core', allow metadata to override
-            if (folderName === 'core' && meta.category) {
-                return meta.category;
-            }
-        }
-        return folderName;
-    };
-
-    for (const category in unifiedCalculations) {
-        for (const subKey in unifiedCalculations[category]) {
-            const item = unifiedCalculations[category][subKey];
-            
-            if (subKey === 'historical' && typeof item === 'object') {
-                for (const calcName in item) {
-                    const cls = item[calcName];
-                    calcMap[calcName] = { 
-                        category: resolveCategory(cls, category), 
-                        class: cls 
-                    };
-                }
-            } else if (typeof item === 'function') {
-                const calcName = subKey;
-                calcMap[calcName] = { 
-                    category: resolveCategory(item, category), 
-                    class: item 
-                };
-            }
-        }
-    }
-    return calcMap;
-};
-
-/**
- * Sub-pipe: pipe.api.helpers.fetchUnifiedData
- * UPDATED: Uses tryDecompress to handle compressed payloads.
- */
-const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, calcMap) => {
-    const { db, logger } = dependencies;
-    const response = {};
-    const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
-    const resultsSub = config.resultsSubcollection || 'results';
-    const compsSub = config.computationsSubcollection || 'computations';
-    
-    if (dateStrings.length === 0) return {};
-
-    try {
-        const readPromises = [];
-
-        for (const date of dateStrings) {
-            response[date] = {}; 
-            
-            for (const key of calcKeys) {
-                const pathInfo = calcMap[key];
-                if (pathInfo) {
-                    const docRef = db.collection(insightsCollection).doc(date)
-                        .collection(resultsSub).doc(pathInfo.category)
-                        .collection(compsSub).doc(key);
-                    
-                    readPromises.push({ date, key, ref: docRef });
-                } else {
-                    // If no path info, we can't fetch.
-                    response[date][key] = null;
-                }
-            }
-        }
-        
-        if (readPromises.length === 0) return response;
-
-        // Batch reads
-        const CHUNK_SIZE = 100;
-        for (let i = 0; i < readPromises.length; i += CHUNK_SIZE) {
-            const chunk = readPromises.slice(i, i + CHUNK_SIZE);
-            const refs = chunk.map(item => item.ref);
-            
-            const snapshots = await db.getAll(...refs);
-            
-            snapshots.forEach((doc, idx) => {
-                const { date, key } = chunk[idx];
-                if (doc.exists) {
-                    // [UPDATED] Decompress data if needed
-                    response[date][key] = tryDecompress(doc.data());
-                } else {
-                    response[date][key] = null;
-                }
-            });
-        }
-
-    } catch (error) {
-        logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
-        throw new Error('Failed to retrieve computation data.');
-    }
-    return response;
-};
-
-/**
- * Factory for the main API handler.
- */
-const createApiHandler = (config, dependencies, calcMap) => {
-    const { logger, db } = dependencies;
-    
-    // Singleton Cache
-    const availabilityCache = new AvailabilityCache(db, logger);
-
-    return async (req, res) => {
-        const validationError = validateRequest(req.query);
-        if (validationError) {
-            logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
-            return res.status(400).send({ status: 'error', message: validationError });
-        }
-        
-        try {
-            const computationKeys = req.query.computations.split(',');
-            const mode = req.query.mode || 'latest'; 
-            const limit = parseInt(req.query.limit) || 30;
-
-            // 1. Resolve Dates
-            const dateStrings = await resolveTargetDates(availabilityCache, computationKeys, mode, limit);
-
-            if (dateStrings.length === 0) {
-                 return res.status(200).send({
-                    status: 'success',
-                    metadata: {
-                        computations: computationKeys,
-                        mode,
-                        count: 0,
-                        dates: []
-                    },
-                    data: {}
-                });
-            }
-
-            // 2. Fetch Data
-            const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
-            
-            res.set('Cache-Control', 'public, max-age=300, s-maxage=3600');
-            res.status(200).send({
-                status: 'success',
-                metadata: {
-                    computations: computationKeys,
-                    mode,
-                    limit: mode === 'series' ? limit : 1,
-                    dateRange: {
-                        start: dateStrings[dateStrings.length - 1], 
-                        end: dateStrings[0]
-                    }
-                },
-                data,
-            });
-        } catch (error) {
-            logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack });
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
-        }
-    };
-};
-
-function createStructureSnippet(data, maxKeys = 20) {
-    if (data === null || typeof data !== 'object') {
-        if (typeof data === 'number') return 0;
-        if (typeof data === 'string') return "string";
-        if (typeof data === 'boolean') return true;
-        return data;
-    }
-    if (Array.isArray(data)) {
-        if (data.length === 0) return "<empty array>";
-        return [ createStructureSnippet(data[0], maxKeys) ];
-    }
-    const newObj = {};
-    const keys = Object.keys(data);
-    
-    if (keys.length > 0 && keys.every(k => k.match(/^[A-Z.]+$/) || k.includes('_') || k.match(/^[0-9]+$/))) {
-        const exampleKey = keys[0];
-        newObj[exampleKey] = createStructureSnippet(data[exampleKey], maxKeys);
-        newObj["... (more items)"] = "...";
-        return newObj;
-    }
-    
-    if (keys.length > maxKeys) {
-        const firstKey = keys[0] || "example_key";
-        newObj[firstKey] = createStructureSnippet(data[firstKey], maxKeys);
-        newObj[`... (${keys.length - 1} more keys)`] = "<object>";
-    } else {
-        for (const key of keys) {
-            newObj[key] = createStructureSnippet(data[key], maxKeys);
-        }
-    }
-    return newObj;
-}
-
-async function getComputationStructure(computationName, calcMap, config, dependencies) {
-    const { db, logger } = dependencies;
-    try {
-        const pathInfo = calcMap[computationName];
-        if (!pathInfo) { return { status: 'error', computation: computationName, message: `Computation not found in calculation map.` }; }
-        const { category } = pathInfo;
-        const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
-        const resultsSub = config.resultsSubcollection || 'results';
-        const compsSub = config.computationsSubcollection || 'computations';
-        const computationQueryPath = `${category}.${computationName}`;
-        const dateQuery = db.collection(insightsCollection)
-            .where(computationQueryPath, '==', true)
-            .orderBy(FieldPath.documentId(), 'desc')
-            .limit(1);
-        const dateSnapshot = await dateQuery.get();
-        if (dateSnapshot.empty) { return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` }; }
-        const latestStoredDate = dateSnapshot.docs[0].id;
-        const docRef = db.collection(insightsCollection).doc(latestStoredDate)
-            .collection(resultsSub).doc(category)
-            .collection(compsSub).doc(computationName);
-        const doc = await docRef.get();
-        if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
-        
-        // [UPDATED] Decompress data for structure inspection
-        const fullData = tryDecompress(doc.data());
-        const structureSnippet = createStructureSnippet(fullData);
-        
-        return { status: 'success', computation: computationName, category: category, latestStoredDate: latestStoredDate, structureSnippet: structureSnippet, };
-    } catch (error) {
-        logger.log('ERROR', `API /structure/${computationName} helper failed.`, { errorMessage: error.message });
-        return { status: 'error', computation: computationName, message: error.message };
-    }
-}
-
-async function getDynamicSchema(CalcClass, calcName) {
-    if (CalcClass && typeof CalcClass.getSchema === 'function') {
-        try {
-            return CalcClass.getSchema();
-        } catch (e) {
-            console.error(`Error running static getSchema() for ${calcName}: ${e.message}`);
-            return { "ERROR": `Failed to get static schema: ${e.message}` };
-        }
-    } else {
-        return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` };
-    }
-}
-
-const createManifestHandler = (config, dependencies, calcMap) => {
-    const { db, logger } = dependencies;
-    const schemaCollection = config.schemaCollection || 'computation_schemas';
-    
-    return async (req, res) => {
-        try {
-            const snapshot = await db.collection(schemaCollection).get();
-            if (snapshot.empty) { return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' }); }
-
-            const manifest = {};
-            const now = Date.now();
-            const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; 
-            
-            let activeCount = 0;
-            let staleCount = 0;
-
-            snapshot.forEach(doc => {
-                const data = doc.data();
-                let lastUpdatedMs = 0;
-                if (data.lastUpdated && typeof data.lastUpdated.toMillis === 'function') {
-                    lastUpdatedMs = data.lastUpdated.toMillis();
-                } else if (data.lastUpdated instanceof Date) {
-                    lastUpdatedMs = data.lastUpdated.getTime();
-                }
-
-                if ((now - lastUpdatedMs) < MAX_AGE_MS) {
-                    manifest[doc.id] = {
-                        category: data.category,
-                        structure: data.schema,
-                        metadata: data.metadata,
-                        lastUpdated: data.lastUpdated
-                    };
-                    activeCount++;
-                } else { staleCount++; }
-            });
-            
-            res.status(200).send({
-                status: 'success',
-                summary: {
-                    source: 'firestore_computation_schemas',
-                    totalComputations: snapshot.size,
-                    schemasAvailable: activeCount,
-                    schemasFiltered: staleCount,
-                    lastUpdated: Date.now()
-                },
-                manifest: manifest
-            });
-
-        } catch (error) {
-            logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message });
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
-        }
-    };
-};
-
-module.exports = {
-    validateRequest,
-    buildCalculationMap,
-    fetchUnifiedData,
-    createApiHandler,
-    getComputationStructure,
-    getDynamicSchema,
-    createManifestHandler 
-};

package/functions/old-generic-api/index.js

@@ -1,204 +0,0 @@
-/**
- * @fileoverview Main entry point for the Generic API module.
- * Export the 'createApiApp' main pipe function.
- * REFACTORED: API V3 - Status-Aware Data Fetching.
- * UPDATED: Mounts both Admin API and User API.
- */
-
-const express = require('express');
-const cors = require('cors');
-const { buildCalculationMap, createApiHandler, getComputationStructure, createManifestHandler, getDynamicSchema } = require('./helpers/api_helpers.js');
-
-// Import Sub-Routers
-const createAdminRouter = require('./admin-api/index');
-const createUserRouter  = require('./user-api/index'); // [FIX] Import User Router
-
-/**
- * In-Memory Cache Handler
- * Wrapper that adds TTL cache to GET requests.
- */
-const createCacheHandler = (handler, { logger }) => {
-    const CACHE = {};
-    const CACHE_TTL_MS = 10 * 60 * 1000; // 10 Minutes
-
-    return async (req, res) => {
-        const cacheKey = req.url; 
-        const now = Date.now();
-
-        if (CACHE[cacheKey] && (now - CACHE[cacheKey].timestamp) < CACHE_TTL_MS) {
-            logger.log('INFO', `[API] Cache HIT for ${cacheKey}`);
-            return res.status(CACHE[cacheKey].status).send(CACHE[cacheKey].data);
-        }
-
-        logger.log('INFO', `[API] Cache MISS for ${cacheKey}`);
-        
-        const originalSend = res.send;
-        const originalStatus = res.status;
-        let capturedData = null;
-        let capturedStatus = 200;
-
-        res.status = (statusCode) => {
-            capturedStatus = statusCode;
-            return originalStatus.call(res, statusCode);
-        };
-        
-        res.send = (data) => {
-            capturedData = data;
-            return originalSend.call(res, data);
-        };
-
-        await handler(req, res);
-
-        if (capturedStatus === 200 && capturedData) {
-            logger.log('INFO', `[API] Caching new entry for ${cacheKey}`);
-            CACHE[cacheKey] = {
-                data: capturedData,
-                status: capturedStatus,
-                timestamp: now
-            };
-        }
-    };
-};
-
-/**
- * Main pipe: pipe.api.createApiApp
- */
-function createApiApp(config, dependencies, unifiedCalculations) {
-    const app = express();
-    const { logger, db } = dependencies;
-    
-    // Build Calc Map once
-    const calcMap = buildCalculationMap(unifiedCalculations);
-
-    // Middleware
-    app.use(cors({ origin: true }));
-    app.use(express.json());
-
-    // --- MOUNT SUB-ROUTERS ---
-    
-    // 1. Admin API
-    // Signature: (config, dependencies, calculations)
-    app.use('/admin', createAdminRouter(config, dependencies, unifiedCalculations));
-
-    // 2. User API [FIX]
-    // Signature: (dependencies, config) - Matching the definition from Phase 4
-    if (createUserRouter) {
-        app.use('/user', createUserRouter(dependencies, config));
-        logger.log('INFO', '[API] Mounted /user routes');
-    }
-
-    // --- Main API V3 Endpoint ---
-    const originalApiHandler = createApiHandler(config, dependencies, calcMap);
-    const cachedApiHandler = createCacheHandler(originalApiHandler, dependencies);
-    
-    app.get('/', cachedApiHandler); 
-
-    // Health Check
-    app.get('/health', (req, res) => { res.status(200).send('OK'); });
-
-    // Debug: List keys
-    app.get('/list-computations', (req, res) => {
-        try {
-            const computationKeys = Object.keys(calcMap); 
-            res.status(200).send({ 
-                status: 'success', 
-                count: computationKeys.length, 
-                computations: computationKeys.sort(), 
-            });
-        } catch (error) { 
-            logger.log('ERROR', 'API /list-computations failed.', { errorMessage: error.message });
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
-        } 
-    });
-
-    // Structure Inspection
-    app.get('/structure/:computationName', async (req, res) => {
-        const { computationName } = req.params;
-        const result = await getComputationStructure(computationName, calcMap, config, dependencies);
-        if (result.status === 'error') { 
-            const statusCode = result.message.includes('not found') ? 404 : 500;
-            return res.status(statusCode).send(result); 
-        }
-        res.status(200).send(result); 
-    });
-
-    // Manifests (Schema Generation)
-    app.get('/manifest', createManifestHandler(config, dependencies, calcMap));
-    
-    // Manual Schema Gen Trigger
-    app.post('/manifest/generate/:computationName', async (req, res) => {
-        const { computationName } = req.params;
-        logger.log('INFO', `Manual static schema generation requested for: ${computationName}`);
-
-        try {
-            const calcInfo = calcMap[computationName];
-            if (!calcInfo || !calcInfo.class) { 
-                return res.status(404).send({ status: 'error', message: `Computation '${computationName}' not found.` }); 
-            }
-            const targetCalcClass = calcInfo.class;
-            const targetCategory = calcInfo.category;
-
-            const schemaStructure = await getDynamicSchema(targetCalcClass, computationName);
-            if (schemaStructure.ERROR) { 
-                return res.status(400).send({ status: 'error', message: `Failed to get static schema: ${schemaStructure.ERROR}` }); 
-            }
-            
-            const { batchStoreSchemas } = require('../computation-system/utils/schema_capture.js');
-
-            const metadata = { 
-                isHistorical: !!(targetCalcClass.toString().includes('yesterdayPortfolio')), 
-                dependencies: (typeof targetCalcClass.getDependencies === 'function') ? targetCalcClass.getDependencies() : [], 
-                rootDataDependencies: [],  
-                type: (targetCategory === 'meta' || targetCategory === 'socialPosts') ? targetCategory : 'standard', 
-                note: "Manually generated via API" 
-            };
-
-            await batchStoreSchemas(dependencies, config, [{ 
-                name: computationName, 
-                category: targetCategory, 
-                schema: schemaStructure, 
-                metadata: metadata 
-            }]);
-
-            res.status(200).send({ 
-                status: 'success', 
-                message: `Static schema read and stored for ${computationName}`, 
-                computation: computationName, 
-                category: targetCategory, 
-                schema: schemaStructure 
-            });
-
-        } catch (error) {
-            logger.log('ERROR', `Failed to generate schema for ${computationName}`, { errorMessage: error.message });
-            res.status(500).send({ status: 'error', message: `Failed: ${error.message}` });
-        }
-    });
-
-    // Single Manifest Get
-    app.get('/manifest/:computationName', async (req, res) => {
-        const { computationName } = req.params;
-        try {
-            const schemaCollection = config.schemaCollection || 'computation_schemas';
-            const schemaDoc = await db.collection(schemaCollection).doc(computationName).get();
-            if (!schemaDoc.exists) { 
-                return res.status(404).send({ status: 'error', message: `Schema not found for ${computationName}` }); 
-            }
-            const data = schemaDoc.data();            
-            res.status(200).send({ 
-                status: 'success', 
-                computation: computationName, 
-                category: data.category, 
-                structure: data.schema, 
-                metadata: data.metadata || {}, 
-                lastUpdated: data.lastUpdated 
-            });
-        } catch (error) { 
-            logger.log('ERROR', `Failed to fetch schema for ${computationName}`, { errorMessage: error.message }); 
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); 
-        }
-    });
-    
-    return app;
-}
-
-module.exports = { createApiApp, helpers: require('./helpers/api_helpers.js') };