bulltrackers-module 1.0.11 → 1.0.13

package/functions/generic-api/helpers/api_helpers.js

@@ -0,0 +1,156 @@
+/**
+ * @fileoverview Core helper logic for the Generic API.
+ * Contains validation, data fetching, and request handling.
+ */
+
+const { FieldPath } = require('@google-cloud/firestore');
+const MAX_DATE_RANGE = 100; // Limit queries to a 100-day range
+
+/**
+ * Validates request parameters.
+ * @param {object} query - The request query object.
+ * @returns {string|null} An error message if validation fails, otherwise null.
+ */
+const validateRequest = (query) => {
+    if (!query.computations) return "Missing 'computations' parameter.";
+    if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
+    if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
+
+    const start = new Date(query.startDate);
+    const end = new Date(query.endDate);
+    if (end < start) return "'endDate' must be after 'startDate'.";
+
+    const diffTime = Math.abs(end - start);
+    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
+    if (diffDays > MAX_DATE_RANGE) return `Date range cannot exceed ${MAX_DATE_RANGE} days.`;
+
+    return null;
+};
+
+/**
+ * Flattens the calculation manifest into a simple lookup map.
+ * @param {Object} unifiedCalculations - The imported calculations object.
+ * @returns {Object} A map of { [calcName]: { category, path } }.
+ */
+const buildCalculationMap = (unifiedCalculations) => {
+    const calcMap = {};
+    for (const category in unifiedCalculations) {
+        for (const calcName in unifiedCalculations[category]) {
+            calcMap[calcName] = {
+                category: category,
+                path: `unified_insights/{date}/results/${category}/computations/${calcName}`
+            };
+        }
+    }
+    return calcMap;
+};
+
+/**
+ * Generates an array of YYYY-MM-DD date strings between two dates.
+ * @param {string} startDate - The start date (YYYY-MM-DD).
+ * @param {string} endDate - The end date (YYYY-MM-DD).
+ * @returns {string[]} An array of date strings.
+ */
+const getDateStringsInRange = (startDate, endDate) => {
+    const dates = [];
+    const current = new Date(startDate + 'T00:00:00Z');
+    const end = new Date(endDate + 'T00:00:00Z');
+    
+    while (current <= end) {
+        dates.push(current.toISOString().slice(0, 10));
+        current.setUTCDate(current.getUTCDate() + 1);
+    }
+    return dates;
+};
+
+/**
+ * Fetches all requested computation data from Firestore for the given date range.
+ * @param {Firestore} firestore - A Firestore client instance.
+ * @param {Logger} logger - A logger instance.
+ * @param {string[]} calcKeys - Array of computation keys to fetch.
+ * @param {string[]} dateStrings - Array of dates to fetch for.
+ * @param {Object} calcMap - The pre-built calculation lookup map.
+ * @returns {Promise<Object>} A nested object of [date][computationKey] = data.
+ */
+const fetchUnifiedData = async (firestore, logger, calcKeys, dateStrings, calcMap) => {
+    const response = {};
+    try {
+        for (const date of dateStrings) {
+            response[date] = {};
+            const docRefs = [];
+            const keyPaths = [];
+
+            for (const key of calcKeys) {
+                const pathInfo = calcMap[key];
+                if (pathInfo) {
+                    const docRef = firestore.collection('unified_insights').doc(date)
+                        .collection('results').doc(pathInfo.category)
+                        .collection('computations').doc(key);
+                    
+                    docRefs.push(docRef);
+                    keyPaths.push(key);
+                } else {
+                    logger.log('WARN', `[${date}] No path found for computation key: ${key}`);
+                }
+            }
+
+            if (docRefs.length === 0) continue;
+
+            const snapshots = await firestore.getAll(...docRefs);
+            snapshots.forEach((doc, i) => {
+                const key = keyPaths[i];
+                if (doc.exists) {
+                    response[date][key] = doc.data();
+                } else {
+                    response[date][key] = null; // Indicate missing data
+                }
+            });
+        }
+    } catch (error) {
+        logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
+        throw new Error('Failed to retrieve computation data.');
+    }
+    return response;
+};
+
+/**
+ * Creates the main Express request handler for the API.
+ * @param {Firestore} firestore - A Firestore client instance.
+ * @param {Logger} logger - A logger instance.
+ * @param {Object} calcMap - The pre-built calculation lookup map.
+ * @returns {Function} An async Express request handler.
+ */
+const createApiHandler = (firestore, logger, calcMap) => {
+    return async (req, res) => {
+        const validationError = validateRequest(req.query);
+        if (validationError) {
+            logger.log('WARN', 'Bad Request', { error: validationError });
+            return res.status(400).send({ status: 'error', message: validationError });
+        }
+
+        try {
+            const computationKeys = req.query.computations.split(',');
+            const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
+
+            const data = await fetchUnifiedData(firestore, logger, computationKeys, dateStrings, calcMap);
+
+            res.status(200).send({
+                status: 'success',
+                metadata: {
+                    computations: computationKeys,
+                    startDate: req.query.startDate,
+                    endDate: req.query.endDate,
+                },
+                data,
+            });
+        } catch (error) {
+            logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
+};
+
+module.exports = {
+    buildCalculationMap,
+    createApiHandler,
+};

package/functions/generic-api/index.js

@@ -0,0 +1,41 @@
+/**
+ * @fileoverview Main entry point for the Generic API module.
+ * Exports a function to create the fully configured Express app.
+ */
+
+const express = require('express');
+const cors = require('cors');
+const { buildCalculationMap, createApiHandler } = require('./helpers/api_helpers.js');
+
+/**
+ * Creates and configures the Express app for the Generic API.
+ * @param {Firestore} firestore A Firestore client instance.
+ * @param {Logger} logger A logger instance.
+ * @param {Object} unifiedCalculations The calculations manifest.
+ * @returns {express.Application} The configured Express app.
+ */
+function createApiApp(firestore, logger, unifiedCalculations) {
+    const app = express();
+    
+    // --- Pre-compute Calculation Map ---
+    const calcMap = buildCalculationMap(unifiedCalculations);
+    
+    // --- Middleware ---
+    app.use(cors({ origin: true }));
+    app.use(express.json());
+
+    // --- Main API Endpoint ---
+    // The handler is created with its dependencies injected
+    app.get('/', createApiHandler(firestore, logger, calcMap));
+
+    // --- Health Check Endpoint ---
+    app.get('/health', (req, res) => {
+        res.status(200).send('OK');
+    });
+
+    return app;
+}
+
+module.exports = {
+    createApiApp,
+};

package/functions/orchestrator/index.js

@@ -1,11 +1,136 @@
 /**
- * @fileoverview Exports modules related to the Orchestrator function.
+ * @fileoverview Main orchestration logic for Discovery and Updates.
  */
+const { logger } = require("sharedsetup")(__filename); // Assuming sharedsetup is available in module context
+const coreUtils = require('../../core/utils');
+const { firestore: firestoreUtils } = coreUtils;
+const { discovery, updates } = require('./helpers'); // Import local helpers
 
-const helpers = require('./helpers');
-// const utils = require('./utils'); // If function-specific utils are needed
+/**
+ * Runs the complete discovery orchestration process for a given user type.
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} config - The specific discovery config slice for the user type.
+ * @param {object} globalConfig - Global config like proxiesCollectionName.
+ */
+async function runDiscovery(userType, config, globalConfig) {
+    logger.log('INFO', `[Module Orchestrator] Starting discovery for ${userType} users...`);
+
+    // Reset locks happens once per entry point, keep it there.
+
+    // 1. Check Need
+    const { needsDiscovery, blocksToFill } = await discovery.checkDiscoveryNeed(
+        userType,
+        config.targetUsersPerBlock,
+        config // Pass the whole config slice down
+    );
+
+    if (!needsDiscovery) {
+        logger.log('INFO', `[Module Orchestrator] No discovery needed for ${userType}.`);
+        return;
+    }
+
+    // 2. Get Candidates
+    const candidates = await discovery.getDiscoveryCandidates(
+        userType,
+        blocksToFill,
+        config // Pass the whole config slice down
+    );
+
+    // 3. Dispatch
+    await discovery.dispatchDiscovery(
+        userType,
+        candidates,
+        config // Pass the whole config slice down
+    );
+
+    logger.log('SUCCESS', `[Module Orchestrator] Dispatched discovery tasks for ${userType}.`);
+}
+
+/**
+ * Runs the complete update orchestration process for a given user type.
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} config - The update configuration object.
+ * @param {object} globalConfig - Global config like dispatcherTopicName.
+ */
+async function runUpdates(userType, config, globalConfig) {
+    logger.log('INFO', `[Module Orchestrator] Collecting users for daily update (${userType})...`);
+
+    // Reset locks happens once per entry point, keep it there.
+
+    // 1. Define Thresholds (Remains the same logic as before)
+    const now = new Date();
+    const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
+    const sevenDaysAgoUTC = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // Speculator grace period
+
+    const thresholds = {
+        dateThreshold: startOfTodayUTC,
+        gracePeriodThreshold: sevenDaysAgoUTC
+    };
+
+    // 2. Get Targets
+    const targets = await updates.getUpdateTargets(
+        userType,
+        thresholds,
+        config // Pass the whole update config down
+    );
+
+    // 3. Dispatch
+    await updates.dispatchUpdates(
+        targets,
+        userType,
+        config.dispatcherTopicName, // Use topic name from config
+        config.taskBatchSize,
+        config.pubsubBatchSize
+    );
+
+    logger.log('SUCCESS', `[Module Orchestrator] Dispatched update tasks for ${userType}.`);
+}
+
+
+/**
+ * Creates the higher-level discovery orchestrator function.
+ * This is the function the Cloud Function entry point will call.
+ * @param {object} config - The full orchestratorConfig object loaded from config.js.
+ * @returns {Function} The async function handler for the Cloud Function.
+ */
+function createDiscoveryOrchestrator(config) {
+  return async (req, res) => {
+    logger.log('INFO', '🚀 Discovery Orchestrator triggered via module...');
+    try {
+      await firestoreUtils.resetProxyLocks(config.proxiesCollectionName);
+      await runDiscovery('normal', config.discoveryConfig.normal, config);
+      await runDiscovery('speculator', config.discoveryConfig.speculator, config);
+      res.status(200).send('Discovery orchestration complete.');
+    } catch (error) {
+      logger.log('ERROR', 'FATAL Error in Discovery Orchestrator (Module)', { errorMessage: error.message, errorStack: error.stack });
+      res.status(500).send("An internal discovery error occurred.");
+    }
+  };
+}
+
+/**
+ * Creates the higher-level update orchestrator function.
+ * This is the function the Cloud Function entry point will call.
+ * @param {object} config - The full orchestratorConfig object loaded from config.js.
+ * @returns {Function} The async function handler for the Cloud Function.
+ */
+function createUpdateOrchestrator(config) {
+  return async (req, res) => {
+    logger.log('INFO', '🚀 Update Orchestrator triggered via module...');
+    try {
+      await firestoreUtils.resetProxyLocks(config.proxiesCollectionName);
+      await runUpdates('normal', config.updateConfig, config);
+      await runUpdates('speculator', config.updateConfig, config);
+      res.status(200).send('Update orchestration complete.');
+    } catch (error) {
+      logger.log('ERROR', 'FATAL Error in Update Orchestrator (Module)', { errorMessage: error.message, errorStack: error.stack });
+      res.status(500).send("An internal update error occurred.");
+    }
+  };
+}
 
 module.exports = {
-  helpers,
-  // utils,
+  createDiscoveryOrchestrator,
+  createUpdateOrchestrator,
+  helpers: { discovery, updates } // Keep exporting helpers if needed elsewhere
 };

package/index.js

@@ -7,11 +7,13 @@
 const core = require('./functions/core');
 const Orchestrator = require('./functions/orchestrator');
 const TaskEngine = require('./functions/task-engine');
-const ComputationSystem = require('./functions/computation-system'); // Add this
+const ComputationSystem = require('./functions/computation-system');
+const GenericAPI = require('./functions/generic-api'); // <-- ADD THIS
 
 module.exports = {
   core,
   Orchestrator,
   TaskEngine,
-  ComputationSystem, // Add this
+  ComputationSystem,
+  GenericAPI, // <-- AND ADD THIS
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -8,7 +8,8 @@
     "functions/orchestrator/",
     "functions/task-engine/",
     "functions/core/",
-    "functions/computation-system/"
+    "functions/computation-system/",
+    "functions/generic-api/"
   ],
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -25,7 +26,9 @@
     "sharedsetup": "latest",
     "require-all": "^3.0.0",
     "aiden-shared-calculations-unified": "1.0.0",
-    "@google-cloud/pubsub": "latest"
+    "@google-cloud/pubsub": "latest",
+    "express": "^4.19.2",
+    "cors": "^2.8.5"
   },
   "engines": {
     "node": ">=20"