bulltrackers-module 1.0.104 → 1.0.106

package/functions/computation-system/utils/utils.js

@@ -1,255 +1,150 @@
-/**
- * @fileoverview Computation system sub-pipes and utils.
- * REFACTORED: Now stateless and receive dependencies where needed.
- * DYNAMIC: Categorization logic is now an exported function.
- */
-
-const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-
-// --- REMOVED ---
-// const { utils } = require('aiden-shared-calculations-unified');
-// const { withRetry } = utils;
-// --- END REMOVED ---
-
-
-/**
- * NEW FUNCTION: This logic is now wrapped and exported.
- * It receives the 'calculations' object from the entry point.
- * @param {object} calculations - The full calculations object from the 'aiden-shared-calculations-unified' package.
- * @returns {object} An object containing categorized calculations and name sets.
- */
-function categorizeCalculations(calculations) {
-    const HISTORICAL_CALC_NAMES = new Set();
-    const META_CALC_NAMES = new Set();
-    const BACKTEST_CALC_NAMES = new Set(); // <-- ADD THIS
-
-    const historicalCalculations = {};
-    const dailyCalculations = {};
-    const metaCalculations = {};
-    const backtestCalculations = {}; // <-- ADD THIS
-
-    for (const category in calculations) {
-        // 1. Check for 'meta' category first (by top-level directory name)
-        if (category === 'meta') {
-            if (!metaCalculations[category]) metaCalculations[category] = {};
-            for (const calcName in calculations[category]) {
-                const CalculationClass = calculations[category][calcName];
-                metaCalculations[category][calcName] = CalculationClass;
-                META_CALC_NAMES.add(calcName);
-            }
-            continue; // Done with this category
-        }
-        
-        // 2. Check for 'backtests' category
-        if (category === 'backtests') { // <-- ADD THIS BLOCK
-            if (!backtestCalculations[category]) backtestCalculations[category] = {};
-            for (const calcName in calculations[category]) {
-                const CalculationClass = calculations[category][calcName];
-                backtestCalculations[category][calcName] = CalculationClass;
-                BACKTEST_CALC_NAMES.add(calcName);
-            }
-            continue;
-        }
-
-
-        // 3. Process other categories (e.g., 'pnl', 'capital_flow')
-        for (const subKey in calculations[category]) {
-            const item = calculations[category][subKey];
-
-            // Check if the key is 'historical' and it contains an object of calculations
-            if (subKey === 'historical' && typeof item === 'object' && item !== null && !Array.isArray(item)) {
-                // This is the historical subdirectory, e.g., calculations.capital_flow.historical
-                
-                if (!historicalCalculations[category]) historicalCalculations[category] = {};
-                
-                for (const calcName in item) {
-                    const CalculationClass = item[calcName];
-                    // Add it to the historical list, using the parent 'category'
-                    historicalCalculations[category][calcName] = CalculationClass;
-                    HISTORICAL_CALC_NAMES.add(calcName);
-                }
-            }
-            // Check if the item is a function (a standard daily calc at the root of the category)
-            else if (typeof item === 'function') {
-                // This is a standard daily calc, e.g., calculations.pnl.average-daily-pnl
-                const calcName = subKey;
-                const CalculationClass = item;
-                
-                if (!dailyCalculations[category]) dailyCalculations[category] = {};
-                dailyCalculations[category][calcName] = CalculationClass;
-            }
-        }
-    }
-    
-    return {
-        historicalCalculations,
-        dailyCalculations,
-        metaCalculations,
-        backtestCalculations, // <-- ADD THIS
-        HISTORICAL_CALC_NAMES,
-        META_CALC_NAMES,
-        BACKTEST_CALC_NAMES  // <-- ADD THIS
-    };
-}
-// --- End Dynamic Categorization ---
-
-
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.commitBatchInChunks
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {Array<object>} writes - Array of { ref: DocumentReference, data: object }.
- * @param {string} operationName - Name for logging.
- */
-async function commitBatchInChunks(config, dependencies, writes, operationName) {
-    // --- MODIFIED: Get withRetry from dependencies ---
-    const { db, logger, calculationUtils } = dependencies;
-    const { withRetry } = calculationUtils;
-    // --- END MODIFIED ---
-    
-    const batchSizeLimit = config.batchSizeLimit || 450;
-    
-    if (writes.length === 0) {
-        logger.log('WARN', `[${operationName}] No writes to commit.`);
-        return;
-    }
-    for (let i = 0; i < writes.length; i += batchSizeLimit) {
-        const batch = db.batch(); // Use db
-        const chunk = writes.slice(i, i + batchSizeLimit);
-        chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
-
-        const chunkNum = Math.floor(i / batchSizeLimit) + 1;
-        const totalChunks = Math.ceil(writes.length / batchSizeLimit);
-        await withRetry(
-            () => batch.commit(),
-            `${operationName} (Chunk ${chunkNum}/${totalChunks})`
-        );
-        logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`);
-    }
-}
-
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.getExpectedDateStrings
- * (Stateless)
- */
-function getExpectedDateStrings(startDate, endDate) {
-    const dateStrings = [];
-    if (startDate <= endDate) {
-        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
-        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
-        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) {
-            dateStrings.push(new Date(d).toISOString().slice(0, 10));
-        }
-    }
-    return dateStrings;
-}
-
-
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.processJobsInParallel
- * (Stateless, as taskFunction will receive dependencies)
- */
-async function processJobsInParallel(jobs, taskFunction, passName, config) {
-    // This function itself doesn't need dependencies, 
-    // but the 'taskFunction' it calls *will* receive them from its caller.
-    const { logger } = require("sharedsetup")(__filename); // Use local logger for this static util
-    const results = [];
-    const maxConcurrentDates = config.maxConcurrentDates || 3;
-     if (jobs.length > 0) {
-        logger.log('INFO', `[${passName}] Processing ${jobs.length} jobs with ${maxConcurrentDates} parallel workers.`);
-        jobs.sort((a, b) => new Date(a.date) - new Date(b.date));
-        for (let i = 0; i < jobs.length; i += maxConcurrentDates) {
-            const jobBatch = jobs.slice(i, i + maxConcurrentDates);
-            const promises = jobBatch.map(job => taskFunction(new Date(job.date + 'T00:00:00Z'), job.missing));
-            results.push(...await Promise.allSettled(promises));
-        }
-    } else {
-        logger.log('INFO', `[${passName}] No jobs to process.`);
-    }
-    return results;
-}
-
-/**
- * Internal helper: Finds the earliest date document in a collection.
- */
-async function getFirstDateFromCollection(config, dependencies, collectionName) {
-    // --- MODIFIED: Get withRetry from dependencies ---
-    const { db, logger, calculationUtils } = dependencies;
-    const { withRetry } = calculationUtils;
-    // --- END MODIFIED ---
-    let earliestDate = null;
-    try {
-        const blockDocRefs = await withRetry(
-            () => db.collection(collectionName).listDocuments(), // Use db
-            `GetBlocks(${collectionName})`
-        );
-
-         if (blockDocRefs.length === 0) {
-             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-             return null;
-         }
-
-         for (const blockDocRef of blockDocRefs) {
-             const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
-                 .where(FieldPath.documentId(), '>=', '2000-01-01')
-                 .orderBy(FieldPath.documentId(), 'asc')
-                 .limit(1);
-
-             const snapshotSnap = await withRetry(
-                 () => snapshotQuery.get(),
-                 `GetEarliestSnapshot(${blockDocRef.path})`
-             );
-
-             if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                 const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-                 if (!earliestDate || foundDate < earliestDate) {
-                     earliestDate = foundDate;
-                 }
-             }
-         }
-
-    } catch (e) {
-        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
-    }
-    return earliestDate;
-}
-
-/**
- * Sub-pipe: pipe.computationSystem.computationUtils.getFirstDateFromSourceData
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<Date>} The earliest date found or a default fallback date.
- */
-async function getFirstDateFromSourceData(config, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', 'Querying for the earliest date from source portfolio data...');
-    
-    // Pass dependencies to sub-pipe
-    const investorDate = await getFirstDateFromCollection(config, dependencies, config.normalUserPortfolioCollection);
-    const speculatorDate = await getFirstDateFromCollection(config, dependencies, config.speculatorPortfolioCollection);
-
-     let earliestDate;
-     if (investorDate && speculatorDate) {
-         earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-     } else {
-         earliestDate = investorDate || speculatorDate;
-     }
-
-     if (earliestDate) {
-         logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-         return earliestDate;
-     } else {
-         const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-         logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
-         return fallbackDate;
-     }
-}
-
-module.exports = {
-    FieldValue, FieldPath,
-    // unifiedUtils: utils, // This is no longer defined here
-    categorizeCalculations, // EXPORT THE NEW FUNCTION
-    // withRetry, // This is no longer defined here
-    commitBatchInChunks,
-    getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
+/**
+ * @fileoverview Computation system sub-pipes and utils.
+ * REFACTORED: Now stateless and receive dependencies where needed.
+ * DYNAMIC: Categorization logic is removed, replaced by manifest.
+ */
+
+const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+
+/**
+ * Normalizes a calculation name to kebab-case.
+ * @param {string} name 
+ * @returns {string}
+ */
+function normalizeName(name) {
+    return name.replace(/_/g, '-');
+}
+
+/**
+ * Sub-pipe: pipe.computationSystem.computationUtils.commitBatchInChunks
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {Array<object>} writes - Array of { ref: DocumentReference, data: object }.
+ * @param {string} operationName - Name for logging.
+ */
+async function commitBatchInChunks(config, dependencies, writes, operationName) {
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
+    
+    const batchSizeLimit = config.batchSizeLimit || 450;
+    
+    if (writes.length === 0) {
+        logger.log('WARN', `[${operationName}] No writes to commit.`);
+        return;
+    }
+    for (let i = 0; i < writes.length; i += batchSizeLimit) {
+        const batch = db.batch(); // Use db
+        const chunk = writes.slice(i, i + batchSizeLimit);
+        chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
+
+        const chunkNum = Math.floor(i / batchSizeLimit) + 1;
+        const totalChunks = Math.ceil(writes.length / batchSizeLimit);
+        await withRetry(
+            () => batch.commit(),
+            `${operationName} (Chunk ${chunkNum}/${totalChunks})`
+        );
+        logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`);
+    }
+}
+
+/**
+ * Sub-pipe: pipe.computationSystem.computationUtils.getExpectedDateStrings
+ * (Stateless)
+ */
+function getExpectedDateStrings(startDate, endDate) {
+    const dateStrings = [];
+    if (startDate <= endDate) {
+        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
+        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
+        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) {
+            dateStrings.push(new Date(d).toISOString().slice(0, 10));
+        }
+    }
+    return dateStrings;
+}
+
+/**
+ * Internal helper: Finds the earliest date document in a collection.
+ */
+async function getFirstDateFromCollection(config, dependencies, collectionName) {
+    // --- MODIFIED: Get withRetry from dependencies ---
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+    // --- END MODIFIED ---
+    let earliestDate = null;
+    try {
+        const blockDocRefs = await withRetry(
+            () => db.collection(collectionName).listDocuments(), // Use db
+            `GetBlocks(${collectionName})`
+        );
+
+         if (blockDocRefs.length === 0) {
+             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
+             return null;
+         }
+
+         for (const blockDocRef of blockDocRefs) {
+             const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
+                 .where(FieldPath.documentId(), '>=', '2000-01-01')
+                 .orderBy(FieldPath.documentId(), 'asc')
+                 .limit(1);
+
+             const snapshotSnap = await withRetry(
+                 () => snapshotQuery.get(),
+                 `GetEarliestSnapshot(${blockDocRef.path})`
+             );
+
+             if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
+                 const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
+                 if (!earliestDate || foundDate < earliestDate) {
+                     earliestDate = foundDate;
+                 }
+             }
+         }
+
+    } catch (e) {
+        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
+    }
+    return earliestDate;
+}
+
+/**
+ * Sub-pipe: pipe.computationSystem.computationUtils.getFirstDateFromSourceData
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @returns {Promise<Date>} The earliest date found or a default fallback date.
+ */
+async function getFirstDateFromSourceData(config, dependencies) {
+    const { logger } = dependencies;
+    logger.log('INFO', 'Querying for the earliest date from source portfolio data...');
+    
+    // Pass dependencies to sub-pipe
+    const investorDate = await getFirstDateFromCollection(config, dependencies, config.normalUserPortfolioCollection);
+    const speculatorDate = await getFirstDateFromCollection(config, dependencies, config.speculatorPortfolioCollection);
+
+     let earliestDate;
+     if (investorDate && speculatorDate) {
+         earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
+     } else {
+         earliestDate = investorDate || speculatorDate;
+     }
+
+     if (earliestDate) {
+         logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
+         return earliestDate;
+     } else {
+         const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+         logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
+         return fallbackDate;
+     }
+}
+
+module.exports = {
+    FieldValue, FieldPath,
+    normalizeName,
+    commitBatchInChunks,
+    getExpectedDateStrings, 
+    getFirstDateFromSourceData,
 };