bulltrackers-module 1.0.21 → 1.0.23

package/functions/computation-system/handler_creator.js

@@ -0,0 +1,41 @@
+/**
+ * @fileoverview Factory function to create the Computation System handler.
+ */
+const { Firestore } = require('@google-cloud/firestore');
+const { logger: defaultLogger } = require("sharedsetup")(__filename);
+const { runComputationOrchestrator } = require('./helpers/orchestration_helpers');
+
+/**
+ * Creates the Computation System Cloud Function handler (HTTP triggered).
+ * @param {object} config - The Computation System configuration object.
+ * @param {object} dependencies - Shared dependencies { firestore, logger }.
+ * @returns {Function} The async Cloud Function handler for HTTP requests.
+ */
+function createComputationSystemHandler(config, dependencies) {
+    // Use dependencies passed from index.js
+    const firestore = dependencies.firestore || new Firestore();
+    const logger = dependencies.logger || defaultLogger;
+
+    return async (req, res) => {
+        const functionName = 'ComputationSystem';
+        logger.log('INFO', `[${functionName}] Starting...`);
+        try {
+            // Pass the config down to the orchestrator
+            const summary = await runComputationOrchestrator(firestore, logger, config);
+
+            logger.log('SUCCESS', `[${functionName}] Done.`);
+            if (!res.headersSent) {
+                res.status(200).send({ message: 'Computation complete', summary });
+            }
+        } catch (error) {
+            logger.log('ERROR', `[${functionName}] Failed.`, { errorMessage: error.message, errorStack: error.stack });
+            if (!res.headersSent) {
+                res.status(500).send({ message: 'Computation Error', error: error.message });
+            }
+        }
+    };
+}
+
+module.exports = {
+    createComputationSystemHandler,
+};

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,21 +1,21 @@
 /**
  * @fileoverview Core orchestration logic for the Computation System.
- * Contains both the high-level pass orchestrator and the low-level
- * function for processing a single date.
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
 const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs } = require('../utils/data_loader.js');
 const {
+    // Note: calculation categories are now sourced from utils.js which gets them from the unified package
     historicalCalculations, dailyCalculations, HISTORICAL_CALC_NAMES,
-    withRetry, getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
-    commitBatchInChunks, unifiedUtils
+    withRetry, // Ensure withRetry is correctly imported if needed here, or rely on utils.js export
+    getExpectedDateStrings, processJobsInParallel, getFirstDateFromSourceData,
+    commitBatchInChunks, unifiedUtils // unifiedUtils comes from the calculations package via utils.js
 } = require('../utils/utils.js');
 
 
-// --- CORE CALCULATION HELPERS (Moved from original utils.js) ---
+// --- CORE CALCULATION HELPERS ---
 
-/** Initializes calculator instances from the provided calculation classes. */
+/** Initializes calculator instances. */
 function initializeCalculators(calculationsToRun, sourcePackage) {
     const state = {};
     for (const { category, calcName } of calculationsToRun) {
@@ -24,54 +24,61 @@ function initializeCalculators(calculationsToRun, sourcePackage) {
             try {
                 state[`${category}/${calcName}`] = new CalculationClass();
             } catch (e) {
-                // Use logger, but it's not passed here, so console.warn
                 console.warn(`[Orchestrator] Init failed for ${category}/${calcName}`, { errorMessage: e.message });
-                state[`${category}/${calcName}`] = null;
+                state[`${category}/${calcName}`] = null; // Mark as failed
             }
         } else {
             console.warn(`[Orchestrator] Calculation class not found: ${category}/${calcName}`);
+            state[`${category}/${calcName}`] = null; // Mark as missing
         }
     }
     return state;
 }
 
-/** Streams day's data and calls the process() method on all calculators. */
-async function streamAndProcess(firestore, dateStr, todayRefs, state, passName, logger, yesterdayPortfolios = {}) {
+/** Streams day's data, calls process() on calculators, using config for batch size. */
+async function streamAndProcess(firestore, dateStr, todayRefs, state, passName, logger, config, yesterdayPortfolios = {}) {
     logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
+    // unifiedUtils comes from calculations package via utils.js
     const { instrumentToTicker, instrumentToSector } = await unifiedUtils.loadInstrumentMappings();
     const context = { instrumentMappings: instrumentToTicker, sectorMapping: instrumentToSector };
+    const batchSize = config.partRefBatchSize || 10; // Use config for chunk size
 
-    for (let i = 0; i < todayRefs.length; i += 10) { // Process in chunks of 10 part files
-        const batchRefs = todayRefs.slice(i, i + 10);
-        const todayPortfoliosChunk = await loadDataByRefs(firestore, batchRefs);
+    for (let i = 0; i < todayRefs.length; i += batchSize) {
+        const batchRefs = todayRefs.slice(i, i + batchSize);
+        // Pass config to data loader
+        const todayPortfoliosChunk = await loadDataByRefs(firestore, batchRefs, config);
 
         for (const uid in todayPortfoliosChunk) {
             const p = todayPortfoliosChunk[uid];
-            if (!p) continue;
-            
+            if (!p) continue; // Skip if portfolio data is null/undefined for this user
+
+            // Determine user type based on portfolio structure
             const userType = p.PublicPositions ? 'speculator' : 'normal';
-            
+
             for (const key in state) {
                 const calc = state[key];
+                // Skip if calculator failed initialization or has no process method
                 if (!calc || typeof calc.process !== 'function') continue;
 
                 const [category, calcName] = key.split('/');
                 let processArgs;
 
                 if (HISTORICAL_CALC_NAMES.has(calcName)) {
+                    // Historical calculations require yesterday's data
                     const pYesterday = yesterdayPortfolios[uid];
-                    if (!pYesterday) continue; // Skip if no yesterday data for historical calc
-                    processArgs = [p, pYesterday, uid];
+                    if (!pYesterday) continue; // Skip if no yesterday data
+                    processArgs = [p, pYesterday, uid]; // pToday, pYesterday, userId
                 } else {
-                    // Filter daily calcs by user type
+                    // Filter daily calculations based on user type and calculation category
                     if ((userType === 'normal' && category === 'speculators') ||
                         (userType === 'speculator' && !['speculators', 'sanity'].includes(category))) {
-                        continue;
+                        continue; // Skip speculator calcs for normal users, and non-speculator/sanity calcs for speculators
                     }
-                    processArgs = [p, uid, context];
+                    processArgs = [p, uid, context]; // pToday, userId, context
                 }
-                
+
                 try {
+                    // Allow process method to be async or sync
                     await Promise.resolve(calc.process(...processArgs));
                 } catch (e) {
                     logger.log('WARN', `Process error in ${key} for user ${uid}`, { err: e.message });
@@ -82,95 +89,128 @@ async function streamAndProcess(firestore, dateStr, todayRefs, state, passName,
 }
 
 
-// --- LOW-LEVEL ORCHESTRATOR (Moved from pass_orchestrators.js) ---
+// --- LOW-LEVEL ORCHESTRATOR ---
 
 /**
- * Runs a set of calculations for a given date and writes the final results.
- * @param {Firestore} firestore A Firestore client instance.
- * @param {Function} logger A logger instance.
- * @param {Date} dateToProcess - The date to run computations for.
- * @param {Array} calculationsToRun - List of { category, calcName } to run.
- * @param {String} passName - The name of the pass for logging.
- * @param {Object} sourcePackage - The object containing calculation classes.
- * @returns {Object} A summary of the execution.
+ * Runs computations for a single date using the provided config.
+ * @param {Firestore} firestore Firestore client.
+ * @param {Function} logger Logger instance.
+ * @param {Date} dateToProcess The date to process.
+ * @param {Array<object>} calculationsToRun List of { category, calcName }.
+ * @param {string} passName Name for logging.
+ * @param {object} sourcePackage Object containing calculation classes.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<object>} Execution summary.
  */
-async function runUnifiedComputation(firestore, logger, dateToProcess, calculationsToRun, passName, sourcePackage) {
+async function runUnifiedComputation(firestore, logger, dateToProcess, calculationsToRun, passName, sourcePackage, config) { // Added config
     const dateStr = dateToProcess.toISOString().slice(0, 10);
     logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
 
     try {
         // --- 1. Data Loading ---
-        const todayRefs = await getPortfolioPartRefs(firestore, dateStr);
+        // Pass config to data loader functions
+        const todayRefs = await getPortfolioPartRefs(firestore, dateStr, config);
         if (todayRefs.length === 0) {
+            logger.log('INFO', `[${passName}] No portfolio data found for ${dateStr}. Skipping.`);
             return { success: true, date: dateStr, message: "No portfolio data for today." };
         }
 
         let yesterdayPortfolios = {};
+        // Check if any calculation *in this run* needs yesterday's data
         const requiresYesterday = calculationsToRun.some(c => sourcePackage[c.category]?.[c.calcName]?.prototype?.process.length === 3);
 
         if (requiresYesterday) {
             const prev = new Date(dateToProcess);
             prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr = prev.toISOString().slice(0, 10);
-            const yesterdayRefs = await getPortfolioPartRefs(firestore, prevStr);
-            yesterdayPortfolios = await loadFullDayMap(firestore, yesterdayRefs);
+            // Pass config here too
+            const yesterdayRefs = await getPortfolioPartRefs(firestore, prevStr, config);
+            if (yesterdayRefs.length > 0) {
+                yesterdayPortfolios = await loadFullDayMap(firestore, yesterdayRefs, config);
+                 logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
+            } else {
+                logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
+            }
         }
 
         // --- 2. In-Memory Processing ---
         const state = initializeCalculators(calculationsToRun, sourcePackage);
-        await streamAndProcess(firestore, dateStr, todayRefs, state, passName, logger, yesterdayPortfolios);
+        // Pass config here
+        await streamAndProcess(firestore, dateStr, todayRefs, state, passName, logger, config, yesterdayPortfolios);
 
-        // --- 3. REFACTORED: Write results PER-CALCULATION ---
+        // --- 3. Write Results Per Calculation ---
         let successCount = 0;
-        const resultsSubcollection = firestore.collection('unified_insights').doc(dateStr).collection('results');
+        // Use config for collection names
+        const resultsCollectionRef = firestore.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
 
         for (const key in state) {
             const calc = state[key];
+            // Skip if calculator failed init or has no getResult
             if (!calc || typeof calc.getResult !== 'function') continue;
-            
+
             const [category, calcName] = key.split('/');
-            const pendingWrites = []; 
-            const summaryData = {};
 
             try {
+                // Allow getResult to be async or sync
                 const result = await Promise.resolve(calc.getResult());
+                const pendingWrites = [];
+                const summaryData = {}; // Summary for this specific calculation's success
 
+                // Only proceed if result is not null/undefined and has data
                 if (result && Object.keys(result).length > 0) {
-                    
                     if (result.sharded_user_profitability) {
+                        // Handle sharded profitability - use config for collection name
                         for (const shardId in result.sharded_user_profitability) {
-                            const shardData = result.sharded_user_profitability[shardId];
-                            if (shardData && Object.keys(shardData).length > 0) {
-                                const shardRef = firestore.collection('unified_insights').doc(shardId);
-                                pendingWrites.push({ ref: shardRef, data: { profits: shardData } });
-                            }
+                             const shardData = result.sharded_user_profitability[shardId];
+                             if (shardData && Object.keys(shardData).length > 0) {
+                                 // Use config collection name
+                                 const shardRef = firestore.collection(config.shardedProfitabilityCollection).doc(shardId);
+                                 // Use merge:true to update existing shard docs safely
+                                 pendingWrites.push({ ref: shardRef, data: { profits: shardData } });
+                             }
                         }
+                        // Mark sharded calculation as processed in summary
+                         if (!summaryData[category]) summaryData[category] = {};
+                         summaryData[category][calcName] = true;
+
                     } else {
-                        const computationDocRef = resultsSubcollection.doc(category)
-                                                          .collection('computations')
+                        // Handle normal results - use config for subcollection names
+                        const computationDocRef = resultsCollectionRef.doc(category)
+                                                          .collection(config.computationsSubcollection)
                                                           .doc(calcName);
                         pendingWrites.push({ ref: computationDocRef, data: result });
+
+                        // Mark normal calculation as processed in summary
+                        if (!summaryData[category]) summaryData[category] = {};
+                        summaryData[category][calcName] = true;
                     }
 
-                    if (!summaryData[category]) summaryData[category] = {};
-                    summaryData[category][calcName] = true; 
-                    
-                    const docRef = firestore.collection('unified_insights').doc(dateStr);
-                    pendingWrites.push({ ref: docRef, data: summaryData });
-                
-                    await commitBatchInChunks(
-                        firestore,
-                        pendingWrites, 
-                        `Commit ${passName} ${dateStr} [${key}]`
-                    );
-                    successCount++;
+                    // Add summary update to the top-level date document using merge
+                     if (Object.keys(summaryData).length > 0) {
+                        const topLevelDocRef = firestore.collection(config.resultsCollection).doc(dateStr);
+                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
+                     }
+
+                    // Commit writes if any were generated
+                    if (pendingWrites.length > 0) {
+                        await commitBatchInChunks(
+                            firestore,
+                            pendingWrites,
+                            `Commit ${passName} ${dateStr} [${key}]`,
+                            config // Pass config
+                        );
+                        successCount++;
+                    }
+                } else {
+                     logger.log('INFO', `[${passName}] Calculation ${key} produced no results for ${dateStr}. Skipping write.`);
                 }
             } catch (e) {
-                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${key}`, { err: e.message });
+                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${key} on ${dateStr}`, { err: e.message });
             }
         }
-        
-        logger.log('SUCCESS', `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
+
+        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
+        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
         return { success: true, date: dateStr, successful: successCount, failed: calculationsToRun.length - successCount };
 
     } catch (err) {
@@ -180,21 +220,22 @@ async function runUnifiedComputation(firestore, logger, dateToProcess, calculati
 }
 
 
-// --- HIGH-LEVEL ORCHESTRATOR (Moved from ComputationSystem.js) ---
+// --- HIGH-LEVEL ORCHESTRATOR ---
 
 /**
- * Main entry point for the Unified Computation System logic.
- * @param {Firestore} firestore A Firestore client instance.
- * @param {Function} logger A logger instance.
- * @returns {Promise<Object>} A summary of all passes.
+ * Main entry point for the Unified Computation System logic, using config.
+ * @param {Firestore} firestore Firestore client.
+ * @param {Function} logger Logger instance.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<Object>} Summary of all passes.
  */
-async function runComputationOrchestrator(firestore, logger) {
+async function runComputationOrchestrator(firestore, logger, config) { // Added config
     const summary = { pass1_results: [], pass2_results: [] };
     const yesterday = new Date();
     yesterday.setUTCDate(yesterday.getUTCDate() - 1);
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
 
-    // --- Master Calculation Lists ---
+    // --- Master Calculation Lists (sourced via utils.js) ---
     const masterHistoricalList = Object.entries(historicalCalculations).flatMap(([cat, calcs]) =>
         Object.keys(calcs).map(name => ({ category: cat, calcName: name }))
     );
@@ -204,63 +245,87 @@ async function runComputationOrchestrator(firestore, logger) {
     const masterFullList = [...masterHistoricalList, ...masterDailyList];
 
     // --- Date & Job Discovery ---
-    const firstDate = await getFirstDateFromSourceData(firestore);
-    const startDateUTC = new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()));
-    
+    // Pass config here
+    const firstDate = await getFirstDateFromSourceData(firestore, config);
+    // Use config for fallback date if needed
+     const startDateUTC = firstDate
+       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
+       : new Date(config.earliestComputationDate + 'T00:00:00Z'); // Ensure UTC
+
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-    const insightDocs = await withRetry(() => firestore.collection('unified_insights').get(), "ListAllInsightDocs");
+    // Use config for resultsCollection
+    const insightDocs = await withRetry(() => firestore.collection(config.resultsCollection).get(), "ListAllInsightDocs");
     const existingDateIds = new Set(insightDocs.docs.map(d => d.id).filter(id => /^\d{4}-\d{2}-\d{2}$/.test(id)));
 
     // --- PASS 1: Find and process entirely missing days (DAILY CALCS ONLY) ---
     const missingDates = allExpectedDates.filter(dateStr => !existingDateIds.has(dateStr));
     const pass1Jobs = missingDates.map(date => ({ date, missing: masterDailyList }));
-    
+
+    logger.log('INFO', `[Orchestrator] Pass 1: Found ${pass1Jobs.length} missing dates to process (daily calcs only).`);
     const pass1Results = await processJobsInParallel(
         pass1Jobs,
-        (date, missing) => runUnifiedComputation(firestore, logger, date, missing, 'Pass 1 (Daily Only)', dailyCalculations),
-        'Pass 1'
+        // Pass config to runUnifiedComputation
+        (date, missing) => runUnifiedComputation(firestore, logger, date, missing, 'Pass 1 (Daily Only)', dailyCalculations, config),
+        'Pass 1',
+        config // Pass config to processJobsInParallel
     );
+    // Process results (handle fulfilled/rejected)
     summary.pass1_results = pass1Results.map((r, i) => r.status === 'fulfilled' ? r.value : { success: false, date: pass1Jobs[i].date, error: r.reason?.message });
 
+
     // --- PASS 2: Find and process incomplete days ---
     const pass2Jobs = [];
-    const updatedInsightDocs = await withRetry(() => firestore.collection('unified_insights').where(FieldPath.documentId(), 'in', allExpectedDates).get(), "ListAllInsightDocs-Pass2");
+    // Use config for resultsCollection
+    const updatedInsightDocs = await withRetry(() => firestore.collection(config.resultsCollection).where(FieldPath.documentId(), 'in', allExpectedDates).get(), "ListAllInsightDocs-Pass2");
 
     updatedInsightDocs.forEach(doc => {
-        if (!/^\d{4}-\d{2}-\d{2}$/.test(doc.id)) return;
+        if (!/^\d{4}-\d{2}-\d{2}$/.test(doc.id)) return; // Skip non-date docs
 
         const data = doc.data();
+        // Check which computations are missing based on the summary flags in the top-level doc
         const missingCalcs = masterFullList.filter(({ category, calcName }) => !data?.[category]?.[calcName]);
 
         if (missingCalcs.length > 0) {
             pass2Jobs.push({ date: doc.id, missing: missingCalcs });
         }
     });
-    
+
+    logger.log('INFO', `[Orchestrator] Pass 2: Found ${pass2Jobs.length} incomplete dates to process.`);
     const pass2Results = await processJobsInParallel(
         pass2Jobs,
-        (date, missing) => {
+        async (date, missing) => { // Make task function async
             const historicalMissing = missing.filter(c => HISTORICAL_CALC_NAMES.has(c.calcName));
             const dailyMissing = missing.filter(c => !HISTORICAL_CALC_NAMES.has(c.calcName));
-            const promises = [];
+            const results = []; // Collect results from both runs if applicable
+
             if (historicalMissing.length > 0) {
-                promises.push(runUnifiedComputation(firestore, logger, date, historicalMissing, 'Pass 2 (Historical)', historicalCalculations));
+                 logger.log('INFO', `[Pass 2] Running ${historicalMissing.length} historical calcs for ${date.toISOString().slice(0, 10)}.`);
+                 // Pass config here
+                const histResult = await runUnifiedComputation(firestore, logger, date, historicalMissing, 'Pass 2 (Historical)', historicalCalculations, config);
+                results.push(histResult);
             }
             if (dailyMissing.length > 0) {
-                promises.push(runUnifiedComputation(firestore, logger, date, dailyMissing, 'Pass 2 (Daily)', dailyCalculations));
+                 logger.log('INFO', `[Pass 2] Running ${dailyMissing.length} daily calcs for ${date.toISOString().slice(0, 10)}.`);
+                 // Pass config here
+                const dailyResult = await runUnifiedComputation(firestore, logger, date, dailyMissing, 'Pass 2 (Daily)', dailyCalculations, config);
+                results.push(dailyResult);
             }
-            return Promise.all(promises);
+            // Return combined results or a summary
+            return results.length === 1 ? results[0] : { date: date.toISOString().slice(0,10), results };
         },
-        'Pass 2'
+        'Pass 2',
+        config // Pass config to processJobsInParallel
     );
+     // Process results (handle fulfilled/rejected)
     summary.pass2_results = pass2Results.map((r, i) => r.status === 'fulfilled' ? r.value : { success: false, date: pass2Jobs[i].date, error: r.reason?.message });
 
+
     // --- FINISH ---
-    // The wrapper function will log success and send the response.
+    logger.log('INFO', '[Orchestrator] Computation orchestration finished.');
     return summary;
 }
 
 module.exports = {
     runComputationOrchestrator,
-    // We don't need to export runUnifiedComputation as it's only used internally
+    // runUnifiedComputation is only used internally now
 };

package/functions/computation-system/index.js

@@ -1,9 +1,11 @@
 /**
- * @fileoverview Exports the computation system's main logic.
+ * @fileoverview Exports the computation system's main logic and handler creator.
  */
 
 const { runComputationOrchestrator } = require('./helpers/orchestration_helpers.js');
+const { createComputationSystemHandler } = require('./handler_creator.js'); // <-- Import the new factory
 
 module.exports = {
   runComputationOrchestrator,
+  createComputationSystemHandler, // <-- Export the factory
 };

package/functions/computation-system/utils/data_loader.js

@@ -4,25 +4,26 @@
  */
 
 const { logger } = require("sharedsetup")(__filename);
-// Import withRetry from the unified calculations package
 const { withRetry } = require('aiden-shared-calculations-unified').utils;
 
-const COLLECTIONS_TO_QUERY = ['NormalUserPortfolios', 'SpeculatorUserPortfolios'];
-const PART_REF_BATCH_SIZE = 50; // Number of document snapshots to load at once
+// REMOVE Constants like COLLECTIONS_TO_QUERY, PART_REF_BATCH_SIZE
 
+// Modify getPortfolioPartRefs to accept config
 /**
  * Gets a list of all portfolio "part" document references for a given date.
- * This function is lightweight and only fetches references, not data.
  *
  * @param {Firestore} firestore A Firestore client instance.
  * @param {string} dateString The date in YYYY-MM-DD format.
+ * @param {object} config The computation system configuration object.
  * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
  */
-async function getPortfolioPartRefs(firestore, dateString) {
+async function getPortfolioPartRefs(firestore, dateString, config) {
     logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
     const allPartRefs = [];
+    // Use config for collection names
+    const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
 
-    for (const collectionName of COLLECTIONS_TO_QUERY) {
+    for (const collectionName of collectionsToQuery) {
         const blockDocsQuery = firestore.collection(collectionName);
         const blockDocRefs = await withRetry(
             () => blockDocsQuery.listDocuments(),
@@ -30,12 +31,13 @@ async function getPortfolioPartRefs(firestore, dateString) {
         );
 
         if (blockDocRefs.length === 0) {
-            logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-            continue; 
-        }
+             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
+             continue;
+         }
 
         for (const blockDocRef of blockDocRefs) {
-            const partsCollectionRef = blockDocRef.collection('snapshots').doc(dateString).collection('parts');
+            // Use config for subcollection names
+            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
             const partDocs = await withRetry(
                 () => partsCollectionRef.listDocuments(),
                 `listDocuments(${partsCollectionRef.path})`
@@ -48,30 +50,26 @@ async function getPortfolioPartRefs(firestore, dateString) {
     return allPartRefs;
 }
 
-
+// Modify loadDataByRefs to accept config
 /**
  * Loads data from an array of DocumentReferences using firestore.getAll().
- * Merges all data into a single map keyed by user ID.
  *
  * @param {Firestore} firestore A Firestore client instance.
  * @param {Firestore.DocumentReference[]} refs An array of DocumentReferences to load.
+ * @param {object} config The computation system configuration object.
  * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
  */
-async function loadDataByRefs(firestore, refs) {
-    if (!refs || refs.length === 0) {
-        return {};
-    }
-
+async function loadDataByRefs(firestore, refs, config) {
+    if (!refs || refs.length === 0) { return {}; }
     const mergedPortfolios = {};
+    const batchSize = config.partRefBatchSize || 50; // Use config
 
-    for (let i = 0; i < refs.length; i += PART_REF_BATCH_SIZE) {
-        const batchRefs = refs.slice(i, i + PART_REF_BATCH_SIZE);
-
+    for (let i = 0; i < refs.length; i += batchSize) { // Use config batch size
+        const batchRefs = refs.slice(i, i + batchSize); // Use config batch size
         const snapshots = await withRetry(
             () => firestore.getAll(...batchRefs),
-            `getAll(batch ${Math.floor(i / PART_REF_BATCH_SIZE)})`
+            `getAll(batch ${Math.floor(i / batchSize)})` // Use config batch size
         );
-
         for (const doc of snapshots) {
             if (doc.exists) {
                 const data = doc.data();
@@ -86,11 +84,19 @@ async function loadDataByRefs(firestore, refs) {
     return mergedPortfolios;
 }
 
-/** Loads all portfolio data for a day by streaming part references. */
-async function loadFullDayMap(firestore, partRefs) {
+// Modify loadFullDayMap to accept config
+/**
+ * Loads all portfolio data for a day by streaming part references.
+ * @param {Firestore} firestore A Firestore client instance.
+ * @param {Firestore.DocumentReference[]} partRefs Array of part document references.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<object>} A single map of { [userId]: portfolioData }.
+ */
+async function loadFullDayMap(firestore, partRefs, config) {
     if (partRefs.length === 0) return {};
     logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
-    const fullMap = await loadDataByRefs(firestore, partRefs);
+    // Pass config here
+    const fullMap = await loadDataByRefs(firestore, partRefs, config);
     logger.log('TRACE', `Full day map loaded with ${Object.keys(fullMap).length} users.`);
     return fullMap;
 }

package/functions/computation-system/utils/utils.js

@@ -1,20 +1,19 @@
 /**
  * @fileoverview Helper functions for the Unified Computation System.
- * This module imports all calculations and utilities from the single
- * `aiden-shared-calculations-unified` package and categorizes them for the orchestrator.
+ * Categorizes calculations and provides utility functions like retry logic,
+ * batch committing, date generation, parallel processing, and finding earliest dates.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const { logger } = require("sharedsetup")(__filename);
+// Import calculations and utils from the unified package
 const { calculations, utils } = require('aiden-shared-calculations-unified');
+const { withRetry } = utils; // Get withRetry from the package utils
 
-const { withRetry } = utils; // Get withRetry from the package.
-
-// --- Configuration ---
-const BATCH_SIZE_LIMIT = 500;
-const MAX_CONCURRENT_DATES = 3;
+// REMOVE Constants like BATCH_SIZE_LIMIT, MAX_CONCURRENT_DATES
 
 // --- Calculation Categorization ---
+// This relies on the structure exported by 'aiden-shared-calculations-unified'
 const HISTORICAL_CALC_NAMES = new Set([
     'paper-vs-diamond-hands', 'smart-money-flow', 'profitability-migration',
     'user-profitability-tracker', 'sector-rotation', 'crowd-conviction-score',
@@ -25,6 +24,7 @@ const HISTORICAL_CALC_NAMES = new Set([
 const historicalCalculations = {};
 const dailyCalculations = {};
 
+// Assuming 'calculations' from the package has the same category structure
 for (const category in calculations) {
     for (const calcName in calculations[category]) {
         if (HISTORICAL_CALC_NAMES.has(calcName)) {
@@ -37,22 +37,28 @@ for (const category in calculations) {
     }
 }
 
-
 // --- UTILITY FUNCTIONS ---
 
-/** Commits Firestore batch writes in chunks to respect the 500 operation limit. */
-async function commitBatchInChunks(firestore, writes, operationName) {
+/**
+ * Commits Firestore batch writes in chunks based on config limit.
+ * @param {Firestore} firestore A Firestore client instance.
+ * @param {Array<object>} writes Array of { ref: DocumentReference, data: object }.
+ * @param {string} operationName Name for logging.
+ * @param {object} config The computation system configuration object.
+ */
+async function commitBatchInChunks(firestore, writes, operationName, config) {
+    const batchSizeLimit = config.batchSizeLimit || 450; // Use config with default
     if (writes.length === 0) {
         logger.log('WARN', `[${operationName}] No writes to commit.`);
         return;
     }
-    for (let i = 0; i < writes.length; i += BATCH_SIZE_LIMIT) {
+    for (let i = 0; i < writes.length; i += batchSizeLimit) { // Use config limit
         const batch = firestore.batch();
-        const chunk = writes.slice(i, i + BATCH_SIZE_LIMIT);
+        const chunk = writes.slice(i, i + batchSizeLimit); // Use config limit
         chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
-        
-        const chunkNum = (i / BATCH_SIZE_LIMIT) + 1;
-        const totalChunks = Math.ceil(writes.length / BATCH_SIZE_LIMIT);
+
+        const chunkNum = Math.floor(i / batchSizeLimit) + 1; // Use config limit
+        const totalChunks = Math.ceil(writes.length / batchSizeLimit); // Use config limit
         await withRetry(
             () => batch.commit(),
             `${operationName} (Chunk ${chunkNum}/${totalChunks})`
@@ -67,22 +73,35 @@ async function commitBatchInChunks(firestore, writes, operationName) {
 function getExpectedDateStrings(startDate, endDate) {
     const dateStrings = [];
     if (startDate <= endDate) {
-        for (let d = new Date(startDate); d <= endDate; d.setUTCDate(d.getUTCDate() + 1)) {
+        // Ensure dates are treated as UTC to avoid timezone issues
+        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
+        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
+        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) {
             dateStrings.push(new Date(d).toISOString().slice(0, 10));
         }
     }
     return dateStrings;
 }
 
-/** Processes job objects in parallel batches. */
-async function processJobsInParallel(jobs, taskFunction, passName) {
+
+/**
+ * Processes job objects in parallel batches based on config limit.
+ * @param {Array<object>} jobs Array of jobs, each with { date, missing }.
+ * @param {Function} taskFunction Async function to process a job (receives Date, missing array).
+ * @param {string} passName Name for logging.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<Array<PromiseSettledResult>>} Array of promise settlement results.
+ */
+async function processJobsInParallel(jobs, taskFunction, passName, config) {
     const results = [];
+    const maxConcurrentDates = config.maxConcurrentDates || 3; // Use config
      if (jobs.length > 0) {
-        logger.log('INFO', `[${passName}] Processing ${jobs.length} jobs with ${MAX_CONCURRENT_DATES} parallel workers.`);
-        jobs.sort((a, b) => new Date(a.date) - new Date(b.date)); // Ensure chronological processing
-        for (let i = 0; i < jobs.length; i += MAX_CONCURRENT_DATES) {
-            const jobBatch = jobs.slice(i, i + MAX_CONCURRENT_DATES);
-            const promises = jobBatch.map(job => taskFunction(new Date(job.date), job.missing));
+        logger.log('INFO', `[${passName}] Processing ${jobs.length} jobs with ${maxConcurrentDates} parallel workers.`); // Use config
+        jobs.sort((a, b) => new Date(a.date) - new Date(b.date)); // Ensure chronological
+        for (let i = 0; i < jobs.length; i += maxConcurrentDates) { // Use config
+            const jobBatch = jobs.slice(i, i + maxConcurrentDates); // Use config
+            // taskFunction now receives config implicitly when called via runUnifiedComputation
+            const promises = jobBatch.map(job => taskFunction(new Date(job.date + 'T00:00:00Z'), job.missing)); // Ensure date is parsed correctly
             results.push(...await Promise.allSettled(promises));
         }
     } else {
@@ -92,10 +111,13 @@ async function processJobsInParallel(jobs, taskFunction, passName) {
 }
 
 /**
- * Finds the earliest date document by searching inside the 'snapshots' subcollection
- * of all block documents (e.g., '1M', '2M') within a given collection.
+ * Finds the earliest date document in a collection's snapshots.
+ * @param {Firestore} firestore A Firestore client instance.
+ * @param {string} collectionName The top-level portfolio collection name.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<Date|null>} The earliest date found, or null.
  */
-async function getFirstDateFromCollection(firestore, collectionName) {
+async function getFirstDateFromCollection(firestore, collectionName, config) {
     let earliestDate = null;
     try {
         const blockDocRefs = await withRetry(
@@ -103,29 +125,32 @@ async function getFirstDateFromCollection(firestore, collectionName) {
             `GetBlocks(${collectionName})`
         );
 
-        if (blockDocRefs.length === 0) {
-            logger.log('WARN', `No block documents found in collection: ${collectionName}`);
-            return null;
-        }
+         if (blockDocRefs.length === 0) {
+             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
+             return null;
+         }
+
+         for (const blockDocRef of blockDocRefs) {
+              // Use config subcollection name
+             const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
+                 .where(FieldPath.documentId(), '>=', '2000-01-01') // Basic date format check
+                 .orderBy(FieldPath.documentId(), 'asc')
+                 .limit(1);
+
+             const snapshotSnap = await withRetry(
+                 () => snapshotQuery.get(),
+                 `GetEarliestSnapshot(${blockDocRef.path})`
+             );
+
+             // Validate the ID format is YYYY-MM-DD before parsing
+             if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
+                 const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z'); // Treat as UTC
+                 if (!earliestDate || foundDate < earliestDate) {
+                     earliestDate = foundDate;
+                 }
+             }
+         }
 
-        for (const blockDocRef of blockDocRefs) {
-            const snapshotQuery = blockDocRef.collection('snapshots')
-                .where(FieldPath.documentId(), '>=', '2000-01-01') 
-                .orderBy(FieldPath.documentId(), 'asc')
-                .limit(1);
-
-            const snapshotSnap = await withRetry(
-                () => snapshotQuery.get(),
-                `GetEarliestSnapshot(${blockDocRef.path})`
-            );
-
-            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                const foundDate = new Date(snapshotSnap.docs[0].id);
-                if (!earliestDate || foundDate < earliestDate) {
-                    earliestDate = foundDate;
-                }
-            }
-        }
     } catch (e) {
         logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
     }
@@ -133,32 +158,34 @@ async function getFirstDateFromCollection(firestore, collectionName) {
 }
 
 /**
- * Determines the absolute earliest date from source portfolio data collections.
+ * Determines the absolute earliest date from source portfolio data collections using config.
+ * @param {Firestore} firestore A Firestore client instance.
+ * @param {object} config The computation system configuration object.
+ * @returns {Promise<Date>} The earliest date found or a default fallback date.
  */
-async function getFirstDateFromSourceData(firestore) {
+async function getFirstDateFromSourceData(firestore, config) {
     logger.log('INFO', 'Querying for the earliest date from source portfolio data...');
-    
-    const investorDate = await getFirstDateFromCollection(firestore, 'NormalUserPortfolios');
-    const speculatorDate = await getFirstDateFromCollection(firestore, 'SpeculatorUserPortfolios');
-
-    let earliestDate;
-
-    if (investorDate && speculatorDate) {
-        earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-    } else {
-        earliestDate = investorDate || speculatorDate;
-    }
-
-    if (earliestDate) {
-        logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-        return earliestDate;
-    }
-
-    // Fallback if no source data is found at all.
-    const d = new Date();
-    d.setUTCDate(d.getUTCDate() - 30);
-    logger.log('WARN', `No source data found. Defaulting first date to: ${d.toISOString().slice(0, 10)}`);
-    return d;
+    // Use config for collection names
+    const investorDate = await getFirstDateFromCollection(firestore, config.normalUserPortfolioCollection, config);
+    const speculatorDate = await getFirstDateFromCollection(firestore, config.speculatorPortfolioCollection, config);
+
+     let earliestDate;
+     // Determine the earlier of the two dates found
+     if (investorDate && speculatorDate) {
+         earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
+     } else {
+         earliestDate = investorDate || speculatorDate; // Use whichever one was found
+     }
+
+     if (earliestDate) {
+         logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
+         return earliestDate;
+     } else {
+         // Fallback using config.earliestComputationDate or a hardcoded default
+         const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+         logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
+         return fallbackDate;
+     }
 }
 
 module.exports = {

package/functions/dispatcher/index.js

@@ -10,7 +10,7 @@ const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
  * @param {object} config - Configuration object loaded from the calling function's context.
  * @returns {Function} The Cloud Function handler.
  */
-function createDispatcherHandler(pubsubClient, config) {
+function createDispatcherHandler(config, pubsubClient) {
     return async (message, context) => {
         try {
             // 1. Decode Message

package/functions/generic-api/helpers/api_helpers.js

@@ -4,14 +4,15 @@
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
-const MAX_DATE_RANGE = 100; // Limit queries to a 100-day range
+// REMOVE: const MAX_DATE_RANGE = 100;
 
 /**
- * Validates request parameters.
+ * Validates request parameters using config.
  * @param {object} query - The request query object.
+ * @param {object} config - The Generic API V2 configuration object.
  * @returns {string|null} An error message if validation fails, otherwise null.
  */
-const validateRequest = (query) => {
+const validateRequest = (query, config) => {
     if (!query.computations) return "Missing 'computations' parameter.";
     if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
     if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
@@ -20,9 +21,10 @@ const validateRequest = (query) => {
     const end = new Date(query.endDate);
     if (end < start) return "'endDate' must be after 'startDate'.";
 
+    const maxDateRange = config.maxDateRange || 100; // Use config value with default
     const diffTime = Math.abs(end - start);
     const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
-    if (diffDays > MAX_DATE_RANGE) return `Date range cannot exceed ${MAX_DATE_RANGE} days.`;
+    if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
 
     return null;
 };
@@ -38,7 +40,7 @@ const buildCalculationMap = (unifiedCalculations) => {
         for (const calcName in unifiedCalculations[category]) {
             calcMap[calcName] = {
                 category: category,
-                path: `unified_insights/{date}/results/${category}/computations/${calcName}`
+                // Path construction moved to fetchUnifiedData where config is available
             };
         }
     }
@@ -55,7 +57,7 @@ const getDateStringsInRange = (startDate, endDate) => {
     const dates = [];
     const current = new Date(startDate + 'T00:00:00Z');
     const end = new Date(endDate + 'T00:00:00Z');
-    
+
     while (current <= end) {
         dates.push(current.toISOString().slice(0, 10));
         current.setUTCDate(current.getUTCDate() + 1);
@@ -64,16 +66,22 @@ const getDateStringsInRange = (startDate, endDate) => {
 };
 
 /**
- * Fetches all requested computation data from Firestore for the given date range.
+ * Fetches all requested computation data from Firestore for the given date range using config.
+ * @param {object} config - The Generic API V2 configuration object.
  * @param {Firestore} firestore - A Firestore client instance.
  * @param {Logger} logger - A logger instance.
  * @param {string[]} calcKeys - Array of computation keys to fetch.
  * @param {string[]} dateStrings - Array of dates to fetch for.
- * @param {Object} calcMap - The pre-built calculation lookup map.
+ * @param {Object} calcMap - The pre-built calculation lookup map (only category needed).
  * @returns {Promise<Object>} A nested object of [date][computationKey] = data.
  */
-const fetchUnifiedData = async (firestore, logger, calcKeys, dateStrings, calcMap) => {
+const fetchUnifiedData = async (config, firestore, logger, calcKeys, dateStrings, calcMap) => {
     const response = {};
+    // Use collection/subcollection names from config
+    const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
+    const resultsSub = config.resultsSubcollection || 'results';
+    const compsSub = config.computationsSubcollection || 'computations';
+
     try {
         for (const date of dateStrings) {
             response[date] = {};
@@ -83,14 +91,15 @@ const fetchUnifiedData = async (firestore, logger, calcKeys, dateStrings, calcMa
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
                 if (pathInfo) {
-                    const docRef = firestore.collection('unified_insights').doc(date)
-                        .collection('results').doc(pathInfo.category)
-                        .collection('computations').doc(key);
-                    
+                    // Construct path using config values
+                    const docRef = firestore.collection(insightsCollection).doc(date)
+                        .collection(resultsSub).doc(pathInfo.category)
+                        .collection(compsSub).doc(key);
+
                     docRefs.push(docRef);
                     keyPaths.push(key);
                 } else {
-                    logger.log('WARN', `[${date}] No path found for computation key: ${key}`);
+                    logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
                 }
             }
 
@@ -114,17 +123,19 @@ const fetchUnifiedData = async (firestore, logger, calcKeys, dateStrings, calcMa
 };
 
 /**
- * Creates the main Express request handler for the API.
+ * Creates the main Express request handler for the API, injecting config.
+ * @param {object} config - The Generic API V2 configuration object.
  * @param {Firestore} firestore - A Firestore client instance.
  * @param {Logger} logger - A logger instance.
  * @param {Object} calcMap - The pre-built calculation lookup map.
  * @returns {Function} An async Express request handler.
  */
-const createApiHandler = (firestore, logger, calcMap) => {
+const createApiHandler = (config, firestore, logger, calcMap) => {
     return async (req, res) => {
-        const validationError = validateRequest(req.query);
+        // Pass config to validator
+        const validationError = validateRequest(req.query, config);
         if (validationError) {
-            logger.log('WARN', 'Bad Request', { error: validationError });
+            logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
             return res.status(400).send({ status: 'error', message: validationError });
         }
 
@@ -132,7 +143,8 @@ const createApiHandler = (firestore, logger, calcMap) => {
             const computationKeys = req.query.computations.split(',');
             const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
 
-            const data = await fetchUnifiedData(firestore, logger, computationKeys, dateStrings, calcMap);
+            // Pass config to data fetcher
+            const data = await fetchUnifiedData(config, firestore, logger, computationKeys, dateStrings, calcMap);
 
             res.status(200).send({
                 status: 'success',
@@ -152,5 +164,6 @@ const createApiHandler = (firestore, logger, calcMap) => {
 
 module.exports = {
     buildCalculationMap,
-    createApiHandler,
+    createApiHandler, // Export the factory for the handler
+    // Internal helpers like validateRequest, getDateStringsInRange, fetchUnifiedData are not exported
 };

package/functions/generic-api/index.js

@@ -9,24 +9,26 @@ const { buildCalculationMap, createApiHandler } = require('./helpers/api_helpers
 
 /**
  * Creates and configures the Express app for the Generic API.
- * @param {Firestore} firestore A Firestore client instance.
- * @param {Logger} logger A logger instance.
- * @param {Object} unifiedCalculations The calculations manifest.
+ * @param {object} config - The Generic API V2 configuration object.
+ * @param {object} dependencies - Shared dependencies { firestore, logger }.
+ * @param {Object} unifiedCalculations - The calculations manifest.
  * @returns {express.Application} The configured Express app.
  */
-function createApiApp(firestore, logger, unifiedCalculations) {
+function createApiApp(config, dependencies, unifiedCalculations) {
     const app = express();
-    
+    const { firestore, logger } = dependencies; // Extract dependencies
+
     // --- Pre-compute Calculation Map ---
+    // This doesn't need config directly
     const calcMap = buildCalculationMap(unifiedCalculations);
-    
+
     // --- Middleware ---
     app.use(cors({ origin: true }));
     app.use(express.json());
 
     // --- Main API Endpoint ---
-    // The handler is created with its dependencies injected
-    app.get('/', createApiHandler(firestore, logger, calcMap));
+    // The handler is created with its dependencies and config injected
+    app.get('/', createApiHandler(config, firestore, logger, calcMap));
 
     // --- Health Check Endpoint ---
     app.get('/health', (req, res) => {

package/functions/task-engine/handler_creator.js

@@ -0,0 +1,103 @@
+/**
+ * @fileoverview Factory function to create the Task Engine handler.
+ */
+const { Firestore } = require('@google-cloud/firestore'); // Needed if db is not passed in
+const { PubSub } = require('@google-cloud/pubsub'); // Needed if pubsub is not passed in
+const { logger: defaultLogger } = require("sharedsetup")(__filename); // Use default logger if none provided
+const { core } = require('../../'); // Access core utilities like managers
+const TaskEngineHelpers = require('./helpers');
+const TaskEngineUtils = require('./utils');
+
+/**
+ * Creates the Task Engine Cloud Function handler.
+ * @param {object} config - The Task Engine configuration object (e.g., cfg.taskEngine).
+ * @param {object} [dependencies] - Optional dependencies. If not provided, initializes defaults.
+ * @param {Firestore} [dependencies.firestore] - Firestore client instance.
+ * @param {PubSub} [dependencies.pubsub] - PubSub client instance.
+ * @param {object} [dependencies.logger] - Logger instance.
+ * @returns {Function} The async Cloud Function handler for Pub/Sub messages.
+ */
+function createTaskEngineHandler(config, dependencies = {}) {
+    // Initialize clients using provided dependencies or defaults
+    const firestore = dependencies.firestore || new Firestore();
+    const pubsub = dependencies.pubsub || new PubSub();
+    const logger = dependencies.logger || defaultLogger; // Use provided or default logger
+
+    // Initialize managers and batch manager within this scope
+    const headerManager = new core.utils.IntelligentHeaderManager(firestore, logger, config);
+    const proxyManager = new core.utils.IntelligentProxyManager(firestore, logger, config);
+    const batchManager = new TaskEngineUtils.FirestoreBatchManager(firestore, headerManager, config); // Pass headerManager
+
+    const CLIENTS = {
+        pubsub,
+        nativeFirestore: firestore, // Pass the initialized Firestore instance
+        headerManager,
+        proxyManager,
+        batchManager
+    };
+
+    // Return the actual handler function
+    return async (message) => {
+        // Basic check for message structure (Cloud Functions Pub/Sub trigger)
+        if (!message || !message.data) {
+            logger.log('ERROR', '[TaskEngine Module] Received invalid message structure.', { message });
+            // Acknowledge Pub/Sub by not throwing, but log error
+            return;
+        }
+
+        let task;
+        try {
+            // Decode the message
+            task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
+        } catch (e) {
+            logger.log('ERROR', '[TaskEngine Module] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
+            // Acknowledge Pub/Sub by not throwing
+            return;
+        }
+
+        // Generate a unique ID for logging this specific task execution
+        const taskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'batch'}-${Date.now()}`;
+        logger.log('INFO', `[TaskEngine/${taskId}] Received.`);
+
+        try {
+            // Optional: Random delay
+            await new Promise(resolve => setTimeout(resolve, Math.random() * 500 + 250));
+
+            // Determine the correct handler function based on task type
+            const handlerFunction = {
+                discover: TaskEngineHelpers.handleDiscover,
+                verify: TaskEngineHelpers.handleVerify,
+                update: TaskEngineHelpers.handleUpdate
+            }[task.type];
+
+            // Execute the handler or log an error if the type is unknown
+            if (handlerFunction) {
+                await handlerFunction(task, taskId, CLIENTS, config); // Pass CLIENTS and config
+                logger.log('SUCCESS', `[TaskEngine/${taskId}] Done.`);
+            } else {
+                logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type received: ${task.type}`);
+                // Acknowledge Pub/Sub by not throwing
+            }
+        } catch (error) {
+            // Log the error but crucially *do not re-throw* if you want Pub/Sub to ack the message
+            // Only re-throw if you want Pub/Sub to attempt redelivery based on its settings.
+            // Your existing logic logs but then re-throws - decide if that's the desired behavior.
+            logger.log('ERROR', `[TaskEngine/${taskId}] Failed.`, { errorMessage: error.message, errorStack: error.stack });
+            // throw error; // Optional: Uncomment if you want Pub/Sub to retry
+        } finally {
+             // Ensure batches are flushed eventually, though the batch manager handles timed flushes.
+             // A final flush here might be redundant if the instance stays alive, but could be useful
+             // if the instance might terminate immediately after processing.
+             try {
+                // Consider if flushing on every single message is desired vs relying on timed flush
+                // await batchManager.flushBatches();
+             } catch (flushError) {
+                 logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
+             }
+        }
+    };
+}
+
+module.exports = {
+    createTaskEngineHandler,
+};

package/functions/task-engine/index.js

@@ -4,8 +4,10 @@
 
 const helpers = require('./helpers');
 const utils = require('./utils');
+const { createTaskEngineHandler } = require('./handler_creator'); // <-- Import the new factory
 
 module.exports = {
   helpers,
   utils,
+  createTaskEngineHandler, // <-- Export the factory
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.21",
+  "version": "1.0.23",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [