bulltrackers-module 1.0.151 → 1.0.153

package/functions/appscript-api/index.js

@@ -13,46 +13,16 @@ const { createErrorResponse } = require('./helpers/errors');
  */
 function handlePost(e) {
   try {
-    // Parse the request details from the POST body
     const requestDetails = JSON.parse(e.postData.contents);
     const { url, headers, method, body } = requestDetails;
-
-    if (!url || !headers) {
-      return createErrorResponse("Invalid request: 'url' and 'headers' are required.", 400);
-    }
-
-    // Prepare the external request options
-    const options = {
-      headers: headers,
-      muteHttpExceptions: true,
-      method: method || 'GET',
-    };
-
-    if (body) {
-      options.payload = body;
-      if (!headers['Content-Type']) {
-        headers['Content-Type'] = 'application/json';
-      }
-    }
-
-    // Make the external request using UrlFetchApp
+    if (!url || !headers) { return createErrorResponse("Invalid request: 'url' and 'headers' are required.", 400); }
+    const options = { headers: headers, muteHttpExceptions: true, method: method || 'GET', };
+    if (body) { options.payload = body;
+    if (!headers['Content-Type']) { headers['Content-Type'] = 'application/json'; } }
     const response = UrlFetchApp.fetch(url, options);
-
-    // Package the response to send back
-    const responseData = {
-      statusCode: response.getResponseCode(),
-      headers: response.getHeaders(),
-      body: response.getContentText(),
-    };
-
-    return ContentService.createTextOutput(JSON.stringify(responseData))
-      .setMimeType(ContentService.MimeType.JSON);
-
-  } catch (error) {
-    return createErrorResponse(error.toString(), 500);
-  }
+    const responseData = { statusCode: response.getResponseCode(), headers: response.getHeaders(), body: response.getContentText(), };
+    return ContentService.createTextOutput(JSON.stringify(responseData)) .setMimeType(ContentService.MimeType.JSON);
+  } catch (error) { return createErrorResponse(error.toString(), 500); }
 }
 
-module.exports = {
-  handlePost,
-};
+module.exports = { handlePost };

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -3,209 +3,64 @@
  * Now calculates earliest date PER CALCULATION, not per pass
  */
 
-const { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
-
+const { groupByPass, checkRootDataAvailability, fetchExistingResults, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+const { getExpectedDateStrings } = require('../utils/utils.js');
 const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
     const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
     if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
-    
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
-    
-    const yesterday = new Date();
-    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
-    const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    
-    // Hardcoded earliest dates
-    logger.log('INFO', 'Using hardcoded earliest data dates to bypass faulty discovery.');
-    const earliestPortfolio = new Date('2025-09-25T00:00:00Z');
-    const earliestHistory = new Date('2025-11-05T00:00:00Z');
-    const earliestSocial = new Date('2025-10-30T00:00:00Z');
-    const earliestInsights = new Date('2025-08-26T00:00:00Z');
-    
-    const earliestDates = {
-        portfolio: earliestPortfolio,
-        history: earliestHistory,
-        social: earliestSocial,
-        insights: earliestInsights,
-        absoluteEarliest: [earliestPortfolio, earliestHistory, earliestSocial, earliestInsights].reduce((a, b) => a < b ? a : b)
-    };
-    
-    logger.log('INFO', `Hardcoded map: portfolio=${earliestDates.portfolio.toISOString().slice(0,10)}, history=${earliestDates.history.toISOString().slice(0,10)}, social=${earliestDates.social.toISOString().slice(0,10)}, insights=${earliestDates.insights.toISOString().slice(0,10)}`);
-    
+
+    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history:   new Date('2025-11-05T00:00:00Z'), social:    new Date('2025-10-30T00:00:00Z'), insights:  new Date('2025-08-26T00:00:00Z') };
+    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
+    logger.log('INFO', `Hardcoded earliest dates: ${Object.entries(earliestDates).map(([k,v]) => `${k}=${v.toISOString().slice(0,10)}`).join(', ')}`);
+
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
     if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-    
-    // ============================================
-    // NEW: Calculate earliest date PER CALCULATION
-    // ============================================
     const calcEarliestDates = new Map();
-    
-    for (const calc of calcsInThisPass) {
-        const deps = calc.rootDataDependencies || [];
-        
-        if (deps.length === 0) {
-            // No dependencies = can run from the absolute earliest
-            calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest);
-            continue;
-        }
-        
-        // Find the LATEST earliest date among THIS calculation's dependencies
-        let latestEarliest = new Date(0);
-        for (const dep of deps) {
-            const depDate = earliestDates[dep];
-            if (depDate && depDate > latestEarliest) {
-                latestEarliest = depDate;
-            }
-        }
-        
-        // If this is a historical calculation, add 1 day (needs yesterday)
-        if (calc.isHistorical) {
-            const adjusted = new Date(latestEarliest);
-            adjusted.setUTCDate(adjusted.getUTCDate() + 1);
-            calcEarliestDates.set(calc.name, adjusted);
-            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${adjusted.toISOString().slice(0,10)} (historical, needs ${latestEarliest.toISOString().slice(0,10)} + 1 day)`);
-        } else {
-            calcEarliestDates.set(calc.name, latestEarliest);
-            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${latestEarliest.toISOString().slice(0,10)}`);
-        }
-    }
-    
-    // The pass can start from the EARLIEST calculation's start date
+    for (const calc of calcsInThisPass) { const deps = calc.rootDataDependencies || []; if (!deps.length) { calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest); continue; }
+    const latestDep = new Date(Math.max(...deps.map(d => earliestDates[d]?.getTime() || 0)));
+    const calcDate = calc.isHistorical ? new Date(latestDep.getTime() + 86400000) : latestDep; // +1 day if historical
+    calcEarliestDates.set(calc.name, calcDate);
+    logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${calcDate.toISOString().slice(0,10)}`); }
+
     const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
-    
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} analysis:`);
-    logger.log('INFO', `  Total calculations: ${calcsInThisPass.length}`);
-    logger.log('INFO', `  Pass can start from: ${passEarliestDate.toISOString().slice(0,10)}`);
-    logger.log('INFO', `  Individual calculation date ranges calculated.`);
-    
-    // ============================================
-    // Generate date range from earliest to yesterday
-    // ============================================
-    const startDateUTC = new Date(Date.UTC(passEarliestDate.getUTCFullYear(), passEarliestDate.getUTCMonth(), passEarliestDate.getUTCDate()));
-    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-    
-    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates from ${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length-1]}`);
-    
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} analysis: Total calcs=${calcsInThisPass.length}, can start from ${passEarliestDate.toISOString().slice(0,10)}`);
+    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
     const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
     const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
-    
-    // ============================================
-    // Process each date
-    // ============================================
+    const checkDeps = (calc, rootData, existingResults, dateToProcess) => { if (existingResults[calc.name]) return false;
+    const earliest = calcEarliestDates.get(calc.name);
+    if (earliest && dateToProcess < earliest) return false;
+
+    const missingRoot = (calc.rootDataDependencies || []).filter(dep => !rootData.status[`has${dep[0].toUpperCase() + dep.slice(1)}`]);
+    if (missingRoot.length) return false;
+
+    if (calc.type === 'meta') { const missingComputed = (calc.dependencies || []).filter(d => !existingResults[d]); if (missingComputed.length) return false; } return true; };
+
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        
         try {
-            // Check root data availability
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) {
-                logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);
-                return;
-            }
-            
-            // Fetch existing results
+            if (!rootData) return logger.log('WARN', `[PassRunner] Skipping ${dateStr}: No root data.`);
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            
-            // ============================================
-            // NEW: Filter based on per-calculation earliest dates
-            // ============================================
-            const standardCalcsToRun = standardCalcs.filter(c => {
-                // Skip if already exists
-                if (existingResults[c.name]) {
-                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);
-                    return false;
-                }
-                
-                // Skip if date is before this calculation's earliest date
-                const calcEarliestDate = calcEarliestDates.get(c.name);
-                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
-                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Date before calc's earliest (${calcEarliestDate.toISOString().slice(0,10)}).`);
-                    return false;
-                }
-                
-                // Check root data dependencies
-                const missingDeps = (c.rootDataDependencies || []).filter(dep => {
-                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
-                    if (dep === 'insights') return !rootData.status.hasInsights;
-                    if (dep === 'social') return !rootData.status.hasSocial;
-                    if (dep === 'history') return !rootData.status.hasHistory;
-                    return false;
-                });
-                
-                if (missingDeps.length > 0) {
-                    logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing deps: ${missingDeps.join(', ')}`);
-                    return false;
-                }
-                
-                return true;
-            });
-            
-            const metaCalcsToRun = metaCalcs.filter(c => {
-                if (existingResults[c.name]) return false;
-                
-                const calcEarliestDate = calcEarliestDates.get(c.name);
-                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
-                    logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Date before calc's earliest.`);
-                    return false;
-                }
-                
-                // Check root dependencies
-                const missingRootDeps = (c.rootDataDependencies || []).filter(dep => {
-                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
-                    if (dep === 'insights') return !rootData.status.hasInsights;
-                    if (dep === 'social') return !rootData.status.hasSocial;
-                    if (dep === 'history') return !rootData.status.hasHistory;
-                    return false;
-                });
-                
-                if (missingRootDeps.length > 0) {
-                    logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root deps: ${missingRootDeps.join(', ')}`);
-                    return false;
-                }
-                
-                // Check computed dependencies
-                const missingComputedDeps = (c.dependencies || []).filter(d => !existingResults[d]);
-                if (missingComputedDeps.length > 0) {
-                    logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: ${missingComputedDeps.join(', ')}`);
-                    return false;
-                }
-                
-                return true;
-            });
-            
-            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {
-                logger.log('INFO', `[PassRunner] All eligible calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);
-                return;
-            }
-            
-            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardCalcsToRun.length} standard, ${metaCalcsToRun.length} meta`);
-            
-            if (standardCalcsToRun.length) {
-                await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
-            }
-            if (metaCalcsToRun.length) {
-                await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
-            }
-            
-        } catch (err) {
-            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        }
+            const standardToRun = standardCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
+            const metaToRun     = metaCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
+            if (!standardToRun.length && !metaToRun.length) return logger.log('INFO', `[PassRunner] All calcs complete for ${dateStr}. Skipping.`);
+            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardToRun.length} standard, ${metaToRun.length} meta`);
+            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
+            if (metaToRun.length) await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+        } catch (err) { logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack }); }
     };
-    
-    // Process in batches
-    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
-        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
-        await Promise.all(batch.map(dateStr => processDate(dateStr)));
-    }
-    
+
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+    logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
+    await Promise.all(batch.map(processDate)); }
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };