bulltrackers-module 1.0.581 → 1.0.583

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,7 @@
 /**
  * FILENAME: computation-system/WorkflowOrchestrator.js
  * UPDATED: Fixed 'Version Mismatch' deadlock for historical chains.
+ * UPDATED: Added Force-Run logic for 'isTest' computations on current day.
  */
 
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
@@ -46,12 +47,17 @@ function isDependencyReady(depName, isHistoricalSelf, currentStatusMap, prevStat
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
     const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
     const simulationStatus = { ...dailyStatus };
+    const isToday = dateStr === new Date().toISOString().slice(0, 10);
 
     for (const calc of calcsInPass) {
         const cName = normalizeName(calc.name);
         const stored = simulationStatus[cName];
         const currentHash = calc.hash;
 
+        // [NEW] Rule: 'isTest' computations always re-run on the current day.
+        // This ensures debug/test probes fire immediately to test system changes.
+        const shouldForceRun = isToday && (calc.isTest === true);
+
         // 1. Root Data Check
         const rootCheck = checkRootDependencies(calc, rootDataStatus);
         if (!rootCheck.canRun) {
@@ -65,7 +71,8 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         }
 
         // --- OPTIMIZATION: Early skip if code matches AND data is stable ---
-        if (stored?.hash === currentHash) {
+        // [UPDATED] We bypass this optimization if shouldForceRun is true
+        if (stored?.hash === currentHash && !shouldForceRun) {
             let hasDataDrift = false;
             let isBlocked = false;
             let missingDeps = [];
@@ -161,6 +168,9 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             report.runnable.push({ ...taskPayload, reason: "New Calculation" });
         } else if (stored.hash !== currentHash) {
             report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Hash Mismatch" });
+        } else if (shouldForceRun) {
+            // [NEW] Logic to handle the forced run for Test probes
+            report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Test Computation (Always Run Today)" });
         } else if (hasDataDrift) {
             report.runnable.push({ ...taskPayload, reason: "Input Data Changed" });
         }

package/functions/computation-system/context/ManifestBuilder.js

@@ -1,13 +1,9 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/context/ManifestBuilder.js
- * }
- */
 /**
  * @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
  * UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
- * UPDATED: Whitelisted 'rootDataSeries', 'dependencySeries', and 'mandatoryRoots' metadata fields.
+ * UPDATED: Whitelisted 'rootDataSeries', 'dependencySeries', 'mandatoryRoots', and 'isTest'.
+ * UPDATED: Whitelisted 'schedule' to allow calculations to define their own execution cadence (Daily/Weekly/Monthly).
+ * UPDATED: Whitelisted 'ttlDays' to allow calculations to define custom retention policies.
  */
 const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
 const { normalizeName } = require('../utils/utils');
@@ -192,7 +188,7 @@ function buildManifest(productLinesToRun = [], calculations) {
         if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" missing static getDependencies().`); hasFatalError = true; return; }
         
         const metadata     = Class.getMetadata();
-        const weight = estimateComplexity(Class, metadata)
+        const weight       = estimateComplexity(Class, metadata)
         const dependencies = Class.getDependencies().map(normalizeName);
         const codeStr      = Class.toString();
         const selfCodeHash = generateCodeHash(codeStr);
@@ -233,22 +229,24 @@ function buildManifest(productLinesToRun = [], calculations) {
         const intrinsicHash = generateCodeHash(compositeHashString);
 
         const manifestEntry = {
-            name: normalizedName,
-            class: Class,
-            category: folderName === 'core' && metadata.category ? metadata.category : folderName,     
+            name:          normalizedName,
+            class:         Class,
+            category:      folderName === 'core' && metadata.category ? metadata.category : folderName,     
             sourcePackage: folderName,   
-            type: metadata.type,
-            isPage: metadata.isPage === true,
-            isHistorical: metadata.isHistorical !== undefined ? metadata.isHistorical : false, 
+            type:                 metadata.type,
+            isPage:               metadata.isPage === true,
+            isHistorical:         metadata.isHistorical !== undefined ? metadata.isHistorical : false, 
+            isTest:               metadata.isTest === true, 
             rootDataDependencies: metadata.rootDataDependencies || [],
-            // [NEW] Pass Series & Mandatory Config
-            rootDataSeries: metadata.rootDataSeries || null,
-            dependencySeries: metadata.dependencySeries || null,
-            mandatoryRoots: metadata.mandatoryRoots || [], // [NEW]
-            
-            canHaveMissingRoots: metadata.canHaveMissingRoots || false,
-            userType: metadata.userType,
+            rootDataSeries:       metadata.rootDataSeries || null,
+            dependencySeries:     metadata.dependencySeries || null,
+            mandatoryRoots:       metadata.mandatoryRoots || [],
+            canHaveMissingRoots:  metadata.canHaveMissingRoots || false,
+            userType:             metadata.userType,
             dependencies: dependencies,
+            schedule:             metadata.schedule || null,
+            // [NEW] Added TTL Policy to Manifest
+            ttlDays:              metadata.ttlDays, 
             pass: 0,
             hash: intrinsicHash,
             weight: weight,

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -4,6 +4,7 @@
  * UPDATED: Implemented "Fast-Forward" Scanning Loop to skip empty dates efficiently.
  * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
  * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
+ * UPDATED: Added Schedule Awareness (Daily, Weekly, Monthly) to filter tasks by date.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -19,6 +20,42 @@ const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
 const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; 
 
+// =============================================================================
+// HELPER: Schedule Logic
+// =============================================================================
+/**
+ * Checks if a computation is scheduled to run on a specific date.
+ * Defaults to DAILY if no schedule is present.
+ * * @param {string} dateStr - YYYY-MM-DD string
+ * @param {Object} scheduleConfig - { type: 'DAILY'|'WEEKLY'|'MONTHLY', days: [] }
+ * @returns {boolean} True if the computation should run
+ */
+function isComputationScheduled(dateStr, scheduleConfig) {
+    // Default: Run every day if no schedule is provided or explicitly DAILY
+    if (!scheduleConfig || !scheduleConfig.type || scheduleConfig.type === 'DAILY') {
+        return true;
+    }
+
+    const date = new Date(dateStr + 'T00:00:00Z'); // Ensure UTC parsing
+    
+    // Weekly Schedule: Check Day of Week (0=Sun, 1=Mon, ..., 6=Sat)
+    if (scheduleConfig.type === 'WEEKLY') {
+        const dayOfWeek = date.getUTCDay();
+        const validDays = Array.isArray(scheduleConfig.days) ? scheduleConfig.days : [scheduleConfig.day];
+        return validDays.includes(dayOfWeek);
+    }
+
+    // Monthly Schedule: Check Day of Month (1-31)
+    if (scheduleConfig.type === 'MONTHLY') {
+        const dayOfMonth = date.getUTCDate();
+        const validDates = Array.isArray(scheduleConfig.days) ? scheduleConfig.days : [scheduleConfig.day];
+        return validDates.includes(dayOfMonth);
+    }
+
+    // Fallback default
+    return true;
+}
+
 // =============================================================================
 // HELPER: Ledger Awareness
 // =============================================================================
@@ -281,19 +318,33 @@ async function handlePassVerification(config, dependencies, computationManifest,
     const missingTasks = [];
 
     for (const date of sessionDates) {
+        // [SCHEDULE CHECK] Filter tasks that are not scheduled for this date
+        const scheduledComputations = calcsInPass.filter(c => 
+            isComputationScheduled(date, c.schedule)
+        );
+
+        if (scheduledComputations.length === 0) continue;
+
         const [dailyStatus, availability] = await Promise.all([
             fetchComputationStatus(date, config, dependencies),
             checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
         ]);
 
         let prevDailyStatus = null;
-        if (calcsInPass.some(c => c.isHistorical)) {
+        if (scheduledComputations.some(c => c.isHistorical)) {
              const prevD = new Date(date + 'T00:00:00Z');
              prevD.setUTCDate(prevD.getUTCDate() - 1);
              prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
         }
 
-        const report = analyzeDateExecution(date, calcsInPass, availability ? availability.status : {}, dailyStatus, manifestMap, prevDailyStatus);
+        const report = analyzeDateExecution(
+            date, 
+            scheduledComputations, // Use filtered list 
+            availability ? availability.status : {}, 
+            dailyStatus, 
+            manifestMap, 
+            prevDailyStatus
+        );
         
         const pending = [...report.runnable, ...report.reRuns];
         
@@ -327,6 +378,16 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
     const passes = groupByPass(computationManifest);
     const calcsInPass = passes[passToRun] || [];
     
+    // [SCHEDULE CHECK] Filter tasks that are not scheduled for this date
+    const scheduledComputations = calcsInPass.filter(c => 
+        isComputationScheduled(date, c.schedule)
+    );
+
+    if (scheduledComputations.length === 0) {
+        logger.log('INFO', `[Sweep] ${date} has no scheduled tasks for Pass ${passToRun}. Ignoring.`);
+        return { dispatched: 0 };
+    }
+
     // 1. Analyze specific date
     const [dailyStatus, availability] = await Promise.all([
         fetchComputationStatus(date, config, dependencies),
@@ -334,14 +395,21 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
     ]);
     
     let prevDailyStatus = null;
-    if (calcsInPass.some(c => c.isHistorical)) {
+    if (scheduledComputations.some(c => c.isHistorical)) {
          const prevD = new Date(date + 'T00:00:00Z');
          prevD.setUTCDate(prevD.getUTCDate() - 1);
          prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
     }
 
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const report = analyzeDateExecution(date, calcsInPass, availability ? availability.status : {}, dailyStatus, manifestMap, prevDailyStatus);
+    const report = analyzeDateExecution(
+        date, 
+        scheduledComputations, // Use filtered list
+        availability ? availability.status : {}, 
+        dailyStatus, 
+        manifestMap, 
+        prevDailyStatus
+    );
     const pending = [...report.runnable, ...report.reRuns];
 
     if (pending.length === 0) {
@@ -491,10 +559,21 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             break;
         }
 
-        // 2. Analyze Date
+        // 2. [SCHEDULE CHECK] Filter computations scheduled for this specific date
+        const scheduledComputations = calcsInThisPass.filter(c => 
+            isComputationScheduled(selectedDate, c.schedule)
+        );
+
+        // Optimization: If nothing is scheduled for today, skip expensive DB checks
+        if (scheduledComputations.length === 0) {
+            currentCursor++;
+            continue;
+        }
+
+        // 3. Analyze Date
         const earliestDates = await getEarliestDataDates(config, dependencies);
         let prevDailyStatusPromise = Promise.resolve(null);
-        if (calcsInThisPass.some(c => c.isHistorical)) {
+        if (scheduledComputations.some(c => c.isHistorical)) {
             const prevD = new Date(selectedDate + 'T00:00:00Z');
             prevD.setUTCDate(prevD.getUTCDate() - 1);
             if (prevD >= earliestDates.absoluteEarliest) {
@@ -509,7 +588,14 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
         ]);
 
         if (availability && availability.status) {
-            const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
+            const report = analyzeDateExecution(
+                selectedDate, 
+                scheduledComputations, // Use filtered list
+                availability.status, 
+                dailyStatus, 
+                manifestMap, 
+                prevDailyStatus
+            );
             let rawTasks = [...report.runnable, ...report.reRuns];
 
             if (rawTasks.length > 0) {

package/functions/computation-system/persistence/ResultCommitter.js

@@ -2,6 +2,7 @@
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
  * UPDATED: Fixed bug where Alert Computations failed to trigger Pub/Sub on empty FINAL flush.
  * UPDATED: Added support for 'isPage' mode to store per-user data in subcollections.
+ * UPDATED: Implemented TTL retention policy. Defaults to 90 days from the computation date.
  */
 const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -17,6 +18,7 @@ const zlib                                 = require('zlib');
 const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 const CONTRACTS_COLLECTION        = 'system_contracts';
+const DEFAULT_TTL_DAYS            = 90;
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false, options = {}) {
     const successUpdates = {};
@@ -56,8 +58,11 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
         // Check metadata for alert flag (defaults to false)
         const isAlertComputation = calc.manifest.isAlertComputation === true;
-        // [NEW] Check metadata for page flag (defaults to false)
+        // Check metadata for page flag (defaults to false)
         const isPageComputation = calc.manifest.isPage === true;
+        
+        // [NEW] Determine TTL Policy
+        const ttlDays = calc.manifest.ttlDays !== undefined ? calc.manifest.ttlDays : DEFAULT_TTL_DAYS;
 
         try {
             const result = await calc.getResult();
@@ -129,24 +134,31 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
-            // [NEW] Page Computation Logic (Fan-Out)
+            // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
                 const mainDocRef = db.collection(config.resultsCollection).doc(dStr)
                     .collection(config.resultsSubcollection).doc(calc.manifest.category)
                     .collection(config.computationsSubcollection).doc(name);
 
+                // Calculate expiration based on computation date
+                const expireAt = calculateExpirationDate(dStr, ttlDays);
+
                 // 1. Fan-out writes for each user
                 const pageWrites = [];
                 // We assume result is { [cid]: { ...data... }, [cid2]: { ... } }
                 for (const [cid, userData] of Object.entries(result)) {
-                    // STRATEGY: Use a fixed collection 'pages' so we can clean it up later
                     // Path: .../{ComputationName}/pages/{cid}
                     const userDocRef = mainDocRef.collection('pages').doc(cid);
                     
+                    // Inject _expireAt into the user data payload for free deletion
+                    const payload = (typeof userData === 'object' && userData !== null) 
+                        ? { ...userData, _expireAt: expireAt }
+                        : { value: userData, _expireAt: expireAt };
+
                     pageWrites.push({
                         ref: userDocRef,
-                        data: userData, // Write the raw data directly
+                        data: payload,
                         options: { merge: false } // Overwrite specifically for this run
                     });
                 }
@@ -156,17 +168,17 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     await commitBatchInChunks(config, deps, pageWrites, `${name}::PageFanOut`);
                     runMetrics.io.writes += pageWrites.length;
                     runMetrics.storage.keys = pageWrites.length;
-                    logger.log('INFO', `[PageMode] ${name}: Wrote ${pageWrites.length} user pages.`);
+                    logger.log('INFO', `[PageMode] ${name}: Wrote ${pageWrites.length} user pages. TTL: ${ttlDays}d.`);
                 }
 
-                // 3. Write the "Header" document (Important for Status/Metrics)
-                // We store NO data here, just metadata saying "Go look in /pages"
+                // 3. Write the "Header" document (Important for Status/Metrics/TTL)
                 if (flushMode !== 'INTERMEDIATE') {
                     const headerData = {
                         _completed: true,
                         _isPageMode: true, // Flag for readers to know where to look
                         _pageCount: pageWrites.length,
-                        _lastUpdated: new Date().toISOString()
+                        _lastUpdated: new Date().toISOString(),
+                        _expireAt: expireAt // Ensure the header also gets deleted
                     };
                     
                     await mainDocRef.set(headerData, { merge: true });
@@ -187,7 +199,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; // Skip the standard writeSingleResult logic
             }
             
-            // Standard Computation Logic (Compression or Sharding)
+            // Standard Computation Logic (Compression or Sharding) with TTL
             if (typeof result === 'object') runMetrics.storage.keys = Object.keys(result).length;
             const resultKeys = Object.keys(result || {});
             const isMultiDate = resultKeys.length > 0 && resultKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
@@ -196,8 +208,12 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 const datePromises  = resultKeys.map((historicalDate) => fanOutLimit(async () => {
                     const dailyData = result[historicalDate];
                     if (!dailyData || Object.keys(dailyData).length === 0) return;
+                    
+                    // Calculate specific TTL for this historical date
+                    const dailyExpireAt = calculateExpirationDate(historicalDate, ttlDays);
+                    
                     const historicalDocRef = db.collection(config.resultsCollection).doc(historicalDate).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, logger, config, deps, 0, 'STANDARD', false);
+                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt);
                     runMetrics.io.writes += stats.opCounts.writes;
                     runMetrics.io.deletes += stats.opCounts.deletes;
 
@@ -213,8 +229,11 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 
                 if (calc.manifest.hash) { successUpdates[name] = { hash: calc.manifest.hash, simHash, resultHash, dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, category: calc.manifest.category, composition: calc.manifest.composition, metrics: runMetrics }; }
             } else {
+                // Calculate TTL for the main run date
+                const runExpireAt = calculateExpirationDate(dStr, ttlDays);
+
                 const mainDocRef = db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, logger, config, deps, currentShardIndex, flushMode, isInitialWrite);
+                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt);
                 
                 runMetrics.storage.sizeBytes  = writeStats.totalSize;
                 runMetrics.storage.isSharded  = writeStats.isSharded;
@@ -291,7 +310,7 @@ async function fetchContracts(db, calcNames) {
     return map;
 }
 
-async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false) {
+async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null) {
     const opCounts = { writes: 0, deletes: 0 };
     
     // Always check for shards if we might compress
@@ -312,7 +331,7 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
         if (rawBuffer.length > 50 * 1024) { 
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             if (compressedBuffer.length < 900 * 1024) {
-                logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB.`);
+                logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. TTL: ${expireAt ? expireAt.toISOString().split('T')[0] : 'None'}`);
                 
                 const payloadBuffer = Buffer.from(compressedBuffer); 
 
@@ -323,6 +342,11 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
                     payload: payloadBuffer 
                 };
                 
+                // Inject TTL if present
+                if (expireAt) {
+                    compressedPayload._expireAt = expireAt;
+                }
+                
                 // Self-Healing: If we are writing compressed, we MUST ensure shards are gone.
                 if (wasSharded) {
                     const updates = [];
@@ -362,7 +386,7 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
         if (committed) break;
         const constraints = strategies[attempt];
         try {
-            const updates = await prepareAutoShardedWrites(result, docRef, logger, constraints.bytes, constraints.keys, startShardIndex, flushMode);
+            const updates = await prepareAutoShardedWrites(result, docRef, logger, constraints.bytes, constraints.keys, startShardIndex, flushMode, expireAt);
             
             if (shouldWipeShards) {
                  const shardCol = docRef.collection('_shards');
@@ -439,15 +463,26 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
     return finalStats;
 }
 
-async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 * 1024, maxKeys = null, startShardIndex = 0, flushMode = 'STANDARD') {
+async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 * 1024, maxKeys = null, startShardIndex = 0, flushMode = 'STANDARD', expireAt = null) {
     const OVERHEAD_ALLOWANCE = 20 * 1024; const CHUNK_LIMIT = maxBytes - OVERHEAD_ALLOWANCE;
     const totalSize = calculateFirestoreBytes(result); const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16;
     const writes = []; const shardCollection = docRef.collection('_shards');
     let currentChunk = {}; let currentChunkSize = 0; let currentKeyCount = 0; 
     let shardIndex = startShardIndex;
 
+    // Helper to inject TTL into chunk/payload
+    const injectTTL = (data) => {
+        if (expireAt) {
+            return { ...data, _expireAt: expireAt };
+        }
+        return data;
+    };
+
     if (!maxKeys && (totalSize + docPathSize) < CHUNK_LIMIT && flushMode === 'STANDARD' && startShardIndex === 0) {  
-        const data = { ...result, _completed: true, _sharded: false, _lastUpdated: new Date().toISOString() };  
+        const data = { ...result, _completed: true, _sharded: false, _lastUpdated: new Date().toISOString() };
+        // If single doc write (no shards), just inject expireAt into the main doc
+        if (expireAt) data._expireAt = expireAt;
+        
         return [{ ref: docRef, data, options: { merge: true } }];  
     }
 
@@ -457,14 +492,18 @@ async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 *
         const byteLimitReached = (currentChunkSize + itemSize > CHUNK_LIMIT); const keyLimitReached = (maxKeys && currentKeyCount + 1 >= maxKeys);
         
         if (byteLimitReached || keyLimitReached) { 
-            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); 
+            // Write chunk with TTL
+            const chunkData = injectTTL(currentChunk);
+            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: chunkData, options: { merge: false } }); 
             shardIndex++; currentChunk = {}; currentChunkSize = 0; currentKeyCount = 0;
         }
         currentChunk[key] = value; currentChunkSize += itemSize; currentKeyCount++;
     }
     
     if (Object.keys(currentChunk).length > 0) { 
-        writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } });
+        // Write remaining chunk with TTL
+        const chunkData = injectTTL(currentChunk);
+        writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: chunkData, options: { merge: false } });
         shardIndex++; 
     }
     
@@ -475,6 +514,9 @@ async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 *
             _shardCount: shardIndex, 
             _lastUpdated: new Date().toISOString() 
         };
+        // Ensure the pointer/metadata document also has the TTL
+        if (expireAt) pointerData._expireAt = expireAt;
+        
         writes.push({ ref: docRef, data: pointerData, options: { merge: true } }); 
     }
 
@@ -489,7 +531,7 @@ async function deleteOldCalculationData(dateStr, oldCategory, calcName, config,
         
         const batch = db.batch(); let ops = 0;
 
-        // [NEW] Clean up 'pages' subcollection if it exists (for Page Mode)
+        // Clean up 'pages' subcollection if it exists (for Page Mode)
         const pagesCol = oldDocRef.collection('pages');
         // Note: listDocuments works nicely for small-ish collections. 
         // If 'pages' has 10k+ docs, we rely on the implementation of listDocuments 
@@ -520,4 +562,17 @@ function calculateFirestoreBytes(value) {
     if (typeof value === 'object') { let sum = 0; for (const k in value) { if (Object.prototype.hasOwnProperty.call(value, k)) { sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]); } } return sum; } return 0;
 }
 
+/**
+ * Calculates the expiration date based on the computation date context (not execution time).
+ * @param {string} dateStr - The YYYY-MM-DD string of the computation context.
+ * @param {number} ttlDays - Days to retain data.
+ * @returns {Date} The expiration Date object.
+ */
+function calculateExpirationDate(dateStr, ttlDays) {
+    const base = new Date(dateStr);
+    // Add days to the base computation date
+    base.setDate(base.getDate() + ttlDays);
+    return base;
+}
+
 module.exports = { commitResults };

package/functions/generic-api/user-api/helpers/verification/verification_helpers.js

@@ -11,9 +11,22 @@ const { FieldValue } = require('@google-cloud/firestore');
  * Generates an OTP and returns it to the user to place in their bio.
  */
 async function initiateVerification(req, res, dependencies, config) {
-    const { db } = dependencies;
+    const { db, logger } = dependencies;
     const { username } = req.body;
+    
+    // Safety check: ensure config exists
+    if (!config) {
+        logger?.log('ERROR', '[Verification] Config is undefined or null in initiateVerification');
+        return res.status(500).json({ error: "Configuration not initialized." });
+    }
+    
     const { verificationsCollection } = config;
+    
+    // Safety check: ensure required config values exist
+    if (!verificationsCollection) {
+        logger?.log('ERROR', '[Verification] verificationsCollection is not defined in config');
+        return res.status(500).json({ error: "Configuration error: verificationsCollection not initialized." });
+    }
 
     if (!username || typeof username !== 'string') {
         return res.status(400).json({ error: "Invalid username." });
@@ -51,15 +64,40 @@ async function initiateVerification(req, res, dependencies, config) {
 async function finalizeVerification(req, res, dependencies, config) {
     const { db, logger } = dependencies;
     const { username } = req.body;
+    
+    // Safety check: ensure config exists and is an object
+    if (!config || typeof config !== 'object') {
+        logger.log('ERROR', '[Verification] Config is undefined, null, or not an object', { configType: typeof config, configValue: config });
+        return res.status(500).json({ error: "Configuration not initialized." });
+    }
+    
+    // Safe destructuring with defaults to prevent ReferenceErrors
     const { 
-        verificationsCollection, 
-        signedInUsersCollection,
-        proxyConfig, 
-        headerConfig,
-        pubsubTopicUserFetch,
-        pubsubTopicUserFetchOnDemand,
-        pubsubTopicSocialFetch
-    } = config;
+        verificationsCollection = null, 
+        signedInUsersCollection = null,
+        proxyConfig = null, 
+        headerConfig = null,
+        pubsubTopicUserFetch = null,
+        pubsubTopicUserFetchOnDemand = null,
+        pubsubTopicSocialFetch = null
+    } = config || {};
+    
+    // Safety check: ensure required config values exist
+    if (!signedInUsersCollection) {
+        logger.log('ERROR', '[Verification] signedInUsersCollection is not defined in config', { 
+            configKeys: Object.keys(config),
+            signedInUsersCollection: config.signedInUsersCollection 
+        });
+        return res.status(500).json({ error: "Configuration error: signedInUsersCollection not initialized." });
+    }
+    
+    if (!verificationsCollection) {
+        logger.log('ERROR', '[Verification] verificationsCollection is not defined in config', { 
+            configKeys: Object.keys(config),
+            verificationsCollection: config.verificationsCollection 
+        });
+        return res.status(500).json({ error: "Configuration error: verificationsCollection not initialized." });
+    }
     
     // Use on-demand topic for user signup (API-triggered)
     const taskEngineTopic = pubsubTopicUserFetchOnDemand || pubsubTopicUserFetch || 'etoro-user-fetch-topic-ondemand';
@@ -173,7 +211,6 @@ async function finalizeVerification(req, res, dependencies, config) {
         // Generate a requestId for tracking and to ensure finalizeOnDemandRequest runs
         // This is critical - without requestId, the root data indexer and computations won't be triggered
         const requestId = `signup-${realCID}-${Date.now()}`;
-        const { signedInUsersCollection } = config;
         
         // Create request tracking document (similar to user sync requests)
         try {
@@ -262,8 +299,24 @@ async function finalizeVerification(req, res, dependencies, config) {
         });
 
     } catch (error) {
-        logger.log('ERROR', `[Verification] System error for ${username}`, error);
-        return res.status(500).json({ error: error.message });
+        // Enhanced error logging to capture full error details
+        const errorDetails = {
+            message: error?.message || 'Unknown error',
+            stack: error?.stack || 'No stack trace',
+            name: error?.name || 'Error',
+            configExists: !!config,
+            configKeys: config ? Object.keys(config) : [],
+            signedInUsersCollection: config?.signedInUsersCollection || 'NOT SET',
+            verificationsCollection: config?.verificationsCollection || 'NOT SET'
+        };
+        
+        logger.log('ERROR', `[Verification] System error for ${username}`, errorDetails);
+        logger.log('ERROR', `[Verification] Full error object:`, error);
+        
+        return res.status(500).json({ 
+            error: error?.message || 'An unexpected error occurred during verification.',
+            details: process.env.NODE_ENV === 'development' ? errorDetails : undefined
+        });
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.581",
+  "version": "1.0.583",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [