bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/legacy/computation_dispatcherold.js

@@ -1,944 +0,0 @@
-/**
- * OLD VERSION KEPT FOR REFERENCE, UNUSED IN PIPELINE
- * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher.
- * UPDATED: Implemented "Fast-Forward" Scanning Loop to skip empty dates efficiently.
- * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
- * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
- * UPDATED: Added Schedule Awareness (Daily, Weekly, Monthly) to filter tasks by date.
- *
- */
-
-const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
-const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.js');
-const { PubSubUtils } = require('../../core/utils/pubsub_utils.js');
-const { fetchComputationStatus } = require('../persistence/StatusRepository.js');
-const { checkRootDataAvailability } = require('../data/AvailabilityChecker.js');
-const { runFinalSweepCheck } = require('../tools/FinalSweepReporter.js'); // [NEW]
-const crypto = require('crypto');
-
-const OOM_THRESHOLD_MB = 1500; // Unused
-const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
-const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
-const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15;
-
-// =============================================================================
-// HELPER: Firestore Timestamp Conversion
-// =============================================================================
-/**
- * Converts a Firestore Timestamp or Date to milliseconds.
- * Firestore stores Date objects as Timestamp objects, which have a .toDate() method.
- * This function handles both cases correctly.
- * @param {any} field - Firestore Timestamp, Date object, or string
- * @returns {number} Milliseconds since epoch, or 0 if invalid
- */
-function getMillis(field) {
-    if (!field) return 0;
-    // Handle Firestore Timestamp (has .toDate() method)
-    if (field.toDate && typeof field.toDate === 'function') {
-        return field.toDate().getTime();
-    }
-    // Handle standard Date object or string
-    const date = new Date(field);
-    return isNaN(date.getTime()) ? 0 : date.getTime();
-} 
-
-// =============================================================================
-// HELPER: Schedule Logic
-// =============================================================================
-/**
- * Checks if a computation is scheduled to run on a specific date.
- * Defaults to DAILY if no schedule is present.
- * * @param {string} dateStr - YYYY-MM-DD string
- * @param {Object} scheduleConfig - { type: 'DAILY'|'WEEKLY'|'MONTHLY', days: [] }
- * @returns {boolean} True if the computation should run
- */
-function isComputationScheduled(dateStr, scheduleConfig) {
-    // Default: Run every day if no schedule is provided or explicitly DAILY
-    if (!scheduleConfig || !scheduleConfig.type || scheduleConfig.type === 'DAILY') {
-        return true;
-    }
-
-    const date = new Date(dateStr + 'T00:00:00Z'); // Ensure UTC parsing
-    
-    // Weekly Schedule: Check Day of Week (0=Sun, 1=Mon, ..., 6=Sat)
-    if (scheduleConfig.type === 'WEEKLY') {
-        const dayOfWeek = date.getUTCDay();
-        const validDays = Array.isArray(scheduleConfig.days) ? scheduleConfig.days : [scheduleConfig.day];
-        return validDays.includes(dayOfWeek);
-    }
-
-    // Monthly Schedule: Check Day of Month (1-31)
-    if (scheduleConfig.type === 'MONTHLY') {
-        const dayOfMonth = date.getUTCDate();
-        const validDates = Array.isArray(scheduleConfig.days) ? scheduleConfig.days : [scheduleConfig.day];
-        return validDates.includes(dayOfMonth);
-    }
-
-    // Fallback default
-    return true;
-}
-
-// =============================================================================
-// HELPER: Ledger Awareness
-// =============================================================================
-async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false) {
-    if (!tasks || tasks.length === 0) return [];
-    if (forceRun) return tasks; 
-    
-    const checkPromises = tasks.map(async (t) => {
-        const taskName = normalizeName(t.name);
-        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${taskName}`;
-        const snap = await db.doc(ledgerPath).get();
-        
-        if (snap.exists) {
-            const data = snap.data();
-            const isActive = ['PENDING', 'IN_PROGRESS'].includes(data.status);
-
-            if (isActive) {
-                const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
-                
-                if ((Date.now() - lastActivityTime) > STALE_LOCK_THRESHOLD_MS) {
-                    if (logger) logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}.`);
-                    return t; 
-                }
-                return null; 
-            }
-            // Note: We do NOT filter COMPLETED here anymore for Sweep. 
-            // If the Orchestrator says it needs to run, we run it.
-        }
-        return t;
-    });
-    const results = await Promise.all(checkPromises);
-    return results.filter(t => t !== null);
-}
-
-// =============================================================================
-// HELPER: SimHash Stability
-// =============================================================================
-async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus, manifestMap) {
-    const { db, logger } = dependencies;
-    const resolvedTasks = [];
-    const remainingTasks = [];
-    const simHashCache = new Map();
-
-    for (const task of tasks) {
-        const currentStatus = dailyStatus ? dailyStatus[task.name] : null;
-        const manifestItem = manifestMap.get(normalizeName(task.name));
-
-        if (currentStatus && currentStatus.simHash && manifestItem) {
-            let newSimHash = simHashCache.get(manifestItem.hash);
-            if (!newSimHash) {
-                const simDoc = await db.collection('system_simhash_registry').doc(manifestItem.hash).get();
-                if (simDoc.exists) {
-                    newSimHash = simDoc.data().simHash;
-                    simHashCache.set(manifestItem.hash, newSimHash);
-                }
-            }
-
-            if (newSimHash && newSimHash === currentStatus.simHash) {
-                resolvedTasks.push({
-                    name: task.name,
-                    hash: manifestItem.hash,
-                    simHash: newSimHash,
-                    prevStatus: currentStatus
-                });
-                continue; 
-            }
-        }
-        remainingTasks.push(task);
-    }
-
-    if (resolvedTasks.length > 0) {
-        const updatePayload = {};
-        resolvedTasks.forEach(t => {
-            updatePayload[t.name] = {
-                ...(t.prevStatus || {}),
-                hash: t.hash,
-                simHash: t.simHash,
-                reason: 'SimHash Stable (Auto-Resolved)',
-                lastUpdated: new Date().toISOString()
-            };
-        });
-        await db.collection('computation_status').doc(date).set(updatePayload, { merge: true });
-        logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date}.`);
-    }
-
-    return remainingTasks;
-}
-
-async function getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild) {
-    const { db, logger } = dependencies;
-    const sessionId = `pass_${passToRun}_${dateLimitStr.replace(/-/g, '')}`;
-    const sessionRef = db.collection('dispatcher_sessions').doc(sessionId);
-
-    if (!forceRebuild) {
-        const sessionSnap = await sessionRef.get();
-        if (sessionSnap.exists) {
-            const data = sessionSnap.data();
-            if ((Date.now() - new Date(data.createdAt).getTime()) < SESSION_CACHE_DURATION_MS) {
-                return data.dates;
-            }
-        }
-    }
-
-    logger.log('INFO', `[Session] 🔄 Rebuilding dispatch session for Pass ${passToRun}...`);
-    const earliestDates = await getEarliestDataDates(config, dependencies);
-    const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
-    
-    await sessionRef.set({ dates: allDates, createdAt: new Date().toISOString(), configHash: dateLimitStr });
-    return allDates;
-}
-
-// MAIN ENTRY POINT
-// =============================================================================
-async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
-    const action = reqBody.action || 'DISPATCH'; 
-    
-    if (action === 'VERIFY') {
-        return handlePassVerification(config, dependencies, computationManifest, reqBody);
-    } 
-    else if (action === 'SWEEP') {
-        return handleSweepDispatch(config, dependencies, computationManifest, reqBody);
-    }
-    else if (action === 'REPORT') {
-        return handleFinalSweepReporting(config, dependencies, computationManifest, reqBody);
-    }
-    // [NEW] FORCE RUN HANDLER
-    else if (action === 'FORCE_RUN') {
-        return handleForceRun(config, dependencies, computationManifest, reqBody);
-    }
-    
-    return handleStandardDispatch(config, dependencies, computationManifest, reqBody);
-}
-
-// =============================================================================
-// NEW: Force Run Handler (Bypasses Checks)
-// =============================================================================
-async function handleForceRun(config, dependencies, computationManifest, reqBody) {
-    const { logger } = dependencies;
-    const pubsubUtils = new PubSubUtils(dependencies);
-    const computationName = reqBody.computation; // Required
-    const dateInput = reqBody.date;              // Optional (YYYY-MM-DD)
-    
-    if (!computationName) {
-        throw new Error('Force Run requires "computation" name.');
-    }
-
-    // 1. Verify Computation Exists
-    const manifestItem = computationManifest.find(c => normalizeName(c.name) === normalizeName(computationName));
-    if (!manifestItem) {
-        throw new Error(`Computation '${computationName}' not found in manifest.`);
-    }
-
-    // 2. Determine Target Dates
-    let candidateDates = [];
-    if (dateInput) {
-        // Single Date Mode
-        candidateDates = [dateInput];
-    } else {
-        // All Dates Mode (Backfill)
-        logger.log('INFO', `[ForceRun] No date provided. Calculating date range for ${computationName}...`);
-        const earliestDates = await getEarliestDataDates(config, dependencies);
-        // Calculate from system start until today
-        candidateDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date());
-    }
-    
-    logger.log('INFO', `[ForceRun] Checking ${candidateDates.length} candidate dates for runnability...`);
-
-    // 3. Filter to only runnable dates using analyzeDateExecution
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const passes = groupByPass(computationManifest);
-    const calcsInPass = passes[manifestItem.pass || "1"] || [];
-    const targetComputationNormalized = normalizeName(computationName);
-    
-    // Filter to only the target computation
-    const targetCalcs = calcsInPass.filter(c => normalizeName(c.name) === targetComputationNormalized);
-    
-    if (targetCalcs.length === 0) {
-        throw new Error(`Computation '${computationName}' not found in pass ${manifestItem.pass || "1"}`);
-    }
-    
-    const runnableDates = [];
-    const skippedDates = [];
-    
-    for (const dateStr of candidateDates) {
-        // Check root data availability
-        const rootDataStatus = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
-        
-        // Get computation status for this date
-        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-        
-        // Check previous day status if needed
-        let prevDailyStatus = null;
-        if (targetCalcs.some(c => c.isHistorical)) {
-            const prevDate = new Date(dateStr + 'T00:00:00Z');
-            prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-            prevDailyStatus = await fetchComputationStatus(prevDate.toISOString().slice(0, 10), config, dependencies);
-        }
-        
-        // Analyze if this computation can run on this date
-        const report = analyzeDateExecution(dateStr, targetCalcs, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
-        
-        // Check if the target computation is runnable, needs re-run, or has failed dependencies
-        const isRunnable = report.runnable.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const needsReRun = report.reRuns.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const hasFailedDep = report.failedDependency.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const isImpossible = report.impossible.some(t => normalizeName(t.name) === targetComputationNormalized);
-        const isBlocked = report.blocked.some(t => normalizeName(t.name) === targetComputationNormalized);
-        
-        if (isRunnable || needsReRun || hasFailedDep) {
-            runnableDates.push(dateStr);
-        } else if (isImpossible) {
-            skippedDates.push({ date: dateStr, reason: report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Impossible' });
-        } else if (isBlocked) {
-            skippedDates.push({ date: dateStr, reason: report.blocked.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Blocked' });
-        } else {
-            skippedDates.push({ date: dateStr, reason: 'Not runnable (unknown reason)' });
-        }
-    }
-    
-    logger.log('INFO', `[ForceRun] ✅ Found ${runnableDates.length} runnable dates out of ${candidateDates.length} candidates`);
-    if (skippedDates.length > 0) {
-        logger.log('INFO', `[ForceRun] ⏭️ Skipped ${skippedDates.length} dates: ${skippedDates.slice(0, 5).map(s => `${s.date} (${s.reason})`).join(', ')}${skippedDates.length > 5 ? '...' : ''}`);
-    }
-    
-    if (runnableDates.length === 0) {
-        return {
-            status: 'NO_RUNNABLE_DATES',
-            computation: computationName,
-            mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
-            datesChecked: candidateDates.length,
-            datesRunnable: 0,
-            skippedReasons: skippedDates.slice(0, 10)
-        };
-    }
-    
-    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${runnableDates.length} runnable dates. Pass: ${manifestItem.pass}`);
-
-    // 4. Construct Tasks (only for runnable dates)
-    const dispatchId = crypto.randomUUID();
-    const tasks = runnableDates.map(date => {
-        const traceId = crypto.randomBytes(16).toString('hex');
-        const spanId = crypto.randomBytes(8).toString('hex');
-        return {
-            action: 'RUN_COMPUTATION_DATE',
-            computation: manifestItem.name,
-            date: date,
-            pass: manifestItem.pass || "1",
-            dispatchId: dispatchId,
-            triggerReason: 'MANUAL_FORCE_API',
-            resources: reqBody.resources || 'standard',
-            // Trace context allows you to find these specific runs in Cloud Trace
-            traceContext: { traceId, spanId, sampled: true }
-        };
-    });
-
-    // 4. Batch Publish (Chunked to stay under Pub/Sub limits)
-    const CHUNK_SIZE = 250; // Safe batch size
-    const topic = (reqBody.resources === 'high-mem') 
-        ? (config.computationTopicHighMem || 'computation-tasks-highmem')
-        : (config.computationTopicStandard || 'computation-tasks');
-        
-    let dispatchedCount = 0;
-    const chunks = [];
-    for (let i = 0; i < tasks.length; i += CHUNK_SIZE) {
-        chunks.push(tasks.slice(i, i + CHUNK_SIZE));
-    }
-
-    // Publish chunks sequentially to avoid memory spikes
-    for (const chunk of chunks) {
-        await pubsubUtils.batchPublishTasks(dependencies, {
-            topicName: topic,
-            tasks: chunk,
-            taskType: 'manual-force-run'
-        });
-        dispatchedCount += chunk.length;
-    }
-
-    return { 
-        status: 'FORCED',
-        computation: computationName,
-        mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
-        datesChecked: candidateDates.length,
-        datesRunnable: runnableDates.length,
-        datesTriggered: dispatchedCount,
-        skippedCount: skippedDates.length,
-        targetTopic: topic
-    };
-}
-
-// =============================================================================
-// NEW: Final Sweep Reporting Handler
-// =============================================================================
-async function handleFinalSweepReporting(config, dependencies, computationManifest, reqBody) {
-    const { logger } = dependencies;
-    const passToRun = String(reqBody.pass || "1");
-    // Target date is required for detailed forensics
-    const date = reqBody.date || new Date().toISOString().slice(0, 10); 
-    
-    logger.log('INFO', `[Dispatcher] 📝 Triggering Final Sweep Forensics for Pass ${passToRun} on ${date}...`);
-
-    try {
-        const result = await runFinalSweepCheck(config, dependencies, date, passToRun, computationManifest);
-        return { 
-            status: 'COMPLETED',
-            date: date,
-            pass: passToRun,
-            issuesFound: result.issuesCount 
-        };
-    } catch (e) {
-        logger.log('ERROR', `[Dispatcher] Forensics failed: ${e.message}`);
-        return { status: 'ERROR', error: e.message };
-    }
-}
-
-// =============================================================================
-// LOGIC: Verify Pass Completion
-// =============================================================================
-async function handlePassVerification(config, dependencies, computationManifest, reqBody) {
-    const { logger } = dependencies;
-    const passToRun = String(reqBody.pass || "1");
-    const dateLimitStr = reqBody.date || "2025-01-01";
-    
-    logger.log('INFO', `[Verify] 🧹 Sweeping Pass ${passToRun} for unfinished work...`);
-    
-    const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, false);
-    const passes = groupByPass(computationManifest);
-    const calcsInPass = passes[passToRun] || [];
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const weightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
-
-    const missingTasks = [];
-
-    for (const date of sessionDates) {
-        // [SCHEDULE CHECK] Filter tasks that are not scheduled for this date
-        const scheduledComputations = calcsInPass.filter(c => 
-            isComputationScheduled(date, c.schedule)
-        );
-
-        if (scheduledComputations.length === 0) continue;
-
-        const [dailyStatus, availability] = await Promise.all([
-            fetchComputationStatus(date, config, dependencies),
-            checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
-        ]);
-
-        let prevDailyStatus = null;
-        if (scheduledComputations.some(c => c.isHistorical)) {
-             const prevD = new Date(date + 'T00:00:00Z');
-             prevD.setUTCDate(prevD.getUTCDate() - 1);
-             prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
-        }
-
-        const report = analyzeDateExecution(
-            date, 
-            scheduledComputations, // Use filtered list 
-            availability ? availability.status : {}, 
-            dailyStatus, 
-            manifestMap, 
-            prevDailyStatus
-        );
-        
-        const pending = [...report.runnable, ...report.reRuns];
-        
-        if (pending.length > 0) {
-            const totalWeight = pending.reduce((sum, t) => sum + (weightMap.get(normalizeName(t.name)) || 1.0), 0);
-            const eta = Math.max(30, Math.ceil(totalWeight * BASE_SECONDS_PER_WEIGHT_UNIT));
-            
-            missingTasks.push({
-                date: date,
-                taskCount: pending.length,
-                eta: eta
-            });
-        }
-    }
-
-    logger.log('INFO', `[Verify] Found ${missingTasks.length} dates with pending work.`);
-    return { missingTasks };
-}
-
-// =============================================================================
-// LOGIC: Sweep Dispatch (Forced High-Mem)
-// =============================================================================
-async function handleSweepDispatch(config, dependencies, computationManifest, reqBody) {
-    const { logger, db } = dependencies;
-    const pubsubUtils = new PubSubUtils(dependencies);
-    const passToRun = String(reqBody.pass || "1");
-    const date = reqBody.date;
-
-    if (!date) throw new Error('Sweep dispatch requires date');
-
-    const passes = groupByPass(computationManifest);
-    const calcsInPass = passes[passToRun] || [];
-    
-    // [SCHEDULE CHECK] Filter tasks that are not scheduled for this date
-    const scheduledComputations = calcsInPass.filter(c => 
-        isComputationScheduled(date, c.schedule)
-    );
-
-    if (scheduledComputations.length === 0) {
-        logger.log('INFO', `[Sweep] ${date} has no scheduled tasks for Pass ${passToRun}. Ignoring.`);
-        return { dispatched: 0 };
-    }
-
-    // 1. Analyze specific date
-    const [dailyStatus, availability] = await Promise.all([
-        fetchComputationStatus(date, config, dependencies),
-        checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
-    ]);
-    
-    let prevDailyStatus = null;
-    if (scheduledComputations.some(c => c.isHistorical)) {
-         const prevD = new Date(date + 'T00:00:00Z');
-         prevD.setUTCDate(prevD.getUTCDate() - 1);
-         prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
-    }
-
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const report = analyzeDateExecution(
-        date, 
-        scheduledComputations, // Use filtered list
-        availability ? availability.status : {}, 
-        dailyStatus, 
-        manifestMap, 
-        prevDailyStatus
-    );
-    const pending = [...report.runnable, ...report.reRuns];
-
-    if (pending.length === 0) {
-        logger.log('INFO', `[Sweep] ${date} is clean. No dispatch.`);
-        return { dispatched: 0 };
-    }
-
-    const validTasks = [];
-    for (const task of pending) {
-        const name = normalizeName(task.name);
-        const ledgerPath = `computation_audit_ledger/${date}/passes/${passToRun}/tasks/${name}`;
-        const doc = await db.doc(ledgerPath).get();
-        
-        if (doc.exists) {
-            const data = doc.data();
-            
-           // 1. ACTIVE CHECK: Don't double-dispatch if already running... UNLESS IT'S A ZOMBIE
-            if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-                const lastActivity = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
-
-                // If it's been silent for > 15 mins, it's a Zombie. Kill it and Re-run.
-                if ((Date.now() - lastActivity) > STALE_LOCK_THRESHOLD_MS) {
-                    logger.log('WARN', `[Sweep] 🧟 Found ZOMBIE lock for ${name}. Breaking lock and re-running.`);
-                    // Don't continue; let it fall through to dispatch
-                } else {
-                    logger.log('INFO', `[Sweep] ⏳ Skipping ${name} - Valid IN_PROGRESS.`);
-                    continue;
-                }
-            }
-
-            // 2. COMPLETION CHECK (GHOST STATE FIX)
-            // We REMOVED the check that skips if (status === 'COMPLETED' && hash === task.hash).
-            // If we are here, 'analyzeDateExecution' (The Brain) decided this task is NOT done
-            // (likely due to a missing or outdated entry in computation_status).
-            // Even if the Ledger (The Log) says it finished, the system state is inconsistent.
-            // We MUST re-run to repair the Status Index.
-            
-            const stage = data.error?.stage;
-            
-            // 3. DETERMINISTIC FAILURE CHECK
-            if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
-                 // If hash matches, it's the exact same code that failed before. Don't retry in loop.
-                 if (data.hash === task.hash) {
-                     logger.log('WARN', `[Sweep] 🛑 Skipping deterministic failure for ${name} (${stage}).`);
-                     continue;
-                 }
-                 logger.log('INFO', `[Sweep] 🔄 Code Updated for ${name}. Retrying sweep despite previous ${stage}.`);
-             }
-             
-             // 4. DEAD END CHECK (High Mem)
-             if (data.resourceTier === 'high-mem' && data.status === 'FAILED') {
-                 // If code hasn't changed, don't hammer it.
-                 if (data.hash === task.hash) {
-                     logger.log('WARN', `[Sweep] 🛑 Skipping ${name} - Already failed on High-Mem.`);
-                     continue;
-                 }
-             }
-        }
-        validTasks.push(task);
-    }
-
-    if (validTasks.length === 0) {
-        logger.log('INFO', `[Sweep] ${date} has no retryable tasks. Ignoring.`);
-        return { dispatched: 0 };
-    }
-
-    // 2. FORCE High Mem & INJECT TRACE
-    const currentDispatchId = crypto.randomUUID();
-    
-    const tasksPayload = validTasks.map(t => {
-        const traceId = crypto.randomBytes(16).toString('hex');
-        const spanId = crypto.randomBytes(8).toString('hex');
-
-        return {
-            ...t,
-            action: 'RUN_COMPUTATION_DATE',
-            computation: t.name,
-            date: date,
-            pass: passToRun,
-            dispatchId: currentDispatchId,
-            triggerReason: 'SWEEP_RECOVERY',
-            resources: 'high-mem', // FORCE
-            traceContext: { traceId, spanId, sampled: true }
-        };
-    });
-
-    const taskNames = tasksPayload.map(t => t.computation || t.name).join(', ');
-    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`, {
-        date: date,
-        pass: passToRun,
-        tasks: tasksPayload.map(t => ({ name: t.computation || t.name, reason: 'sweep' })),
-        topic: config.computationTopicHighMem || 'computation-tasks-highmem'
-    });
-
-    await pubsubUtils.batchPublishTasks(dependencies, {
-        topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
-        tasks: tasksPayload,
-        taskType: `pass-${passToRun}-sweep`
-    });
-
-    return { dispatched: tasksPayload.length };
-}
-
-// =============================================================================
-// LOGIC: Standard Dispatch (Fast-Forward Enabled)
-// =============================================================================
-async function handleStandardDispatch(config, dependencies, computationManifest, reqBody) {
-    const { logger, db } = dependencies;
-    const pubsubUtils = new PubSubUtils(dependencies);
-    
-    const passToRun = String(reqBody.pass || "1");
-    const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr = reqBody.date || "2025-01-01";
-    const forceRebuild = reqBody.forceRebuild === true;
-
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const passes = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || [];
-    const manifestWeightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
-
-    if (!calcsInThisPass.length) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-
-    const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
-    if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-
-    // --- Fast-Forward Loop Configuration ---
-    // Scans up to 50 dates or 40 seconds to find work, avoiding empty "wait loops"
-    const MAX_SCAN_DEPTH = 50; // This is actually 50 + Today, so 51.
-    const TIME_LIMIT_MS = 40000; 
-    const startTime = Date.now();
-
-    let currentCursor = targetCursorN;
-    let selectedTasks = [];
-    let selectedDate = null;
-    let datesScanned = 0;
-
-    // Loop until work is found, end is reached, or safety limits hit
-    while (currentCursor <= sessionDates.length) {
-        datesScanned++;
-        selectedDate = sessionDates[currentCursor - 1]; // 0-indexed array
-
-        // 1. Safety Break (Prevent Timeout)
-        if ((Date.now() - startTime) > TIME_LIMIT_MS || datesScanned > MAX_SCAN_DEPTH) {
-            logger.log('INFO', `[Dispatcher] ⏩ Fast-forward paused at ${selectedDate} after scanning ${datesScanned} dates.`);
-            break;
-        }
-
-        // 2. [SCHEDULE CHECK] Filter computations scheduled for this specific date
-        const scheduledComputations = calcsInThisPass.filter(c => 
-            isComputationScheduled(selectedDate, c.schedule)
-        );
-
-        // Optimization: If nothing is scheduled for today, skip expensive DB checks
-        if (scheduledComputations.length === 0) {
-            // DEBUG: Log when schedule filtering removes all tasks
-            if (calcsInThisPass.length > 0) {
-                logger.log('TRACE', `[Dispatcher] Date ${selectedDate}: ${calcsInThisPass.length} pass computations, but 0 scheduled for this date. Skipping.`);
-            }
-            currentCursor++;
-            continue;
-        }
-
-        // 3. Analyze Date
-        const earliestDates = await getEarliestDataDates(config, dependencies);
-        let prevDailyStatusPromise = Promise.resolve(null);
-        if (scheduledComputations.some(c => c.isHistorical)) {
-            const prevD = new Date(selectedDate + 'T00:00:00Z');
-            prevD.setUTCDate(prevD.getUTCDate() - 1);
-            if (prevD >= earliestDates.absoluteEarliest) {
-                prevDailyStatusPromise = fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
-            }
-        }
-
-        const [dailyStatus, prevDailyStatus, availability] = await Promise.all([
-            fetchComputationStatus(selectedDate, config, dependencies),
-            prevDailyStatusPromise,
-            checkRootDataAvailability(selectedDate, config, dependencies, DEFINITIVE_EARLIEST_DATES)
-        ]);
-
-        // DEBUG: Log availability check
-        if (!availability || !availability.status) {
-            logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: Availability check failed or returned null. Skipping analysis.`);
-        }
-
-        if (availability && availability.status) {
-            const report = analyzeDateExecution(
-                selectedDate, 
-                scheduledComputations, // Use filtered list
-                availability.status, 
-                dailyStatus, 
-                manifestMap, 
-                prevDailyStatus
-            );
-            let rawTasks = [...report.runnable, ...report.reRuns];
-            
-            // DEBUG: Log analysis results
-            if (rawTasks.length === 0 && (report.runnable.length > 0 || report.reRuns.length > 0)) {
-                logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: analyzeDateExecution found ${report.runnable.length} runnable + ${report.reRuns.length} reRuns, but rawTasks is empty!`);
-            }
-            if (rawTasks.length > 0) {
-                logger.log('TRACE', `[Dispatcher] Date ${selectedDate}: analyzeDateExecution found ${report.runnable.length} runnable, ${report.reRuns.length} reRuns. Total: ${rawTasks.length}`);
-            }
-
-            if (rawTasks.length > 0) {
-                rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-                const activeTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
-                
-                if (activeTasks.length > 0) {
-                    // DEBUG: Log what we're about to route
-                    logger.log('INFO', `[Dispatcher] 🔍 Date ${selectedDate}: ${rawTasks.length} raw tasks → ${activeTasks.length} after filtering. Routing...`);
-                    const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, activeTasks, logger);
-                    selectedTasks = [...standard, ...highMem];
-                    
-                    // DEBUG: Log routing results
-                    if (selectedTasks.length === 0 && activeTasks.length > 0) {
-                        logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: ${activeTasks.length} tasks filtered out by splitRoutes! Tasks: ${activeTasks.map(t => t.name).join(', ')}`);
-                    }
-                    
-                    if (selectedTasks.length > 0) {
-                        // Found work! Break loop to dispatch.
-                        break; 
-                    }
-                } else if (rawTasks.length > 0) {
-                    // DEBUG: Log if filterActiveTasks removed all tasks
-                    logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: ${rawTasks.length} raw tasks all filtered out by filterActiveTasks! Tasks: ${rawTasks.map(t => t.name).join(', ')}`);
-                }
-            }
-        }
-
-        // No work found for this date. Fast-forward to next.
-        currentCursor++;
-    }
-
-    // --- Result Handling ---
-
-    // Case 1: Satiated (Scanned to end of session with no work)
-    if (currentCursor > sessionDates.length && selectedTasks.length === 0) {
-        return {
-            status: 'CONTINUE_PASS',
-            dateProcessed: selectedDate,
-            dispatched: 0,
-            n_cursor_ignored: false,
-            remainingDates: 0,
-            nextCursor: currentCursor // Matches length + 1
-        };
-    }
-
-    // Case 2: Paused by Limit (No work found yet, but more dates remain)
-    if (selectedTasks.length === 0) {
-        return {
-            status: 'CONTINUE_PASS',
-            dateProcessed: selectedDate,
-            dispatched: 0,
-            n_cursor_ignored: false,
-            remainingDates: sessionDates.length - currentCursor + 1,
-            nextCursor: currentCursor // Resume from here
-        };
-    }
-
-    // Case 3: Work Found (Dispatching)
-    const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
-    const currentDispatchId = crypto.randomUUID();
-    const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
-
-    if (datesScanned > 1) {
-        logger.log('INFO', `[Dispatcher] ⏩ Fast-forwarded ${datesScanned - 1} empty dates. Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
-    } else {
-        logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
-    }
-
-    const mapToTaskPayload = (t) => {
-        const traceId = crypto.randomBytes(16).toString('hex');
-        const spanId = crypto.randomBytes(8).toString('hex');
-        return {
-            ...t, 
-            action: 'RUN_COMPUTATION_DATE', 
-            computation: t.name, 
-            date: selectedDate, 
-            pass: passToRun,
-            dispatchId: currentDispatchId, 
-            triggerReason: t.reason, 
-            resources: t.resources || 'standard',
-            traceContext: {
-                traceId: traceId,
-                spanId: spanId,
-                sampled: true
-            }
-        };
-    };
-
-    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(mapToTaskPayload);
-    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(mapToTaskPayload);
-
-    const pubPromises = [];
-    if (standardTasks.length > 0) {
-        const taskNames = standardTasks.map(t => t.computation || t.name).join(', ');
-        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${standardTasks.length} standard tasks: ${taskNames}`, {
-            date: selectedDate,
-            pass: passToRun,
-            tasks: standardTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'new' })),
-            topic: config.computationTopicStandard || 'computation-tasks'
-        });
-        pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
-            topicName: config.computationTopicStandard || 'computation-tasks',
-            tasks: standardTasks,
-            taskType: `pass-${passToRun}-std`
-        }));
-    }
-    if (highMemTasks.length > 0) {
-        const taskNames = highMemTasks.map(t => t.computation || t.name).join(', ');
-        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${highMemTasks.length} high-memory tasks: ${taskNames}`, {
-            date: selectedDate,
-            pass: passToRun,
-            tasks: highMemTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'retry' })),
-            topic: config.computationTopicHighMem || 'computation-tasks-highmem'
-        });
-        pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
-            topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
-            tasks: highMemTasks,
-            taskType: `pass-${passToRun}-high`
-        }));
-    }
-    await Promise.all(pubPromises);
-
-    return {
-        status: 'CONTINUE_PASS',
-        dateProcessed: selectedDate,
-        dispatched: selectedTasks.length,
-        n_cursor_ignored: false,
-        etaSeconds: etaSeconds,
-        remainingDates: sessionDates.length - targetCursorN,
-        nextCursor: currentCursor + 1 // Start next scan AFTER this date
-    };
-}
-
-// =============================================================================
-// HELPER: Route Splitting (One-Shot Enforcement)
-// =============================================================================
-// [UPDATED] Route Splitting with Version-Aware Dead Lettering
-async function splitRoutes(db, date, pass, tasks, logger) {
-    const standard = [];
-    const highMem = [];
-
-    for (const task of tasks) {
-        const name = normalizeName(task.name);
-        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
-        const doc = await db.doc(ledgerPath).get();
-        
-        if (!doc.exists) {
-            // No ledger entry - trust analyzeDateExecution, dispatch as standard
-            standard.push(task);
-            continue;
-        }
-
-        const data = doc.data();
-        
-        // CRITICAL FIX: If analyzeDateExecution says this task should run, we MUST trust it.
-        // The ledger might say COMPLETED, but if computation_status is missing/outdated,
-        // we need to re-run to repair the state. Only skip if actively running.
-        // Note: filterActiveTasks already filtered out non-stale PENDING/IN_PROGRESS,
-        // but we double-check here in case of race conditions.
-        if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-            // Check if it's stale (should have been caught by filterActiveTasks, but double-check)
-            const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
-            
-            if ((Date.now() - lastActivityTime) > STALE_LOCK_THRESHOLD_MS) {
-                // Stale lock - break it and continue
-                logger.log('WARN', `[Dispatcher] 🧟 splitRoutes: Breaking stale lock for ${name}.`);
-                // Fall through to handle as if no active lock
-            } else {
-                // Valid active lock - skip (shouldn't happen if filterActiveTasks worked correctly)
-                logger.log('TRACE', `[Dispatcher] splitRoutes: Skipping ${name} - Valid IN_PROGRESS (should have been filtered earlier).`);
-                continue;
-            }
-        }
-        
-        if (data.status === 'FAILED') {
-            const stage = data.error?.stage;
-
-            // 1. DETERMINISTIC FAILURES (Never Retry UNLESS Code Updated)
-            if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
-                 // [FIX] Check if the hash matches. If code changed, we MUST retry.
-                 if (data.hash === task.hash) {
-                     logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Deterministic Failure (${stage}).`);
-                     continue; 
-                 }
-                 
-                 // If hashes differ, we reset to Standard execution to give the new code a chance
-                 logger.log('INFO', `[Dispatcher] 🔄 Code Updated for ${name}. Retrying despite previous ${stage}.`);
-                 standard.push({ ...task, reason: 'Retry: Code Version Changed' });
-                 continue;
-            }
-
-            // 2. HIGH MEMORY FAILURE HANDLING (The New Logic)
-            if (data.resourceTier === 'high-mem') {
-                const failedHash = data.hash || data.composition?.code; // Support legacy or new structure
-                const currentHash = task.hash;
-
-                // A. EXACT CODE MATCH: It failed High-Mem with THIS code.
-                if (failedHash === currentHash) {
-                    logger.log('WARN', `[Dispatcher] 💀 Dead End: ${name} failed High-Mem on this version (${currentHash.slice(0,6)}). Waiting for code fix.`);
-                    continue; // STOP. Do not retry.
-                } 
-                
-                // B. CODE MISMATCH: The code has changed since the High-Mem failure.
-                // We reset it to 'standard' to see if the fix optimized memory usage.
-                else {
-                    logger.log('INFO', `[Dispatcher] 🔄 Code Updated for ${name}. Resetting High-Mem failure to Standard retry.`);
-                    standard.push({
-                        ...task,
-                        reason: 'Retry: Code Version Changed'
-                    });
-                    continue;
-                }
-            }
-
-            // 3. STANDARD FAILURE -> PROMOTE TO HIGH MEM
-            highMem.push({ 
-                ...task, 
-                resources: 'high-mem', 
-                reason: `Retry: ${data.error?.message || 'Standard Failure'}` 
-            });
-
-        } else {
-            // Status is likely COMPLETED or some other state.
-            // CRITICAL: If analyzeDateExecution says this should run, we MUST trust it.
-            // The ledger might show COMPLETED, but if computation_status is missing/outdated,
-            // we need to re-run to repair the state. This is the "ghost state fix" logic.
-            // Trust the Brain (analyzeDateExecution) over the Log (ledger).
-            logger.log('INFO', `[Dispatcher] 🔄 splitRoutes: ${name} has ledger status '${data.status}', but analyzeDateExecution says it should run. Trusting analysis and dispatching.`);
-            standard.push(task);
-        }
-    }
-
-    return { standard, highMem };
-}
-
-module.exports = { dispatchComputationPass };