bulltrackers-module 1.0.336 → 1.0.338

package/functions/computation-system/config/validation_overrides.js

@@ -6,4 +6,5 @@ module.exports = {
 
     "instrument-price-change-1d":        { maxZeroPct: 100 }, // Because weekeends/holidays return 0 change, technically crypto means this can't hit 100% but it's usually quite close, so we override
     "instrument-price-momentum-20d ":    { maxZeroPct: 100 }, // Some assets can be very stagnant over a month, especially bonds or stablecoins
+    "mimetic-latency":                   { maxFlatlinePct: 100 },
 };

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -3,6 +3,7 @@
  * PURPOSE: Sequential Cursor-Based Dispatcher.
  * BEHAVIOR: Dispatch -> Wait ETA -> Next Date.
  * UPDATED: Added "Sweep" Protocol for OOM recovery & High-Mem Verification.
+ * UPDATED: Added Safety Checks to permanently skip Deterministic Failures.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -22,7 +23,7 @@ const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15;
 // =============================================================================
 async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false) {
     if (!tasks || tasks.length === 0) return [];
-    if (forceRun) return tasks; // Bypass check for Sweep Mode
+    if (forceRun) return tasks; // Bypass check for Sweep Mode (Handled separately)
     
     const checkPromises = tasks.map(async (t) => {
         const taskName = normalizeName(t.name);
@@ -54,6 +55,16 @@ async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false
                                    (Date.now() - new Date(data.completedAt).getTime() < 60 * 1000);
             
             if (isJustFinished) return null; 
+
+            // 3. DETERMINISTIC FAILURE CHECK (Break Infinite Loops)
+            // If the task failed due to Logic/Quality issues, never retry it automatically.
+            if (data.status === 'FAILED') {
+                 const stage = data.error?.stage;
+                 if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
+                     if (logger) logger.log('WARN', `[Dispatcher] 🛑 Skipping deterministic failure for ${taskName} (${stage}).`);
+                     return null;
+                 }
+            }
         }
         return t;
     });
@@ -173,15 +184,12 @@ async function handlePassVerification(config, dependencies, computationManifest,
 
     const missingTasks = [];
 
-    // Optimize: Batch fetch statuses if possible, but for now loop is safer for memory
-    // In production, we might want p-limit here.
     for (const date of sessionDates) {
         const [dailyStatus, availability] = await Promise.all([
             fetchComputationStatus(date, config, dependencies),
             checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
         ]);
 
-        // Need previous status for historical calcs
         let prevDailyStatus = null;
         if (calcsInPass.some(c => c.isHistorical)) {
              const prevD = new Date(date + 'T00:00:00Z');
@@ -191,8 +199,6 @@ async function handlePassVerification(config, dependencies, computationManifest,
 
         const report = analyzeDateExecution(date, calcsInPass, availability ? availability.status : {}, dailyStatus, manifestMap, prevDailyStatus);
         
-        // We only care about Runnable (New) or ReRuns (Changed/Failed)
-        // We ignore Blocked (impossible to run) and Impossible (permanent fail)
         const pending = [...report.runnable, ...report.reRuns];
         
         if (pending.length > 0) {
@@ -233,7 +239,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
         checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
     ]);
     
-    // Previous Status Fetch (simplified for brevity, assume historical dependency check works or fails safe)
     let prevDailyStatus = null;
     if (calcsInPass.some(c => c.isHistorical)) {
          const prevD = new Date(date + 'T00:00:00Z');
@@ -249,12 +254,33 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
         return { dispatched: 0 };
     }
 
+    // [FIX] Filter out deterministic failures from Sweep.
+    // If it failed due to 'QUALITY_CIRCUIT_BREAKER', High-Mem won't fix it.
+    const validTasks = [];
+    for (const task of pending) {
+        const name = normalizeName(task.name);
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${passToRun}/tasks/${name}`;
+        const doc = await db.doc(ledgerPath).get();
+        if (doc.exists) {
+            const data = doc.data();
+            const stage = data.error?.stage;
+             if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
+                 logger.log('WARN', `[Sweep] 🛑 Skipping deterministic failure for ${name} (${stage}).`);
+                 continue;
+             }
+        }
+        validTasks.push(task);
+    }
+
+    if (validTasks.length === 0) {
+        logger.log('INFO', `[Sweep] ${date} only has deterministic failures. No dispatch.`);
+        return { dispatched: 0 };
+    }
+
     // 2. FORCE High Mem & Skip Zombie Check
-    // We do NOT filterActiveTasks because we assume they might be OOM'd zombies
-    // We do NOT check resource tier from previous runs, we FORCE 'high-mem'
     const currentDispatchId = crypto.randomUUID();
     
-    const tasksPayload = pending.map(t => ({
+    const tasksPayload = validTasks.map(t => ({
         ...t,
         action: 'RUN_COMPUTATION_DATE',
         computation: t.name,

package/functions/computation-system/helpers/computation_worker.js

@@ -1,5 +1,6 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
+ * UPDATED: Fixed Error Propagation Bug. Preserves 'stage' property when re-throwing logic errors.
  * UPDATED: Fixed Firestore 'undefined' field error for dispatchId.
  */
 
@@ -46,7 +47,7 @@ async function handleComputationTask(message, config, dependencies) {
 
     logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}]`);
 
-    // [FIX] Build document object and only add dispatchId if it is defined
+    // [FIX] Build document object and only add dispatchId if it is defined (prevents Firestore "undefined" error)
     const leaseData = {
         status: 'IN_PROGRESS',
         workerId: process.env.K_REVISION || os.hostname(),
@@ -71,7 +72,16 @@ async function handleComputationTask(message, config, dependencies) {
         const failureReport = result?.updates?.failureReport || [];
         const successUpdates = result?.updates?.successUpdates || {};
 
-        if (failureReport.length > 0) throw new Error(failureReport[0].error.message);
+        // [CRITICAL FIX] Correctly propagate the Error Stage.
+        // Previously, 'throw new Error(msg)' stripped the 'stage' property, causing the 
+        // catch block to treat Deterministic errors (Quality/Logic) as System errors (Transient),
+        // triggering infinite Pub/Sub retries.
+        if (failureReport.length > 0) {
+            const reportedError = failureReport[0].error;
+            const errorObj = new Error(reportedError.message);
+            errorObj.stage = reportedError.stage; // Preserve stage (e.g. 'QUALITY_CIRCUIT_BREAKER')
+            throw errorObj;
+        }
 
         const calcUpdate = successUpdates[normalizeName(computation)] || {};
         const metrics = {
@@ -91,11 +101,27 @@ async function handleComputationTask(message, config, dependencies) {
         clearInterval(heartbeat.timer);
         const isDeterministic = ['SHARDING_LIMIT_EXCEEDED', 'QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(err.stage);
 
+        // If error is deterministic (Logic/Quality), we record FAILURE and RETURN.
+        // This ACKs the message and stops the retry loop.
         if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
-            await db.doc(ledgerPath).set({ status: 'FAILED', error: err.message, failedAt: new Date() }, { merge: true });
+            
+            // Write structured error to Ledger so Dispatcher can see the 'stage' later
+            const errorPayload = { 
+                message: err.message, 
+                stage: err.stage || 'FATAL' 
+            };
+
+            await db.doc(ledgerPath).set({ 
+                status: 'FAILED', 
+                error: errorPayload, 
+                failedAt: new Date() 
+            }, { merge: true });
+
             await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeat.getPeak() }, triggerReason, resourceTier);
             return;
         }
+        
+        // If non-deterministic (Network/System), throw to trigger Pub/Sub Retry
         throw err;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.336",
+  "version": "1.0.338",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [