bulltrackers-module 1.0.705 → 1.0.707

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,7 +1,8 @@
 /**
  * @fileoverview Executor for "Standard" (User-Level) calculations.
  * REFACTORED: Hoisted data loading, centralized Series/Root logic.
- * UPDATED: Calls getResult() on computations to trigger summary logging and finalization logic.
+ * UPDATED: Calls getResult() on computations to trigger summary logging.
+ * FIXED: Force-increments shard indexes after every flush to prevent batch overwrites.
  */
 const { normalizeName, getEarliestDataDates } = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
@@ -81,8 +82,6 @@ class StandardExecutor {
         const loader = new CachedDataLoader(config, deps);
         
         // --- 1. PRE-LOAD GLOBAL DATA (Hoisted) ---
-        // Load all "Singleton" datasets once (Ratings, Rankings, Series, Mappings)
-        // instead of checking/loading per-user inside the loop.
         const startSetup = performance.now();
         
         const [
@@ -269,8 +268,10 @@ class StandardExecutor {
     // =========================================================================
     static async flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, mode, skipStatus, isInitial) {
         const transformedState = {};
+        const activeCalcs = [];
+
         for (const [name, inst] of Object.entries(state)) {
-            // [FIX] Prefer getResult() to allow computation to perform finalization/logging
+            // [FIX 1] Allow computations to finalize/log summary stats via getResult()
             let data;
             if (typeof inst.getResult === 'function') {
                 data = await inst.getResult(); 
@@ -278,6 +279,11 @@ class StandardExecutor {
                 data = inst.results || {};
             }
 
+            // Track active calculations to increment their shards later
+            if (Object.keys(data).length > 0) {
+                activeCalcs.push(name);
+            }
+
             // Pivot user-date structure if needed
             const first = Object.keys(data)[0];
             if (first && data[first] && typeof data[first] === 'object' && /^\d{4}-\d{2}-\d{2}$/.test(Object.keys(data[first])[0])) {
@@ -293,8 +299,19 @@ class StandardExecutor {
             transformedState[name] = { manifest: inst.manifest, getResult: async () => data, _executionStats: stats[name] };
             inst.results = {}; // Clear buffer
         }
+
         const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
+        
+        // Update shardMap from result
         if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
+        
+        // [FIX 2] Force increment shard indexes for active calculations.
+        // This ensures the NEXT batch writes to a NEW shard (e.g. results_1) 
+        // instead of overwriting the current one (results_0).
+        activeCalcs.forEach(name => {
+            shardMap[name] = (shardMap[name] || 0) + 1;
+        });
+
         return res;
     }
 

package/functions/computation-system/persistence/ResultCommitter.js

@@ -3,6 +3,7 @@
  * UPDATED: Added GCS Offloading logic (Hybrid Pointer System).
  * UPDATED: Preserved Legacy Sharding/Compression for backward compatibility.
  * UPDATED: Auto-cleanup of old Firestore shards when migrating a doc to GCS.
+ * FIXED: Disabled "Single Doc Compression" strategy during intermediate flushes to ensure consistent sharding.
  */
 const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -366,7 +367,10 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
     }
 
     // --- STRATEGY 2: FIRESTORE COMPRESSION ---
-    if (totalSize > 50 * 1024) { 
+    // FIX: Only compress if this is a single, atomic write (not part of a stream).
+    // Streaming relies on Strategy 3 (Sharding) to create distinct files (shard_0, shard_1...).
+    // If flushMode is INTERMEDIATE or we are already at a high shard index, we MUST fall through to sharding.
+    if (totalSize > 50 * 1024 && startShardIndex === 0 && flushMode !== 'INTERMEDIATE') { 
         try {
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             if (compressedBuffer.length < 900 * 1024) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.705",
+  "version": "1.0.707",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [