bulltrackers-module 1.0.611 → 1.0.612

package/functions/computation-system/persistence/ResultCommitter.js

@@ -15,6 +15,7 @@ const ContractValidator                    = require('./ContractValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
 const zlib                                 = require('zlib'); 
+const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils');
 
 const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
@@ -137,6 +138,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
+            // [NEW] Page Computation Logic (Fan-Out) with TTL
+            // Bypasses standard compression/sharding to write per-user documents
             // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
@@ -145,26 +148,27 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     .collection(config.computationsSubcollection).doc(name);
 
                 // --- CLEANUP START: Remove old storage formats (Sharded/Compressed) ---
-                // If the computation previously ran in Standard mode, we must remove shards
-                // and clear the main document data to avoid conflicting flags.
-                try {
-                    const docSnap = await mainDocRef.get();
-                    if (docSnap.exists) {
-                        const dData = docSnap.data();
-                        if (dData._sharded) {
-                            const shardCol = mainDocRef.collection('_shards');
-                            const shardDocs = await withRetry(() => shardCol.listDocuments());
-                            
-                            if (shardDocs.length > 0) {
-                                const cleanupOps = shardDocs.map(d => ({ type: 'DELETE', ref: d }));
-                                await commitBatchInChunks(config, deps, cleanupOps, `${name}::PageModeCleanup`);
-                                runMetrics.io.deletes += cleanupOps.length;
-                                logger.log('INFO', `[PageMode] ${name}: Cleaned up ${cleanupOps.length} old shard documents.`);
+                // Optimization: Only attempt cleanup on the initial write to save reads
+                if (isInitialWrite) {
+                    try {
+                        const docSnap = await mainDocRef.get();
+                        if (docSnap.exists) {
+                            const dData = docSnap.data();
+                            if (dData._sharded) {
+                                const shardCol = mainDocRef.collection('_shards');
+                                const shardDocs = await withRetry(() => shardCol.listDocuments());
+                                
+                                if (shardDocs.length > 0) {
+                                    const cleanupOps = shardDocs.map(d => ({ type: 'DELETE', ref: d }));
+                                    await commitBatchInChunks(config, deps, cleanupOps, `${name}::PageModeCleanup`);
+                                    runMetrics.io.deletes += cleanupOps.length;
+                                    logger.log('INFO', `[PageMode] ${name}: Cleaned up ${cleanupOps.length} old shard documents.`);
+                                }
                             }
                         }
+                    } catch (cleanupErr) {
+                        logger.log('WARN', `[PageMode] ${name}: Cleanup warning: ${cleanupErr.message}`);
                     }
-                } catch (cleanupErr) {
-                    logger.log('WARN', `[PageMode] ${name}: Cleanup warning: ${cleanupErr.message}`);
                 }
                 // --- CLEANUP END ---
 
@@ -198,31 +202,47 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     logger.log('INFO', `[PageMode] ${name}: Wrote ${pageWrites.length} user pages. TTL: ${ttlDays}d.`);
                 }
 
-                // 3. Write the "Header" document (Important for Status/Metrics/TTL)
-                if (flushMode !== 'INTERMEDIATE') {
-                    const headerData = {
-                        _completed: true,
-                        _isPageMode: true, // Flag for readers to know where to look
-                        _pageCount: pageWrites.length,
-                        _lastUpdated: new Date().toISOString(),
-                        _expireAt: expireAt // Ensure the header also gets deleted
-                    };
-                    
-                    // CHANGED: Use merge: false to FORCE overwrite. 
-                    // This clears old "payload" (compressed), raw data keys, and old flags like "_sharded".
-                    await mainDocRef.set(headerData, { merge: false });
-                    runMetrics.io.writes += 1;
-                    
-                    if (calc.manifest.hash) { 
-                        successUpdates[name] = { 
-                            hash: calc.manifest.hash, 
-                            simHash: simHash, 
-                            resultHash: resultHash,
-                            category: calc.manifest.category, 
-                            composition: calc.manifest.composition, 
-                            metrics: runMetrics 
-                        }; 
-                    }
+                // 3. Write or Update the "Header" document
+                // FIXED: Now runs on every batch to ensure counts are accumulated correctly.
+                
+                const isFinalFlush = (flushMode !== 'INTERMEDIATE');
+                
+                // Determine Page Count Value: Raw number for initial, Increment for updates
+                let pageCountValue = pageWrites.length;
+                if (!isInitialWrite) {
+                    pageCountValue = FieldValue.increment(pageWrites.length);
+                }
+
+                const headerData = {
+                    _isPageMode: true, // Flag for readers to know where to look
+                    _pageCount: pageCountValue,
+                    _lastUpdated: new Date().toISOString(),
+                    _expireAt: expireAt // Ensure the header also gets deleted
+                };
+
+                // Handle Completion Status
+                if (isFinalFlush) {
+                    headerData._completed = true;
+                } else if (isInitialWrite) {
+                    headerData._completed = false; // Initialize as incomplete
+                }
+
+                // Write Strategy:
+                // isInitialWrite = TRUE  -> merge: false (Wipes old Standard Mode data/schema)
+                // isInitialWrite = FALSE -> merge: true (Updates count and status, preserves data)
+                await mainDocRef.set(headerData, { merge: !isInitialWrite });
+                
+                runMetrics.io.writes += 1;
+                
+                if (isFinalFlush && calc.manifest.hash) { 
+                    successUpdates[name] = { 
+                        hash: calc.manifest.hash, 
+                        simHash: simHash, 
+                        resultHash: resultHash,
+                        category: calc.manifest.category, 
+                        composition: calc.manifest.composition, 
+                        metrics: runMetrics 
+                    }; 
                 }
                 
                 continue; // Skip the standard writeSingleResult logic

package/functions/computation-system/utils/utils.js

@@ -6,6 +6,7 @@
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
+
 // [UPDATED] Registry for Data Availability. 
 // Populated dynamically by getEarliestDataDates().
 const DEFINITIVE_EARLIEST_DATES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.611",
+  "version": "1.0.612",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [