bulltrackers-module 1.0.180 → 1.0.182

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,13 +6,15 @@ const {
     groupByPass, 
     checkRootDataAvailability, 
     fetchExistingResults, 
-    fetchGlobalComputationStatus,  // <--- New Import
-    updateGlobalComputationStatus, // <--- New Import
+    fetchComputationStatus,   
+    updateComputationStatus,  
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies
 } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings } = require('../utils/utils.js');
+
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+
 const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
@@ -21,7 +23,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Global Status Check...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
     // Hardcoded earliest dates
     const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
@@ -40,36 +42,47 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
     const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
 
-    // 1. Fetch Global Status ONCE (Memory Cache)
-    // Returns { "2023-10-27": { calcA: true, calcB: false }, ... }
-    const globalStatusData = await fetchGlobalComputationStatus(config, dependencies);
-
-    // Helper: Check status using in-memory data
-    const shouldRun = (calc, dateStr) => {
-        const dailyStatus = globalStatusData[dateStr] || {};
-        
-        // 1. If explicitly TRUE, ignore.
-        if (dailyStatus[calc.name] === true) return false;
-
-        // 2. Check dependencies (using same in-memory status)
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            const depsMet = calc.dependencies.every(depName => dailyStatus[depName] === true);
-            if (!depsMet) return false; 
-        }
-        return true;
-    };
-
-    // Process a single date and RETURN updates (do not write)
+    // Process a single date
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        const standardToRun = standardCalcs.filter(c => shouldRun(c, dateStr));
-        const metaToRun     = metaCalcs.filter(c => shouldRun(c, dateStr));
+
+        // 1. Fetch Status for THIS specific date only
+        // This ensures Pass 2 sees exactly what Pass 1 wrote for this date.
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+        // Helper: Check status using the fetched daily data
+        const shouldRun = (calc) => {
+            const cName = normalizeName(calc.name);
+            
+            // A. If recorded as TRUE -> Ignore (already ran)
+            if (dailyStatus[cName] === true) return false;
+
+            // B. If recorded as FALSE or UNDEFINED -> Run it (retry or new)
+            // But first, check if we have the necessary data dependencies.
+            
+            if (calc.dependencies && calc.dependencies.length > 0) {
+                // Check if prerequisites (from previous passes on THIS date) are complete
+                const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
+                if (missing.length > 0) {
+                    // Dependency missing: cannot run yet.
+                    return false;
+                }
+            }
+            
+            // If we are here, status is false/undefined AND dependencies are met.
+            return true;
+        };
+
+        const standardToRun = standardCalcs.filter(shouldRun);
+        const metaToRun     = metaCalcs.filter(shouldRun);
         
-        if (!standardToRun.length && !metaToRun.length) return null; // No work
+        if (!standardToRun.length && !metaToRun.length) return null; // No work for this date
 
+        // 2. Check Root Data Availability (Portfolio, History, etc.)
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
         if (!rootData) return null;
 
+        // 3. Filter again based on Root Data availability
         const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
         const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
@@ -85,52 +98,35 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
+            
+            // Note: We use skipStatusWrite=true because we want to batch write the status at the end of this function
             if (finalStandardToRun.length) {
-                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); // skipStatusWrite=true
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
                 Object.assign(dateUpdates, updates);
             }
             if (finalMetaToRun.length) {
-                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); // skipStatusWrite=true
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); 
                 Object.assign(dateUpdates, updates);
             }
         } catch (err) { 
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-            // Mark failures
-            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[c.name] = false);
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        }
+
+        // 4. Write "true" or "false" results for THIS specific date immediately
+        if (Object.keys(dateUpdates).length > 0) {
+            await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
         }
 
-        // Return the updates for this date
         return { date: dateStr, updates: dateUpdates };
     };
 
     // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        
-        // Run batch in parallel
-        const results = await Promise.all(batch.map(processDate));
-        
-        // Aggregate updates from the batch
-        const batchUpdates = {};
-        let hasUpdates = false;
-        
-        results.forEach(res => {
-            if (res && res.updates && Object.keys(res.updates).length > 0) {
-                batchUpdates[res.date] = res.updates;
-                hasUpdates = true;
-                
-                // Also update our local in-memory copy so subsequent logic in this run sees it (though passes usually rely on prev days)
-                if (!globalStatusData[res.date]) globalStatusData[res.date] = {};
-                Object.assign(globalStatusData[res.date], res.updates);
-            }
-        });
-
-        // Write status ONCE per batch
-        if (hasUpdates) {
-            await updateGlobalComputationStatus(batchUpdates, config, dependencies);
-            logger.log('INFO', `[PassRunner] Batched status update for ${Object.keys(batchUpdates).length} dates.`);
-        }
+        await Promise.all(batch.map(processDate));
     }
+    
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -1,251 +1,219 @@
-/** 
-@fileoverview Utility class to manage stateful Firestore write batches.
-
+/** @fileoverview Utility class to manage stateful Firestore write batches.
 REFACTORED: Renamed 'firestore' to 'db' for consistency.
-
 OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
-
 --- MODIFIED: Added username map caching and trading history batching. ---
-
---- MODIFIED: Added cross-invocation cache for history fetches. ---
-
---- FIXED: Implemented deterministic sharding (part_N) to prevent document fragmentation. --- **/ 
+--- FIXED: Implemented Modulo Sharding to pack sparse IDs into dense documents. --- 
+--- FIXED: Removed aggressive auto-flush timeout to prevent cost explosion. ---
+**/ 
 
 const { FieldValue } = require('@google-cloud/firestore');
 
-class FirestoreBatchManager { constructor(db, headerManager, logger, config) { this.db = db; this.headerManager = headerManager; this.logger = logger; this.config = config; this.portfolioBatch = {}; this.timestampBatch = {}; this.tradingHistoryBatch = {}; this.speculatorTimestampFixBatch = {};
-
-    // Username map cache
-    this.usernameMap = new Map();
-    this.usernameMapUpdates = {};
-    this.usernameMapLastLoaded = 0;
-    
-    // History fetch cache (NEW)
-    this.historyFetchedUserIds = new Set();
-    this.historyCacheTimestamp = Date.now();
-    // Set a 10-minute TTL on this cache (600,000 ms)
-    this.HISTORY_CACHE_TTL_MS = config.HISTORY_CACHE_TTL_MS || 600000; 
-
-    this.processedSpeculatorCids = new Set();
-    this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
-    this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
-    this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
-    this.batchTimeout = null;
-    logger.log('INFO', 'FirestoreBatchManager initialized.');
-}
-
-/* * NEW: Checks if a user's history has been fetched in the last 10 minutes.
- * If not, it logs them as fetched and returns false (to trigger a fetch).
- * @param {string} userId 
- * @returns {boolean} True if already fetched, false if not.
- */
-checkAndSetHistoryFetched(userId) {
-    // Check if the cache is stale
-    if (Date.now() - this.historyCacheTimestamp > this.HISTORY_CACHE_TTL_MS) {
-        this.logger.log('INFO', '[BATCH] History fetch cache (10m TTL) expired. Clearing set.');
-        this.historyFetchedUserIds.clear();
+class FirestoreBatchManager { 
+    constructor(db, headerManager, logger, config) { 
+        this.db            = db; 
+        this.headerManager = headerManager; 
+        this.logger        = logger; 
+        this.config        = config; 
+        this.portfolioBatch              = {}; 
+        this.timestampBatch              = {}; 
+        this.tradingHistoryBatch         = {}; 
+        this.speculatorTimestampFixBatch = {};
+
+        // Username map cache
+        this.usernameMap           = new Map();
+        this.usernameMapUpdates    = {};
+        this.usernameMapLastLoaded = 0;
+        
+        // History fetch cache (NEW)
+        this.historyFetchedUserIds = new Set();
         this.historyCacheTimestamp = Date.now();
-    }
+        this.HISTORY_CACHE_TTL_MS  = config.HISTORY_CACHE_TTL_MS || 600000; 
 
-    if (this.historyFetchedUserIds.has(userId)) {
-        return true; // Yes, already fetched
+        this.processedSpeculatorCids         = new Set();
+        this.usernameMapCollectionName       = config.FIRESTORE_COLLECTION_USERNAME_MAP;
+        this.normalHistoryCollectionName     = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
+        this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
+        this.batchTimeout                    = null;
+
+        logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
-    // Not fetched yet. Mark as fetched and return false.
-    this.historyFetchedUserIds.add(userId);
-    return false;
-}
+    checkAndSetHistoryFetched(userId) {
+        if (Date.now() - this.historyCacheTimestamp > this.HISTORY_CACHE_TTL_MS) {
+            this.logger.log('INFO', '[BATCH] History fetch cache (10m TTL) expired. Clearing set.');
+            this.historyFetchedUserIds.clear();
+            this.historyCacheTimestamp = Date.now();
+        }
+        if (this.historyFetchedUserIds.has(userId)) { return true; }
+        this.historyFetchedUserIds.add(userId);
+        return false;
+    }
 
-_getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
+    _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
-// _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); } Old version
+    _scheduleFlush() {
+        const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 400;
+        const totalOps = this._estimateBatchSize();
+        if (totalOps >= maxBatch) {  this.flushBatches();  return; }
+    }
 
-_scheduleFlush() {
-    const totalOps = this._estimateBatchSize();
-    if (totalOps >= 400) { this.flushBatches(); return; }
-    if (!this.batchTimeout) { this.batchTimeout = setTimeout(() => this.flushBatches(),  this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
-}
+    _estimateBatchSize() {
+        let ops = 0;
+        ops += Object.keys(this.portfolioBatch).length;
+        ops += Object.keys(this.tradingHistoryBatch).length;
+        ops += Object.keys(this.timestampBatch).length;
+        ops += Object.keys(this.speculatorTimestampFixBatch).length;
+        return ops;
+    }
 
-_estimateBatchSize() {
-    let ops = 0;
-    ops += Object.keys(this.portfolioBatch).length;
-    ops += Object.keys(this.tradingHistoryBatch).length;
-    ops += Object.keys(this.timestampBatch).length;
-    ops += Object.keys(this.speculatorTimestampFixBatch).length;
-    return ops;
-}
+    async loadUsernameMap() {
+        if (Date.now() - this.usernameMapLastLoaded < 3600000) return;
+        this.usernameMap.clear();
+        this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
+        try {
+            const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
+            snapshot.forEach(doc => { const data = doc.data(); for (const cid in data) if (data[cid]?.username) this.usernameMap.set(String(cid), data[cid].username); });
+            this.usernameMapLastLoaded = Date.now();
+            this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames.`);
+        } catch (e) { this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: e.message }); }
+    }
 
-async loadUsernameMap() {
-    if (Date.now() - this.usernameMapLastLoaded < 3600000) return;
-    this.usernameMap.clear();
-    this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
-    try {
-        const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
-        snapshot.forEach(doc => { const data = doc.data(); for (const cid in data) if (data[cid]?.username) this.usernameMap.set(String(cid), data[cid].username); });
-        this.usernameMapLastLoaded = Date.now();
-        this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames.`);
-    } catch (e) { this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: e.message }); }
-}
+    getUsername(cid) { return this.usernameMap.get(String(cid)); }
 
-getUsername(cid) { return this.usernameMap.get(String(cid)); }
-
-addUsernameMapUpdate(cid, username) {
-    if (!username) return;
-    const cidStr = String(cid);
-    this.usernameMap.set(cidStr, username);
-    const shardId = this._getUsernameShardId(cidStr);
-    if (!this.usernameMapUpdates[shardId]) { this.usernameMapUpdates[shardId] = {}; }
-    this.usernameMapUpdates[shardId][cidStr] = { username };
-    this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr} in ${shardId}.`);
-    this._scheduleFlush();
-}
+    addUsernameMapUpdate(cid, username) {
+        if (!username) return;
+        const cidStr = String(cid);
+        this.usernameMap.set(cidStr, username);
+        const shardId = this._getUsernameShardId(cidStr);
+        if (!this.usernameMapUpdates[shardId]) { this.usernameMapUpdates[shardId] = {}; }
+        this.usernameMapUpdates[shardId][cidStr] = { username };
+        this._scheduleFlush();
+    }
 
-async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
-    const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
-    const path = `${collection}/${blockId}/snapshots/${date}`;
-    this.tradingHistoryBatch[path] ??= {};
-    this.tradingHistoryBatch[path][userId] = historyData;
-    this._scheduleFlush();
-}
+    async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
+        const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
+        const path = `${collection}/${blockId}/snapshots/${date}`;
+        this.tradingHistoryBatch[path] ??= {};
+        this.tradingHistoryBatch[path][userId] = historyData;
+        this._scheduleFlush();
+    }
 
-async addToPortfolioBatch(userId, blockId, date, portfolioData, userType) {
-    const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-    const path = `${collection}/${blockId}/snapshots/${date}`;
-    this.portfolioBatch[path] ??= {};
-    this.portfolioBatch[path][userId] = portfolioData;
-    this._scheduleFlush();
-}
+    async addToPortfolioBatch(userId, blockId, date, portfolioData, userType) {
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const path = `${collection}/${blockId}/snapshots/${date}`;
+        this.portfolioBatch[path] ??= {};
+        this.portfolioBatch[path][userId] = portfolioData;
+        this._scheduleFlush();
+    }
 
-async updateUserTimestamp(userId, userType, instrumentId = null) {
-    const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-    const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
-    this.timestampBatch[docPath] ??= {};
-    const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
-    this.timestampBatch[docPath][key] = new Date();
-    this._scheduleFlush();
-}
+    async updateUserTimestamp(userId, userType, instrumentId = null) {
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
+        this.timestampBatch[docPath] ??= {};
+        const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
+        this.timestampBatch[docPath][key] = new Date();
+        this._scheduleFlush();
+    }
 
-deleteFromTimestampBatch(userId, userType, instrumentId) {
-    const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-    const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
-    if (this.timestampBatch[docPath]) { const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId; delete this.timestampBatch[docPath][key]; }
-}
+    deleteFromTimestampBatch(userId, userType, instrumentId) {
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
+        if (this.timestampBatch[docPath]) { const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId; delete this.timestampBatch[docPath][key]; }
+    }
 
-addProcessedSpeculatorCids(cids) { cids.forEach(cid => this.processedSpeculatorCids.add(cid)); }
+    addProcessedSpeculatorCids(cids) { cids.forEach(cid => this.processedSpeculatorCids.add(cid)); }
 
-async addSpeculatorTimestampFix(userId, orchestratorBlockId) {
-    const docPath = `${this.config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS}/${orchestratorBlockId}`;
-    this.speculatorTimestampFixBatch[docPath] ??= {};
-    this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastVerified`] = new Date();
-    this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastHeldSpeculatorAsset`] = new Date();
-    this.logger.log('TRACE', `[BATCH] Queued speculator timestamp fix for ${userId} in block ${orchestratorBlockId}`);
-    this._scheduleFlush();
-}
+    async addSpeculatorTimestampFix(userId, orchestratorBlockId) {
+        const docPath = `${this.config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS}/${orchestratorBlockId}`;
+        this.speculatorTimestampFixBatch[docPath] ??= {};
+        this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastVerified`] = new Date();
+        this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastHeldSpeculatorAsset`] = new Date();
+        this._scheduleFlush();
+    }
 
-/**
- * --- REFACTORED: Deterministic Sharding ---
- * Groups users into 'part_N' shards based on their CID.
- * Uses { merge: true } to allow concurrent/sequential writes to fill documents
- * instead of creating fragmented random docs.
- */
-_flushDataBatch(batchData, firestoreBatch, logName) {
-    let count = 0;
-    
-    // Use 200 as requested, or fall back to config if higher/defined.
-    // This ensures we fit ~200 users per document to save space/reads.
-    const SHARD_CAPACITY = this.config.TASK_ENGINE_MAX_USERS_PER_SHARD || 200;
-
-    for (const basePath in batchData) {
-        const users = batchData[basePath];
-        const userIds = Object.keys(users);
-        if (!userIds.length) continue;
-
-        // 1. Group updates by Deterministic Shard ID
-        const updatesByShard = {};
-
-        for (const userId of userIds) {
-            const cid = parseInt(userId, 10);
-            let shardId;
-
-            if (!isNaN(cid)) {
-                // Block Logic: 
-                // Users are already partitioned into 1M blocks (basePath has blockId).
-                // Within a block (0-999,999), we want shards of size SHARD_CAPACITY.
-                // shardIndex = floor((cid % 1,000,000) / SHARD_CAPACITY)
-                const blockOffset = cid % 1000000;
-                const shardIndex = Math.floor(blockOffset / SHARD_CAPACITY);
-                shardId = `part_${shardIndex}`;
-            } else {
-                // Fallback for non-numeric IDs (unlikely for CIDs)
-                shardId = 'part_misc';
+    /**
+     * --- REFACTORED: Modulo Sharding ---
+     * Fixes the issue where sparse IDs created fragmented documents.
+     * We now calculate the number of shards needed to hold the target population
+     * and bucket users into them using modulo arithmetic.
+     */
+    _flushDataBatch(batchData, firestoreBatch, logName) {
+        let count = 0;
+        const TARGET_USERS = this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK ? Number(this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK) : 1500;
+        const SHARD_CAPACITY = this.config.TASK_ENGINE_MAX_USERS_PER_SHARD ? Number(this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) : 200;
+        const TOTAL_SHARDS = Math.max(1, Math.ceil(TARGET_USERS / SHARD_CAPACITY));
+        for (const basePath in batchData) {
+            const users = batchData[basePath];
+            const userIds = Object.keys(users);
+            if (!userIds.length) continue;
+            const updatesByShard = {};
+            for (const userId of userIds) {
+                const cid = parseInt(userId, 10);
+                let shardId;
+
+                if (!isNaN(cid)) {
+                    const shardIndex = cid % TOTAL_SHARDS;
+                    shardId = `part_${shardIndex}`;
+                } else { shardId = 'part_misc'; }
+
+                if (!updatesByShard[shardId]) { updatesByShard[shardId] = {}; }
+                updatesByShard[shardId][userId] = users[userId];
             }
-
-            if (!updatesByShard[shardId]) {
-                updatesByShard[shardId] = {};
+            for (const shardId in updatesByShard) {
+                const chunkData = updatesByShard[shardId];
+                const docRef = this.db.collection(`${basePath}/parts`).doc(shardId);
+                firestoreBatch.set(docRef, chunkData, { merge: true });
+                count++;
             }
-            updatesByShard[shardId][userId] = users[userId];
+            this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users into ${Object.keys(updatesByShard).length} buckets (Modulo ${TOTAL_SHARDS}) for ${basePath}.`);
+            delete batchData[basePath];
         }
+        return count;
+    }
 
-        // 2. Queue Writes with Merge
-        for (const shardId in updatesByShard) {
-            const chunkData = updatesByShard[shardId];
-            const docRef = this.db.collection(`${basePath}/parts`).doc(shardId);
-            
-            // CRITICAL: Use merge: true to append to existing shards
-            firestoreBatch.set(docRef, chunkData, { merge: true });
-            count++;
+    async flushBatches() {
+        if (this.batchTimeout) { clearTimeout(this.batchTimeout); this.batchTimeout = null; }
+        const firestoreBatch = this.db.batch();
+        let batchOps = 0;
+        batchOps += this._flushDataBatch(this.portfolioBatch, firestoreBatch, 'Portfolio');
+        batchOps += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, 'Trade History');
+        for (const docPath in this.timestampBatch) {
+            const timestamps = this.timestampBatch[docPath];
+            if (!Object.keys(timestamps).length) continue;
+            const docRef = this.db.collection(docPath.split('/')[0]).doc('timestamps').collection('users').doc('normal');
+            firestoreBatch.set(docRef, { users: timestamps }, { merge: true });
+            batchOps++;
+            delete this.timestampBatch[docPath];
         }
 
-        this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users into ${Object.keys(updatesByShard).length} deterministic shards for ${basePath}.`);
-        
-        delete batchData[basePath];
-    }
-    return count;
-}
-
-async flushBatches() {
-    if (this.batchTimeout) { clearTimeout(this.batchTimeout); this.batchTimeout = null; }
-    const firestoreBatch = this.db.batch();
-    let batchOps = 0;
-    batchOps += this._flushDataBatch(this.portfolioBatch, firestoreBatch, 'Portfolio');
-    batchOps += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, 'Trade History');
-    for (const docPath in this.timestampBatch) {
-        const timestamps = this.timestampBatch[docPath];
-        if (!Object.keys(timestamps).length) continue;
-        const docRef = this.db.collection(docPath.split('/')[0]).doc('timestamps').collection('users').doc('normal');
-        firestoreBatch.set(docRef, { users: timestamps }, { merge: true });
-        batchOps++;
-        delete this.timestampBatch[docPath];
-    }
+        for (const docPath in this.speculatorTimestampFixBatch) {
+            const updates = this.speculatorTimestampFixBatch[docPath];
+            if (!Object.keys(updates).length) continue;
+            firestoreBatch.set(this.db.doc(docPath), updates, { merge: true });
+            batchOps++;
+            delete this.speculatorTimestampFixBatch[docPath];
+        }
 
-    for (const docPath in this.speculatorTimestampFixBatch) {
-        const updates = this.speculatorTimestampFixBatch[docPath];
-        if (!Object.keys(updates).length) continue;
-        firestoreBatch.set(this.db.doc(docPath), updates, { merge: true });
-        batchOps++;
-        delete this.speculatorTimestampFixBatch[docPath];
+        for (const shardId in this.usernameMapUpdates) {
+            const updates = this.usernameMapUpdates[shardId];
+            if (updates && Object.keys(updates).length > 0) { firestoreBatch.set( this.db.collection(this.usernameMapCollectionName).doc(shardId), updates, { merge: true } ); batchOps++; this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`); } }
+        this.usernameMapUpdates = {}; 
+
+        if (this.processedSpeculatorCids.size) {
+            const cids = Array.from(this.processedSpeculatorCids);
+            this.processedSpeculatorCids.clear();
+            const snapshot = await this.db.collection(this.config.PENDING_SPECULATORS_COLLECTION).get();
+            snapshot.forEach(doc => { const docData = doc.data().users || {}; const cidsInDoc = cids.filter(cid => docData[cid]); if (!cidsInDoc.length) return;
+            const delBatch = this.db.batch();
+            const updates = Object.fromEntries(cidsInDoc.map(cid => [`users.${cid}`, FieldValue.delete()]));
+            delBatch.update(doc.ref, updates);
+            delBatch.commit();
+            this.logger.log('INFO', `[BATCH] Deleted ${cidsInDoc.length} CIDs from ${doc.id}`); }); }
+            
+        if (batchOps) await firestoreBatch.commit();
+        await this.headerManager.flushPerformanceUpdates();
+        this.logger.log('INFO', '[BATCH] All batches flushed successfully.');
     }
-
-    for (const shardId in this.usernameMapUpdates) {
-        const updates = this.usernameMapUpdates[shardId];
-        if (updates && Object.keys(updates).length > 0) { firestoreBatch.set( this.db.collection(this.usernameMapCollectionName).doc(shardId), updates, { merge: true } ); batchOps++; this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`); } }
-    this.usernameMapUpdates = {}; 
-
-    if (this.processedSpeculatorCids.size) {
-        const cids = Array.from(this.processedSpeculatorCids);
-        this.processedSpeculatorCids.clear();
-        const snapshot = await this.db.collection(this.config.PENDING_SPECULATORS_COLLECTION).get();
-        snapshot.forEach(doc => { const docData = doc.data().users || {}; const cidsInDoc = cids.filter(cid => docData[cid]); if (!cidsInDoc.length) return;
-        const delBatch = this.db.batch();
-        const updates = Object.fromEntries(cidsInDoc.map(cid => [`users.${cid}`, FieldValue.delete()]));
-        delBatch.update(doc.ref, updates);
-        delBatch.commit();
-        this.logger.log('INFO', `[BATCH] Deleted ${cidsInDoc.length} CIDs from ${doc.id}`); }); }
-        
-    if (batchOps) await firestoreBatch.commit();
-    await this.headerManager.flushPerformanceUpdates();
-    this.logger.log('INFO', '[BATCH] All batches flushed successfully.');
-}
 }
 
-module.exports = { FirestoreBatchManager }; 
+module.exports = { FirestoreBatchManager };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.180",
+  "version": "1.0.182",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [