bulltrackers-module 1.0.151 → 1.0.153

package/functions/computation-system/utils/data_loader.js

@@ -2,10 +2,9 @@
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
  * --- NEW: Added streamPortfolioData async generator ---
+ * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
  */
 
-// <<< FIX: REMOVED all top-level 'require' and 'dependencies' lines >>>
-
 /**
  * Sub-pipe: pipe.computationSystem.dataLoader.getPortfolioPartRefs
  * @param {object} config - The computation system configuration object.
@@ -19,34 +18,16 @@
 async function getPortfolioPartRefs(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     logger.log('INFO', `Getting portfolio part references for date: ${dateString}`);
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserPortfolioCollection, config.speculatorPortfolioCollection];
-
     for (const collectionName of collectionsToQuery) {
-        const blockDocsQuery = db.collection(collectionName);
-        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-
-        // --- START MODIFICATION ---
-        // Run all "listDocuments" calls in parallel instead of a sequential loop
-        const partsPromises = blockDocRefs.map(blockDocRef => {
-            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-            // Each call is individually retried
-            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`);
-        });
-
-        // Wait for all parallel queries to finish
-        const partDocArrays = await Promise.all(partsPromises);
-        
-        // Flatten the arrays of arrays into the final list
-        partDocArrays.forEach(partDocs => {
-            allPartRefs.push(...partDocs);
-        });
-        // --- END MODIFICATION ---
-    }
-
+    const blockDocsQuery = db.collection(collectionName);
+    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection); return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
+    const partDocArrays = await Promise.all(partsPromises);
+    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
     logger.log('INFO', `Found ${allPartRefs.length} portfolio part refs for ${dateString}`);
     return allPartRefs;
 }
@@ -55,23 +36,11 @@ async function getPortfolioPartRefs(config, deps, dateString) {
 async function loadDataByRefs(config, deps, refs) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
     const batchSize = config.partRefBatchSize || 50;
-
-    for (let i = 0; i < refs.length; i += batchSize) {
-        const batchRefs = refs.slice(i, i + batchSize);
-        const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
-
-        for (const doc of snapshots) {
-            if (!doc.exists) continue;
-            const data = doc.data();
-            if (data && typeof data === 'object') Object.assign(mergedPortfolios, data);
-            else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data);
-        }
-    }
-
+    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
+    for (const doc of snapshots) { if (!doc.exists) continue; const data = doc.data(); if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); } }
     return mergedPortfolios;
 }
 
@@ -79,7 +48,6 @@ async function loadDataByRefs(config, deps, refs) {
 async function loadFullDayMap(config, deps, partRefs) {
     const { logger } = deps;
     if (!partRefs.length) return {};
-
     logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
     const fullMap = await loadDataByRefs(config, deps, partRefs);
     logger.log('TRACE', `Full day map loaded with ${Object.keys(fullMap).length} users`);
@@ -90,19 +58,15 @@ async function loadFullDayMap(config, deps, partRefs) {
 async function loadDailyInsights(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
     logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
-
-    try {
-        const docRef = db.collection(insightsCollectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
-        logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-        return docSnap.data();
-    } catch (error) {
-        logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
-        return null;
+    try { const docRef = db.collection(insightsCollectionName).doc(dateString);
+    const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
+    if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
+    logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
+    return docSnap.data();
+    } catch (error) { logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
+    return null;
     }
 }
 
@@ -110,22 +74,17 @@ async function loadDailyInsights(config, deps, dateString) {
 async function loadDailySocialPostInsights(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     const collectionName = config.socialInsightsCollectionName || 'daily_social_insights';
     logger.log('INFO', `Loading social post insights for ${dateString} from ${collectionName}`);
-
-    try {
-        const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
-        const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
-        if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
-
-        const postsMap = {};
-        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
-        logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
-        return postsMap;
-    } catch (error) {
-        logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
-        return null;
+    try { const postsCollectionRef = db.collection(collectionName).doc(dateString).collection('posts');
+    const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
+    if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
+    const postsMap = {};
+    querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+    logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
+    return postsMap;
+    } catch (error) { logger.log('ERROR', `Failed to load social post insights for ${dateString}`, { errorMessage: error.message });
+    return null;
     }
 }
 
@@ -133,35 +92,18 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
 async function getHistoryPartRefs(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     logger.log('INFO', `Getting history part references for ${dateString}`);
     const allPartRefs = [];
     const collectionsToQuery = [config.normalUserHistoryCollection, config.speculatorHistoryCollection];
-
     for (const collectionName of collectionsToQuery) {
-        if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
-        const blockDocsQuery = db.collection(collectionName);
-        const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
-
-        // --- START MODIFICATION ---
-        // Run all "listDocuments" calls in parallel instead of a sequential loop
-        const partsPromises = blockDocRefs.map(blockDocRef => {
-            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
-            // Each call is individually retried
-            return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`);
-        });
-
-        // Wait for all parallel queries to finish
-        const partDocArrays = await Promise.all(partsPromises);
-        
-        // Flatten the arrays of arrays into the final list
-        partDocArrays.forEach(partDocs => {
-            allPartRefs.push(...partDocs);
-        });
-        // --- END MODIFICATION ---
-    }
-
+    if (!collectionName) { logger.log('WARN', `History collection undefined. Skipping.`); continue; }
+    const blockDocsQuery = db.collection(collectionName);
+    const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
+    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in ${collectionName}`); continue; }
+    const partsPromises = blockDocRefs.map(blockDocRef => { const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
+    return withRetry(() => partsCollectionRef.listDocuments(), `listDocuments(${partsCollectionRef.path})`); });
+    const partDocArrays = await Promise.all(partsPromises);
+    partDocArrays.forEach(partDocs => { allPartRefs.push(...partDocs); }); }
     logger.log('INFO', `Found ${allPartRefs.length} history part refs for ${dateString}`);
     return allPartRefs;
 }
@@ -173,61 +115,34 @@ async function getHistoryPartRefs(config, deps, dateString) {
  * @param {object} config - The computation system configuration object.
  * @param {object} deps - Contains db, logger, calculationUtils.
  * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
  */
-async function* streamPortfolioData(config, deps, dateString) {
+async function* streamPortfolioData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
-    const refs = await getPortfolioPartRefs(config, deps, dateString);
-    if (refs.length === 0) {
-        logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`);
-        return;
-    }
-
-    // Use the same batch size as loadDataByRefs for consistency
+    const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
+    if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
-    
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
-
-    for (let i = 0; i < refs.length; i += batchSize) {
-        const batchRefs = refs.slice(i, i + batchSize);
-        // Load one chunk of data
-        const data = await loadDataByRefs(config, deps, batchRefs);
-        // Yield it to the consumer, then release it from memory
-        yield data; 
-    }
+    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs);yield data; }
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
-// --- END: Stage 7 ---
 
 /**
  * --- NEW: Stage 8: Stream history data in chunks ---
  * Streams history data in chunks for a given date.
+ * @param {object} config - The computation system configuration object.
+ * @param {object} deps - Contains db, logger, calculationUtils.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
  */
-async function* streamHistoryData(config, deps, dateString) {
+async function* streamHistoryData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
-    const refs = await getHistoryPartRefs(config, deps, dateString); // <-- Uses history refs
-    if (refs.length === 0) {
-        logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`);
-        return;
-    }
-
+    const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
+    if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
     const batchSize = config.partRefBatchSize || 50; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
-
-    for (let i = 0; i < refs.length; i += batchSize) {
-        const batchRefs = refs.slice(i, i + batchSize);
-        const data = await loadDataByRefs(config, deps, batchRefs);
-        yield data; 
-    }
+    for (let i = 0; i < refs.length; i += batchSize) { const batchRefs = refs.slice(i, i + batchSize); const data = await loadDataByRefs(config, deps, batchRefs); yield data;  }
     logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
 }
 
-module.exports = {
-    getPortfolioPartRefs,
-    loadDataByRefs,
-    loadFullDayMap,
-    loadDailyInsights,
-    loadDailySocialPostInsights,
-    getHistoryPartRefs,
-    streamPortfolioData, // <-- EXPORT NEW FUNCTION
-    streamHistoryData  // <-- EXPORT NEW FUNCTION
-};
+module.exports = {getPortfolioPartRefs, loadDataByRefs, loadFullDayMap, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData };

package/functions/computation-system/utils/schema_capture.js

@@ -13,52 +13,18 @@
  */
 async function batchStoreSchemas(dependencies, config, schemas) {
     const { db, logger } = dependencies;
-    
-    // Check if schema capture is disabled
-    if (config.captureSchemas === false) {
-        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
-        return;
-    }
-    
+    if (config.captureSchemas === false)  { logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.'); return; }
     const batch = db.batch();
     const schemaCollection = config.schemaCollection || 'computation_schemas';
-    
     for (const item of schemas) {
-        try {
-            // 'item.schema' is now the pre-built schema from static getSchema()
-            if (!item.schema) {
-                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
-                continue;
-            }
-            
-            const docRef = db.collection(schemaCollection).doc(item.name);
-            
-            batch.set(docRef, {
-                computationName: item.name,
-                category: item.category,
-                schema: item.schema, // Use the provided schema directly
-                metadata: item.metadata || {},
-                lastUpdated: new Date()
-                // 'sampleKeys' is removed as we no longer have the output
-            }, { merge: true });
-        } catch (error) {
-            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, {
-                errorMessage: error.message
-            });
-        }
-    }
-    
-    try {
-        await batch.commit();
-        logger.log('INFO', `[SchemaCapture] Batch stored ${schemas.length} computation schemas`);
-    } catch (error) {
-        logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', {
-            errorMessage: error.message
-        });
-    }
+    try { if (!item.schema) { logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`); continue; }
+    const docRef = db.collection(schemaCollection).doc(item.name);
+    batch.set(docRef, { computationName: item.name, category: item.category, schema: item.schema, metadata: item.metadata || {}, lastUpdated: new Date() }, { merge: true });
+    } catch (error) { logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message }); } }
+    try { await batch.commit(); logger.log('INFO', `[SchemaCapture] Batch stored ${schemas.length} computation schemas`);
+    } catch (error) { logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message }); }
 }
 
 module.exports = {
-    // generateSchema and storeComputationSchema are now obsolete
     batchStoreSchemas
 };

package/functions/computation-system/utils/utils.js

@@ -10,41 +10,31 @@
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
 /** Stage 1: Normalize a calculation name to kebab-case */
-function normalizeName(name) {
-    return name.replace(/_/g, '-');
-}
+function normalizeName(name) { return name.replace(/_/g, '-'); }
 
 /** Stage 2: Commit a batch of writes in chunks */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     const batchSizeLimit = config.batchSizeLimit || 450;
     if (!writes.length) { logger.log('WARN', `[${operationName}] No writes to commit.`); return; }
-
     for (let i = 0; i < writes.length; i += batchSizeLimit) {
-        const chunk = writes.slice(i, i + batchSizeLimit);
-        const batch = db.batch();
-        chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
-
-        const chunkNum = Math.floor(i / batchSizeLimit) + 1;
-        const totalChunks = Math.ceil(writes.length / batchSizeLimit);
-        await withRetry(() => batch.commit(), `${operationName} (Chunk ${chunkNum}/${totalChunks})`);
-
-        logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`);
-    }
+    const chunk = writes.slice(i, i + batchSizeLimit);
+    const batch = db.batch();
+    chunk.forEach(write => batch.set(write.ref, write.data, { merge: true }));
+    const chunkNum = Math.floor(i / batchSizeLimit) + 1;
+    const totalChunks = Math.ceil(writes.length / batchSizeLimit);
+    await withRetry(() => batch.commit(), `${operationName} (Chunk ${chunkNum}/${totalChunks})`);
+    logger.log('INFO', `[${operationName}] Committed chunk ${chunkNum}/${totalChunks} (${chunk.length} ops).`); }
 }
 
 /** Stage 3: Generate an array of expected date strings between two dates */
 function getExpectedDateStrings(startDate, endDate) {
     const dateStrings = [];
     if (startDate <= endDate) {
-        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
-        const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
-        for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) {
-            dateStrings.push(new Date(d).toISOString().slice(0, 10));
-        }
-    }
+    const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
+    const endUTC = new Date(Date.UTC(endDate.getUTCFullYear(), endDate.getUTCMonth(), endDate.getUTCDate()));
+    for (let d = startUTC; d <= endUTC; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); } }
     return dateStrings;
 }
 
@@ -55,25 +45,12 @@ function getExpectedDateStrings(startDate, endDate) {
 async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     try {
-        if (!collectionName) {
-            logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`);
-            return null;
-        }
-        const query = db.collection(collectionName)
-            .where(FieldPath.documentId(), '>=', '2000-01-01')
-            .orderBy(FieldPath.documentId(), 'asc')
-            .limit(1);
-        
-        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
-        
-        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) {
-            return new Date(snapshot.docs[0].id + 'T00:00:00Z');
-        }
-    } catch (e) {
-        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
-    }
+    if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`); return null; }
+    const query = db.collection(collectionName) .where(FieldPath.documentId(), '>=', '2000-01-01') .orderBy(FieldPath.documentId(), 'asc') .limit(1);
+    const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
+    if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) { return new Date(snapshot.docs[0].id + 'T00:00:00Z'); }
+    } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
     return null;
 }
 
@@ -81,32 +58,14 @@ async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
 async function getFirstDateFromCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-
     let earliestDate = null;
-    try {
-        if (!collectionName) {
-            logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`);
-            return null;
-        }
-        const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
-        if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
-
-        for (const blockDocRef of blockDocRefs) {
-            const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection)
-                .where(FieldPath.documentId(), '>=', '2000-01-01')
-                .orderBy(FieldPath.documentId(), 'asc')
-                .limit(1);
-
-            const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
-            if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) {
-                const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
-                if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate;
-            }
-        }
-    } catch (e) {
-        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
-    }
-
+    try { if (!collectionName) { logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`); return null; }
+    const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
+    if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
+    for (const blockDocRef of blockDocRefs) { const snapshotQuery = blockDocRef.collection(config.snapshotsSubcollection) .where(FieldPath.documentId(), '>=', '2000-01-01') .orderBy(FieldPath.documentId(), 'asc') .limit(1);
+    const snapshotSnap = await withRetry(() => snapshotQuery.get(), `GetEarliestSnapshot(${blockDocRef.path})`);
+    if (!snapshotSnap.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshotSnap.docs[0].id)) { const foundDate = new Date(snapshotSnap.docs[0].id + 'T00:00:00Z');
+    if (!earliestDate || foundDate < earliestDate) earliestDate = foundDate; } } } catch (e) { logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message }); }
     return earliestDate;
 }
 
@@ -116,71 +75,25 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
     logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-
-    // These return null on error or if empty
-    const [
-        investorDate,
-        speculatorDate,
-        investorHistoryDate,
-        speculatorHistoryDate,
-        insightsDate,
-        socialDate
-    ] = await Promise.all([
-        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
-        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
-        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
-        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
+    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate ] = await Promise.all([
+        getFirstDateFromCollection      (config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection      (config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection      (config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection      (config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
         getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
     ]);
     
-    // Helper to find the minimum (earliest) of a set of dates
-    const getMinDate = (...dates) => {
-        const validDates = dates.filter(Boolean); // Filter out nulls
-        if (validDates.length === 0) return null;
-        return new Date(Math.min(...validDates));
-    };
-
+    const getMinDate            = (...dates) => { const validDates = dates.filter(Boolean);  if (validDates.length === 0) return null; return new Date(Math.min(...validDates)); };
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate = getMinDate(insightsDate); // Already a single date
-    const earliestSocialDate = getMinDate(socialDate); // Already a single date
-
-    const absoluteEarliest = getMinDate(
-        earliestPortfolioDate,
-        earliestHistoryDate,
-        earliestInsightsDate,
-        earliestSocialDate
-    );
-
-    // Fallback date
-    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-
-    const result = {
-        portfolio: earliestPortfolioDate || new Date('2999-12-31'), // Use a 'far future' date if null
-        history: earliestHistoryDate || new Date('2999-12-31'),
-        insights: earliestInsightsDate || new Date('2999-12-31'),
-        social: earliestSocialDate || new Date('2999-12-31'),
-        absoluteEarliest: absoluteEarliest || fallbackDate // Use fallback for the main loop
-    };
-
-    logger.log('INFO', 'Earliest data availability map built:', {
-        portfolio: result.portfolio.toISOString().slice(0, 10),
-        history: result.history.toISOString().slice(0, 10),
-        insights: result.insights.toISOString().slice(0, 10),
-        social: result.social.toISOString().slice(0, 10),
-        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
-    });
-
+    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate  = getMinDate(insightsDate);
+    const earliestSocialDate    = getMinDate(socialDate); 
+    const absoluteEarliest      = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate );
+    const fallbackDate          = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+    const result                = { portfolio: earliestPortfolioDate || new Date('2999-12-31'),  history: earliestHistoryDate || new Date('2999-12-31'), insights: earliestInsightsDate || new Date('2999-12-31'), social: earliestSocialDate || new Date('2999-12-31'), absoluteEarliest: absoluteEarliest || fallbackDate };
+    logger.log('INFO', 'Earliest data availability map built:', { portfolio: result.portfolio.toISOString().slice(0, 10), history: result.history.toISOString().slice(0, 10), insights: result.insights.toISOString().slice(0, 10), social: result.social.toISOString().slice(0, 10), absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) });
     return result;
 }
 
-module.exports = {
-    FieldValue,
-    FieldPath,
-    normalizeName,
-    commitBatchInChunks,
-    getExpectedDateStrings,
-    // getFirstDateFromSourceData, // This is replaced
-    getEarliestDataDates, // <-- EXPORT NEW FUNCTION
-};
+module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates };

package/functions/core/utils/firestore_utils.js

@@ -260,18 +260,9 @@ async function getSpeculatorsToUpdate(dependencies, config) {
     const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
     logger.log('INFO','[Core Utils] Getting speculators to update...');
     const updates = [];
-    
-    // ⚠️ NEW: Collect per user first
-    const userMap = new Map(); // userId -> { instruments: Set }
-    
-    try {
-        const blocksRef = db.collection(speculatorBlocksCollectionName);
-        const snapshot = await blocksRef.get();
-        if (snapshot.empty) { 
-            logger.log('INFO','[Core Utils] No speculator blocks found.'); 
-            return []; 
-        }
-        
+    const userMap = new Map(); 
+    try { const blocksRef = db.collection(speculatorBlocksCollectionName); const snapshot = await blocksRef.get();
+    if (snapshot.empty) {  logger.log('INFO','[Core Utils] No speculator blocks found.');  return [];  }
         snapshot.forEach(doc => {
             const blockData = doc.data();
             for (const key in blockData) {
@@ -281,42 +272,13 @@ async function getSpeculatorsToUpdate(dependencies, config) {
                 const userData = blockData[key]; 
                 const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
                 const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
-                
                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
-                    if (!userMap.has(userId)) {
-                        userMap.set(userId, new Set());
-                    }
-                    if (userData.instruments && Array.isArray(userData.instruments)) {
-                        userData.instruments.forEach(id => userMap.get(userId).add(id));
-                    }
-                }
-            }
-        });
-        
-        // ⚠️ NEW: Return one task per user with ALL instruments
-        for (const [userId, instrumentSet] of userMap) {
-            updates.push({ 
-                userId, 
-                instruments: Array.from(instrumentSet) // ⚠️ Array of all instruments
-            });
-        }
-        
+                    if (!userMap.has(userId)) { userMap.set(userId, new Set()); }
+                    if (userData.instruments && Array.isArray(userData.instruments)) { userData.instruments.forEach(id => userMap.get(userId).add(id)); } } } });
+        for (const [userId, instrumentSet] of userMap) { updates.push({  userId,  instruments: Array.from(instrumentSet) }); }
         logger.log('INFO',`[Core Utils] Found ${updates.length} speculator users to update (covering ${[...userMap.values()].reduce((sum, set) => sum + set.size, 0)} total instruments).`);
         return updates;
-    } catch (error) {
-        logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message });
-        throw error;
-    }
+    } catch (error) { logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message }); throw error; }
 }
 
-module.exports = {
-  getLatestNormalUserPortfolios,
-  resetProxyLocks,
-  getBlockCapacities,
-  getExclusionIds,
-  getPrioritizedSpeculators,
-  clearCollection,
-  batchWriteShardedIds,
-  getNormalUsersToUpdate,
-  getSpeculatorsToUpdate,
-};
+module.exports = { getLatestNormalUserPortfolios, resetProxyLocks, getBlockCapacities, getExclusionIds, getPrioritizedSpeculators, clearCollection, batchWriteShardedIds, getNormalUsersToUpdate, getSpeculatorsToUpdate, };