bulltrackers-module 1.0.96 → 1.0.98

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -1,6 +1,8 @@
 /**
  * @fileoverview Main pipe: pipe.maintenance.runFetchPrices
  * REFACTORED: Now writes to the new sharded `asset_prices` collection.
+ * FIXED: Uses docRef.update() to correctly merge nested price maps
+ * instead of docRef.set({ merge: true }) which creates flat keys.
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
@@ -77,7 +79,7 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
                     shardUpdates[shardId] = {};
                 }
                 
-                // Use dot notation to update only the specific price for the specific instrument
+                // Use dot notation to define the update path
                 const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
                 const updatePath = `${instrumentIdStr}.lastUpdated`;
 
@@ -92,8 +94,11 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
             const docRef = db.collection(priceCollectionName).doc(shardId);
             const payload = shardUpdates[shardId];
             
-            // Use set with merge:true to update all instruments in the shard doc at once
-            batchPromises.push(docRef.set(payload, { merge: true }));
+            // --- THIS IS THE FIX ---
+            // Use .update() to correctly merge data into nested maps.
+            // Using .set(payload, { merge: true }) creates the flat, broken keys.
+            batchPromises.push(docRef.update(payload));
+            // --- END FIX ---
         }
 
         await Promise.all(batchPromises);

package/index.js

@@ -80,8 +80,6 @@ const maintenance = {
     handleInvalidSpeculator: require('./functions/invalid-speculator-handler/helpers/handler_helpers').handleInvalidSpeculator,
     runFetchInsights: require('./functions/fetch-insights/helpers/handler_helpers').fetchAndStoreInsights,
     runFetchPrices: require('./functions/etoro-price-fetcher/helpers/handler_helpers').fetchAndStorePrices,
-    runUserActivitySamplerOrchestrator: require('./functions/user-activity-sampler/helpers/sampler_helpers').runUserActivitySamplerOrchestrator, 
-    handleSampleBlockTask: require('./functions/user-activity-sampler/helpers/sampler_helpers').handleSampleBlockTask, 
     runSocialOrchestrator: require('./functions/social-orchestrator/helpers/orchestrator_helpers').runSocialOrchestrator,
     handleSocialTask: require('./functions/social-task-handler/helpers/handler_helpers').handleSocialTask,
     runBackfillAssetPrices: require('./functions/price-backfill/helpers/handler_helpers').runBackfillAssetPrices,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.96",
+  "version": "1.0.98",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -16,7 +16,6 @@
     "functions/fetch-insights/",
     "functions/etoro-price-fetcher/",
     "functions/appscript-api/",
-    "functions/user-activity-sampler/",
     "functions/social-orchestrator/",
     "functions/social-task-handler/",
     "functions/price-backfill/"

package/functions/user-activity-sampler/helpers/sampler_helpers.js

@@ -1,307 +0,0 @@
-/**
- * @fileoverview Main pipe: pipe.maintenance.runUserActivitySampler
- * REFACTORED: Now split into an orchestrator and a task handler.
- * - runUserActivitySamplerOrchestrator: Publishes one task per block.
- * - handleSampleBlockTask: Processes a single block with parallel fetching.
- */
-const { FieldValue } = require('@google-cloud/firestore');
-const pLimit = require('p-limit'); // npm install p-limit
-
-/**
- * Helper: delay
- */
-function delay(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/**
- * Helper: fetch with retry and exponential backoff on 429
- */
-async function fetchWithRetry(fetchFn, maxRetries = 5, logger, taskId) {
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-        try {
-            const response = await fetchFn();
-            if (response.status !== 429) return response;
-
-            const backoff = 1000 * Math.pow(2, attempt); // 1s, 2s, 4s, 8s...
-            logger.log('WARN', `[SamplerTask/${taskId}] Received 429. Retrying in ${backoff}ms (attempt ${attempt + 1}/${maxRetries})`);
-            await delay(backoff);
-        } catch (err) {
-            logger.log('ERROR', `[SamplerTask/${taskId}] Fetch failed on attempt ${attempt + 1}.`, { errorMessage: err.message });
-            if (attempt === maxRetries) throw err;
-            const backoff = 500 * Math.pow(2, attempt); // slightly faster backoff for network errors
-            await delay(backoff);
-        }
-    }
-    throw new Error('Max retries reached for fetchWithRetry');
-}
-
-/**
- * Main pipe (Orchestrator): pipe.maintenance.runUserActivitySamplerOrchestrator
- * This function is triggered by a schedule. It fans out the work by
- * publishing one Pub/Sub message for each block to be sampled.
- *
- * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, firestoreUtils, pubsubUtils.
- * @returns {Promise<object>} Summary of the orchestration.
- */
-exports.runUserActivitySamplerOrchestrator = async (config, dependencies) => {
-    const { logger, firestoreUtils, pubsubUtils } = dependencies;
-    logger.log('INFO', '[SamplerOrchestrator] Starting user activity sampling orchestration...');
-
-    // Validate configuration
-    if (!config.allHighValueBlocks || !Array.isArray(config.allHighValueBlocks) || !config.samplerTaskTopicName) {
-        logger.log('ERROR', '[SamplerOrchestrator] Missing required configuration: allHighValueBlocks (array) or samplerTaskTopicName.');
-        throw new Error('Missing required configuration for Sampler Orchestrator.');
-    }
-
-    try {
-        // Reset locks once for all tasks
-        await firestoreUtils.resetProxyLocks(dependencies, config);
-
-        const tasks = [];
-        for (const block of config.allHighValueBlocks) {
-            // Ensure block and block.startId exist
-            if (!block || typeof block.startId === 'undefined') {
-                logger.log('WARN', '[SamplerOrchestrator] Skipping invalid block configuration:', block);
-                continue;
-            }
-            tasks.push({
-                type: 'sample-block',
-                blockId: block.startId
-            });
-        }
-
-        if (tasks.length === 0) {
-            logger.log('WARN', '[SamplerOrchestrator] No valid blocks found to sample.');
-            return { success: true, message: "No valid blocks configured." };
-        }
-
-        // Use pubsubUtils to batch publish all block tasks
-        await pubsubUtils.batchPublishTasks(dependencies, {
-            topicName: config.samplerTaskTopicName,
-            tasks: tasks,
-            taskType: 'sampler-block-task'
-        });
-
-        logger.log('SUCCESS', `[SamplerOrchestrator] Successfully published ${tasks.length} block sampling tasks.`);
-        return { success: true, blocksQueued: tasks.length };
-
-    } catch (error) {
-        logger.log('ERROR', '[SamplerOrchestrator] Fatal error during orchestration.', { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    }
-};
-
-/**
- * Internal Helper: Fetches a single batch of CIDs for the sampler.
- * This is designed to be called in parallel.
- */
-async function fetchSampleBatch(blockId, config, dependencies, processedInThisRun) {
-    const { logger, headerManager, proxyManager } = dependencies;
-    const cidsToSample = [];
-
-    // 1. Generate CIDs for this batch
-    while (cidsToSample.length < config.apiBatchSize) {
-        const randomId = String(Math.floor(Math.random() * 1000000) + blockId);
-        // Use a Set to prevent processing the same ID twice *within this run*
-        if (!processedInThisRun.has(randomId)) {
-            cidsToSample.push(parseInt(randomId, 10));
-            processedInThisRun.add(randomId);
-        }
-        // Note: If this loops too long, it could be inefficient, but it's
-        // unlikely with a large ID space and a reasonable sample size.
-    }
-
-    let selectedHeader = null;
-    let wasSuccess = false;
-    try {
-        selectedHeader = await headerManager.selectHeader();
-        if (!selectedHeader) throw new Error("Could not select header.");
-        
-        const urlWithParam = `${config.rankingsApiUrl}?Period=LastTwoYears`;
-        const response = await proxyManager.fetch(urlWithParam, {
-            method: 'POST',
-            headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
-            body: JSON.stringify(cidsToSample),
-        });
-
-        if (!response || typeof response.json !== 'function') {
-            logger.log('WARN', `[SamplerTask] Invalid response structure from proxy for block ${blockId}.`);
-            logger.log('DEBUG', `[SamplerTask] Response details: ${JSON.stringify(response)}`);
-            return { success: false, cidsSent: cidsToSample.length, publicUsers: [] };
-        }
-
-        if (!response.ok) {
-            const errorText = await response.text();
-            logger.log('WARN', `[SamplerTask] API error ${response.status} for block ${blockId}. Error: ${errorText}`);
-            wasSuccess = false;
-            return { success: false, cidsSent: cidsToSample.length, publicUsers: [] };
-        }
-
-        wasSuccess = true;
-        const publicUsersBatch = await response.json();
-
-        if (!Array.isArray(publicUsersBatch)) {
-            logger.log('WARN', `[SamplerTask] API response was not an array for block ${blockId}.`);
-            return { success: false, cidsSent: cidsToSample.length, publicUsers: [] };
-        }
-        
-        // Return a successful result
-        return {
-            success: true,
-            cidsSent: cidsToSample.length,
-            publicUsers: publicUsersBatch.map(u => ({
-                CID: u.CID,
-                LastActivity: u.Value?.LastActivity
-            }))
-        };
-
-    } catch (fetchError) {
-        logger.log('ERROR', `[SamplerTask] Fetch failed for block ${blockId}.`, { errorMessage: fetchError.message });
-        wasSuccess = false;
-        return { success: false, cidsSent: cidsToSample.length, publicUsers: [] };
-    } finally {
-        if (selectedHeader) {
-            headerManager.updatePerformance(selectedHeader.id, wasSuccess);
-        }
-    }
-}
-
-/**
- * Updated Task Handler: handleSampleBlockTask
- */
-exports.handleSampleBlockTask = async (message, context, config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
-
-    let task;
-    try {
-        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
-    } catch (e) {
-        logger.log('ERROR', '[SamplerTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return;
-    }
-
-    const { blockId } = task;
-    const taskId = `block-${blockId}-${context.eventId || Date.now()}`;
-    const today = new Date().toISOString().slice(0, 10);
-    logger.log('INFO', `[SamplerTask/${taskId}] Processing block ${blockId}...`);
-
-    // Config validation
-    if (!config.rankingsApiUrl || !config.targetPublicUsersPerBlock || !config.apiBatchSize || !config.outputCollectionName || !config.parallelRequests) {
-        logger.log('ERROR', `[SamplerTask/${taskId}] Missing required configuration for task execution.`);
-        throw new Error('Missing required configuration for Sampler Task.');
-    }
-
-    const processedInThisRun = new Set();
-    let N_public_sampled_block = 0;
-    let N_sampled_total_block = 0;
-    const public_users_data = [];
-    const MAX_ATTEMPTS = config.maxSamplingAttemptsPerBlock || 1000;
-    let totalBatchesAttempted = 0;
-
-    const limit = pLimit(config.parallelRequests || 3);
-
-    try {
-        while (N_public_sampled_block < config.targetPublicUsersPerBlock && totalBatchesAttempted < MAX_ATTEMPTS) {
-            const numRequests = Math.min(config.parallelRequests, MAX_ATTEMPTS - totalBatchesAttempted);
-            const promises = [];
-
-            for (let i = 0; i < numRequests; i++) {
-                promises.push(limit(async () => {
-                    // Wrap fetchSampleBatch to include fetchWithRetry
-                    async function fetchBatchWithRetry() {
-                        return await fetchSampleBatch(blockId, config, {
-                            ...dependencies,
-                            proxyManager: {
-                                fetch: (url, options) =>
-                                    fetchWithRetry(() => proxyManager.fetch(url, options), 5, logger, taskId)
-                            }
-                        }, processedInThisRun);
-                    }
-                    return await fetchBatchWithRetry();
-                }));
-            }
-
-            const results = await Promise.allSettled(promises);
-            totalBatchesAttempted += numRequests;
-
-            // Process results
-            for (const result of results) {
-                if (result.status === 'fulfilled' && result.value.success) {
-                    const batchResult = result.value;
-                    N_sampled_total_block += batchResult.cidsSent;
-                    N_public_sampled_block += batchResult.publicUsers.length;
-                    public_users_data.push(...batchResult.publicUsers);
-                } else if (result.status === 'fulfilled' && !result.value.success) {
-                    N_sampled_total_block += result.value.cidsSent;
-                } else {
-                    logger.log('WARN', `[SamplerTask/${taskId}] A sample fetch promise was rejected.`, { reason: result.reason });
-                }
-            }
-
-            logger.log('INFO', `[SamplerTask/${taskId}] Batch complete. Total public sampled: ${N_public_sampled_block}/${config.targetPublicUsersPerBlock}`);
-        }
-
-        if (totalBatchesAttempted >= MAX_ATTEMPTS) {
-            logger.log('WARN', `[SamplerTask/${taskId}] Reached max sampling attempts (${MAX_ATTEMPTS}). Proceeding with ${N_public_sampled_block} users.`);
-        }
-
-        // --- Calculate and Store Results ---
-        const f_private_block = N_sampled_total_block > 0 ? 1 - (N_public_sampled_block / N_sampled_total_block) : 0;
-        const counts = { A1: 0, A2: 0, A3: 0, A4: 0 };
-        const now = new Date();
-        const oneDayAgo = new Date(now.getTime() - (24 * 60 * 60 * 1000));
-        const oneWeekAgo = new Date(now.getTime() - (7 * 24 * 60 * 60 * 1000));
-        const threeMonthsAgo = new Date(now.getTime() - (90 * 24 * 60 * 60 * 1000));
-
-        public_users_data.forEach(user => {
-            if (!user.LastActivity) { counts.A4++; return; }
-            try {
-                const lastActivityDate = new Date(user.LastActivity);
-                if (isNaN(lastActivityDate)) { counts.A4++; return; }
-                if (lastActivityDate >= oneDayAgo) counts.A1++;
-                else if (lastActivityDate >= oneWeekAgo) counts.A2++;
-                else if (lastActivityDate >= threeMonthsAgo) counts.A3++;
-                else counts.A4++;
-            } catch {
-                counts.A4++;
-            }
-        });
-
-        const fractions = {};
-        for (const category in counts) fractions[category] = N_public_sampled_block > 0 ? (counts[category] / N_public_sampled_block) : 0;
-
-        const N_block = 1000000;
-        const estimatedCounts = {};
-        for (const category in fractions) estimatedCounts[category] = Math.round(fractions[category] * N_block);
-
-        const result = {
-            blockId,
-            sampledDate: today,
-            f_private: f_private_block,
-            publicSampleSize: N_public_sampled_block,
-            totalSampleSize: N_sampled_total_block,
-            totalUsersInBlock: N_block,
-            activityCounts_Sample: counts,
-            activityFractions_Sample: fractions,
-            estimatedCounts_TotalBlock: estimatedCounts,
-            lastUpdated: FieldValue.serverTimestamp()
-        };
-
-        const docRef = db.collection(config.outputCollectionName).doc(`${blockId}_${today}`);
-        await docRef.set(result);
-        logger.log('SUCCESS', `[SamplerTask/${taskId}] Stored results for block ${blockId}. Sampled: ${N_public_sampled_block}. Private fraction: ${f_private_block.toFixed(3)}.`);
-
-    } catch (error) {
-        logger.log('ERROR', `[SamplerTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    } finally {
-        try {
-            await headerManager.flushPerformanceUpdates();
-            logger.log('INFO', `[SamplerTask/${taskId}] Header performance flushed.`);
-        } catch (flushError) {
-            logger.log('ERROR', `[SamplerTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message });
-        }
-    }
-};