bulltrackers-module 1.0.56 → 1.0.58

package/functions/user-activity-sampler/helpers/sampler_helpers.js

@@ -0,0 +1,207 @@
+/**
+ * @fileoverview Main pipe: pipe.maintenance.runUserActivitySampler
+ * Samples user activity levels based on the Rankings API.
+ */
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * Main pipe: pipe.maintenance.runUserActivitySampler
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, firestoreUtils.
+ * @returns {Promise<object>} Summary of the sampling process.
+ */
+exports.runUserActivitySampler = async (config, dependencies) => {
+    const { db, logger, headerManager, proxyManager, firestoreUtils } = dependencies;
+    const today = new Date().toISOString().slice(0, 10);
+    logger.log('INFO', '[UserActivitySampler] Starting user activity sampling...');
+
+    // Validate configuration
+    if (!config.rankingsApiUrl || !config.targetPublicUsersPerBlock || !config.apiBatchSize || !config.outputCollectionName || !config.allHighValueBlocks || !config.exclusionConfig) {
+        logger.log('ERROR', '[UserActivitySampler] Missing required configuration.');
+        throw new Error('Missing required configuration for User Activity Sampler.');
+    }
+
+    try {
+        await firestoreUtils.resetProxyLocks(dependencies, config); // Use shared config for proxy path
+
+        const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, config.exclusionConfig, 'sampler'); // Pass 'sampler' or similar unique type
+        logger.log('INFO', `[UserActivitySampler] Fetched ${exclusionIds.size} exclusion IDs.`);
+
+        const blockResults = [];
+        let totalPublicSampled = 0;
+        let totalPrivateEstimate = 0;
+
+        for (const block of config.allHighValueBlocks) {
+            const blockId = block.startId;
+            logger.log('INFO', `[UserActivitySampler] Processing block ${blockId}...`);
+
+            let N_public_sampled_block = 0;
+            let N_sampled_total_block = 0;
+            const public_users_data = []; // Store { CID, LastActivity }
+
+            const MAX_ATTEMPTS = config.maxSamplingAttemptsPerBlock || 1000; // Prevent infinite loops
+            let attempts = 0;
+
+            while (N_public_sampled_block < config.targetPublicUsersPerBlock && attempts < MAX_ATTEMPTS) {
+                attempts++;
+                const cidsToSample = [];
+                // Generate CIDs, ensuring they are not excluded
+                while (cidsToSample.length < config.apiBatchSize) {
+                    const randomId = String(Math.floor(Math.random() * 1000000) + blockId);
+                    if (!exclusionIds.has(randomId)) {
+                        cidsToSample.push(parseInt(randomId, 10));
+                        // Add temporarily to exclusion set for this run to avoid duplicates within batches
+                        exclusionIds.add(randomId);
+                    }
+                }
+                N_sampled_total_block += cidsToSample.length;
+
+                let selectedHeader = null;
+                let wasSuccess = false;
+                try {
+                    selectedHeader = await headerManager.selectHeader();
+                    if (!selectedHeader) throw new Error("Could not select header.");
+
+                    const response = await proxyManager.fetch(config.rankingsApiUrl, {
+                        method: 'POST',
+                        headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
+                        body: JSON.stringify(cidsToSample),
+                    });
+
+                    if (!response || typeof response.json !== 'function') {
+                         logger.log('WARN', `[UserActivitySampler] Invalid response structure from proxy for block ${blockId}, batch attempt ${attempts}. Skipping batch.`);
+                         // Consider retrying or specific error handling
+                         continue; // Skip to next attempt
+                    }
+
+
+                    if (!response.ok) {
+                        // Log API errors but continue sampling other batches/blocks
+                        const errorText = await response.text();
+                        logger.log('WARN', `[UserActivitySampler] API error ${response.status} for block ${blockId}, batch attempt ${attempts}. Skipping batch. Error: ${errorText}`);
+                        wasSuccess = false;
+                        continue; // Skip to next attempt
+                    }
+
+                    wasSuccess = true;
+                    const publicUsersBatch = await response.json();
+
+                    if (Array.isArray(publicUsersBatch)) {
+                        const N_public_returned_batch = publicUsersBatch.length;
+                        N_public_sampled_block += N_public_returned_batch;
+                        public_users_data.push(...publicUsersBatch.map(u => ({
+                            CID: u.CID,
+                            LastActivity: u.Value?.LastActivity // Handle potential missing Value
+                        })));
+                    } else {
+                         logger.log('WARN', `[UserActivitySampler] API response was not an array for block ${blockId}, batch attempt ${attempts}. Skipping batch.`);
+                         continue;
+                    }
+
+
+                } catch (fetchError) {
+                    logger.log('ERROR', `[UserActivitySampler] Fetch failed for block ${blockId}, batch attempt ${attempts}. Skipping batch.`, { errorMessage: fetchError.message });
+                    wasSuccess = false; // Mark as failure for header performance
+                    // Continue to next attempt
+                } finally {
+                    if (selectedHeader) {
+                        headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+                    }
+                }
+                 // Small delay between batches to avoid overwhelming API/proxies
+                 await new Promise(resolve => setTimeout(resolve, config.delayBetweenBatchesMs || 200));
+
+            } // End while loop for block sampling
+
+            if (attempts >= MAX_ATTEMPTS) {
+                 logger.log('WARN', `[UserActivitySampler] Reached max sampling attempts (${MAX_ATTEMPTS}) for block ${blockId}. Proceeding with ${N_public_sampled_block} users.`);
+            }
+
+
+            const f_private_block = N_sampled_total_block > 0 ? 1 - (N_public_sampled_block / N_sampled_total_block) : 0;
+
+            // Categorize users
+            const counts = { A1: 0, A2: 0, A3: 0, A4: 0 };
+            const now = new Date();
+            const oneDayAgo = new Date(now.getTime() - (24 * 60 * 60 * 1000));
+            const oneWeekAgo = new Date(now.getTime() - (7 * 24 * 60 * 60 * 1000));
+            const oneMonthAgo = new Date(now.getTime() - (30 * 24 * 60 * 60 * 1000)); // Approx
+             const threeMonthsAgo = new Date(now.getTime() - (90 * 24 * 60 * 60 * 1000)); // Approx
+
+
+            public_users_data.forEach(user => {
+                if (!user.LastActivity) {
+                    counts.A4++; // Assume inactive if LastActivity is missing
+                    return;
+                }
+                try {
+                    const lastActivityDate = new Date(user.LastActivity);
+                     if (isNaN(lastActivityDate)) {
+                         counts.A4++; // Treat invalid dates as inactive
+                         return;
+                     }
+
+                    if (lastActivityDate >= oneDayAgo) counts.A1++;
+                    else if (lastActivityDate >= oneWeekAgo) counts.A2++;
+                    else if (lastActivityDate >= threeMonthsAgo) counts.A3++; // Changed from oneMonthAgo based on description
+                    else counts.A4++;
+                } catch(e) {
+                     logger.log('WARN', `[UserActivitySampler] Error parsing LastActivity date '${user.LastActivity}' for user ${user.CID}. Counting as A4.`);
+                     counts.A4++;
+                }
+
+            });
+
+            const fractions = {};
+            for (const category in counts) {
+                fractions[category] = N_public_sampled_block > 0 ? (counts[category] / N_public_sampled_block) : 0;
+            }
+
+            // TODO: Fetch or define N_block (total users for this block)
+            const N_block =  1000000; 
+
+            const estimatedCounts = {};
+            for (const category in fractions) {
+                estimatedCounts[category] = Math.round(fractions[category] * N_block);
+            }
+
+            const result = {
+                blockId: blockId,
+                sampledDate: today,
+                f_private: f_private_block,
+                publicSampleSize: N_public_sampled_block,
+                totalUsersInBlock: N_block, // Add this if available
+                activityCounts_Sample: counts,
+                activityFractions_Sample: fractions,
+                estimatedCounts_TotalBlock: estimatedCounts,
+                lastUpdated: FieldValue.serverTimestamp()
+            };
+
+            blockResults.push(result);
+            totalPublicSampled += N_public_sampled_block;
+            totalPrivateEstimate += f_private_block * N_block;
+
+            // Store result for this block
+            const docRef = db.collection(config.outputCollectionName).doc(`${blockId}_${today}`);
+            await docRef.set(result);
+            logger.log('INFO', `[UserActivitySampler] Stored results for block ${blockId}. Sampled: ${N_public_sampled_block}. Private fraction: ${f_private_block.toFixed(3)}.`);
+
+        } // End for loop through blocks
+
+        await headerManager.flushPerformanceUpdates();
+        logger.log('SUCCESS', `[UserActivitySampler] Sampling complete. Processed ${blockResults.length} blocks. Total public sampled: ${totalPublicSampled}.`);
+
+        return {
+            success: true,
+            processedBlocks: blockResults.length,
+            totalPublicSampled: totalPublicSampled,
+            estimatedTotalPrivate: Math.round(totalPrivateEstimate)
+        };
+
+    } catch (error) {
+        logger.log('ERROR', '[UserActivitySampler] Fatal error during sampling process.', { errorMessage: error.message, errorStack: error.stack });
+        // Attempt to flush performance even on error
+        try { await headerManager.flushPerformanceUpdates(); } catch (e) { /* ignore flush error */ }
+        throw error; // Re-throw to indicate failure
+    }
+};

package/index.js

@@ -81,6 +81,7 @@ const maintenance = {
     handleInvalidSpeculator: require('./functions/invalid-speculator-handler/helpers/handler_helpers').handleInvalidSpeculator,
     runFetchInsights: require('./functions/fetch-insights/helpers/handler_helpers').fetchAndStoreInsights,
     runFetchPrices: require('./functions/etoro-price-fetcher/helpers/handler_helpers').fetchAndStorePrices,
+    runUserActivitySampler: require('./functions/user-activity-sampler/helpers/sampler_helpers').runUserActivitySampler, 
 };
 
 // --- Pipe 7: Proxy ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.56",
+  "version": "1.0.58",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -15,7 +15,8 @@
     "functions/speculator-cleanup-orchestrator/",
     "functions/fetch-insights/",
     "functions/etoro-price-fetcher/",
-    "functions/appscript-api/"
+    "functions/appscript-api/",
+    "functions/user-activity-sampler/"
   ],
   "keywords": [
     "bulltrackers",