bulltrackers-module 1.0.723 → 1.0.725

package/functions/computation-system/context/ManifestBuilder.js

@@ -231,7 +231,6 @@ function buildManifest(productLinesToRun = [], calculations) {
         const manifestEntry = {
             name:          normalizedName,
             class:         Class,
-            // [CHANGED] Strictly use the folderName as the category.
             category:      folderName,     
             sourcePackage: folderName,   
             type:                 metadata.type,
@@ -247,7 +246,6 @@ function buildManifest(productLinesToRun = [], calculations) {
             userType:             metadata.userType,
             dependencies: dependencies,
             schedule:             metadata.schedule || null,
-            // [NEW] Added TTL Policy to Manifest
             ttlDays:              metadata.ttlDays, 
             pass: 0,
             hash: intrinsicHash,

package/functions/computation-system/executors/MetaExecutor.js

@@ -130,7 +130,7 @@ class MetaExecutor {
             previousComputedDependencies: prevDeps, config, deps,
             allRankings: rankings,
             allRankingsYesterday: rankingsYesterday,
-            piMasterList, // <--- INJECTED HERE
+            piMasterList, 
             ...variableRoots,
             seriesData
         };

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -112,16 +112,17 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
             }
         }
         
-        // Write to BigQuery using load jobs (free, batched)
+        // Write to BigQuery using MERGE (prevents duplicates if fetcher runs twice on same day)
         if (process.env.BIGQUERY_ENABLED !== 'false' && bigqueryRows.length > 0) {
             try {
-                const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                const { insertRowsWithMerge, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
                 await ensureAssetPricesTable(logger);
                 
                 const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                // Use MERGE with dedup keys to prevent duplicates on re-runs
+                await insertRowsWithMerge(datasetId, 'asset_prices', bigqueryRows, ['date', 'instrument_id'], logger);
                 
-                logger.log('INFO', `[PriceFetcherHelpers] Successfully stored ${bigqueryRows.length} daily price records to BigQuery`);
+                logger.log('INFO', `[PriceFetcherHelpers] Successfully stored ${bigqueryRows.length} daily price records to BigQuery [MERGE]`);
             } catch (bqError) {
                 logger.log('ERROR', `[PriceFetcherHelpers] BigQuery write failed: ${bqError.message}`);
                 // Continue - don't fail the entire fetch for BigQuery errors

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -90,10 +90,10 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
         let storageSuccess = false;
         let storageMethod = 'NONE';
 
-        // 1. Attempt BigQuery Write
+        // 1. Attempt BigQuery Write (using MERGE to prevent duplicates on re-runs)
         if (process.env.BIGQUERY_ENABLED !== 'false') {
             try {
-                const { insertRows, ensureInstrumentInsightsTable } = require('../../core/utils/bigquery_utils');
+                const { insertRowsWithMerge, ensureInstrumentInsightsTable } = require('../../core/utils/bigquery_utils');
                 await ensureInstrumentInsightsTable(logger);
                 
                 const fetchedAt = new Date().toISOString();
@@ -107,9 +107,10 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
                 });
                 
                 const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
+                // Use MERGE with dedup keys to prevent duplicates if fetch runs twice on same day
+                await insertRowsWithMerge(datasetId, 'instrument_insights', bigqueryRows, ['date', 'instrument_id'], logger);
                 
-                logger.log('INFO', `[FetchInsightsHelpers] ✅ Successfully stored ${bigqueryRows.length} insight records to BigQuery.`);
+                logger.log('INFO', `[FetchInsightsHelpers] ✅ Successfully stored ${bigqueryRows.length} insight records to BigQuery [MERGE].`);
                 storageSuccess = true;
                 storageMethod = 'BIGQUERY';
             } catch (bqError) {

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -124,19 +124,48 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
         const fetchedCids = new Set(data.Items.map(item => String(item.CustomerId)));
         const knownCids = Object.keys(bqMasterList);
         const missingCids = knownCids.filter(cid => !fetchedCids.has(cid));
+        
+        logger.log('INFO', `[PopularInvestorFetch] Comparison: API returned ${fetchedCids.size} PIs, Master List has ${knownCids.length} known PIs, ${missingCids.length} missing from API response`);
 
         if (missingCids.length > 0) {
+            // Log the missing users with their usernames for visibility
+            const missingUsernames = missingCids.map(cid => {
+                const userData = bqMasterList[cid];
+                return userData?.username || `unknown_${cid}`;
+            });
             logger.log('INFO', `[PopularInvestorFetch] Found ${missingCids.length} missing users. Fetching individually...`);
+            logger.log('INFO', `[PopularInvestorFetch] Missing users: ${missingUsernames.slice(0, 20).join(', ')}${missingCids.length > 20 ? ` ... and ${missingCids.length - 20} more` : ''}`);
+            
             const { header } = await headerManager.selectHeader();
             
-            for (const cid of missingCids) {
+            let successCount = 0;
+            let failCount = 0;
+            
+            for (let i = 0; i < missingCids.length; i++) {
+                const cid = missingCids[i];
+                const username = bqMasterList[cid]?.username || `unknown_${cid}`;
+                
+                // Log progress every 10 users or at start
+                if (i === 0 || (i + 1) % 10 === 0) {
+                    logger.log('INFO', `[PopularInvestorFetch] Fetching missing user ${i + 1}/${missingCids.length}: ${username} (CID: ${cid})`);
+                }
+                
                 const userData = await fetchIndividualUserRankings(cid, { ...header, 'Referer': 'https://www.etoro.com/' }, proxyManager, logger);
                 if (userData) {
                     data.Items.push(userData);
                     data.TotalRows++;
+                    successCount++;
+                    logger.log('TRACE', `[PopularInvestorFetch] ✅ Successfully fetched ${username} (CID: ${cid})`);
+                } else {
+                    failCount++;
+                    logger.log('WARN', `[PopularInvestorFetch] ❌ Failed to fetch ${username} (CID: ${cid})`);
                 }
                 await new Promise(r => setTimeout(r, 200)); // Rate limit
             }
+            
+            logger.log('INFO', `[PopularInvestorFetch] Missing user fetch complete: ${successCount} recovered, ${failCount} failed`);
+        } else {
+            logger.log('INFO', `[PopularInvestorFetch] No missing users - all ${knownCids.length} known PIs were returned by the API`);
         }
     } catch (e) {
         logger.log('WARN', `[PopularInvestorFetch] Failed to check master list: ${e.message}`);

package/functions/maintenance/backfill-pi-master-list-rankings/index.js

@@ -191,9 +191,10 @@ async function backfillRankings(startDate = null, endDate = null, logger = conso
                     };
                 });
                 
-                // Write to BigQuery using load jobs (free, batched)
+                // Write to BigQuery using MERGE (prevents duplicates if backfill is re-run)
                 const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                await insertRows(datasetId, 'pi_rankings', bigqueryRows, logger);
+                // Use MERGE with dedup keys to prevent duplicates on re-runs
+                await insertRowsWithMerge(datasetId, 'pi_rankings', bigqueryRows, ['date', 'pi_id'], logger);
                 
                 totalRows += bigqueryRows.length;
                 processedDates++;

package/functions/price-backfill/helpers/handler_helpers.js

@@ -68,16 +68,17 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
                     };
                 });
                 
-                // Write to BigQuery using load jobs (free, batched)
+                // Write to BigQuery using MERGE (prevents duplicates if backfill is re-run)
                 if (process.env.BIGQUERY_ENABLED !== 'false') {
                     try {
-                        const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                        const { insertRowsWithMerge, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
                         await ensureAssetPricesTable(logger);
                         
                         const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                        await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                        // Use MERGE with dedup keys to prevent duplicates on re-runs
+                        await insertRowsWithMerge(datasetId, 'asset_prices', bigqueryRows, ['date', 'instrument_id'], logger);
                         
-                        logger.log('TRACE', `[PriceBackfill] Successfully stored ${bigqueryRows.length} price records for ${ticker} (${instrumentId}) to BigQuery`);
+                        logger.log('TRACE', `[PriceBackfill] Successfully stored ${bigqueryRows.length} price records for ${ticker} (${instrumentId}) to BigQuery [MERGE]`);
                     } catch (bqError) {
                         logger.log('ERROR', `[PriceBackfill] BigQuery write failed for ${ticker} (${instrumentId}): ${bqError.message}`);
                         // Continue - don't fail the entire backfill for one instrument

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -14,9 +14,14 @@ const {
     ensurePortfolioSnapshotsTable, 
     ensureTradeHistorySnapshotsTable, 
     ensureSocialPostSnapshotsTable,
-    insertRows 
+    insertRowsWithMerge 
 } = require('../../core/utils/bigquery_utils');
 
+// Deduplication keys for BigQuery MERGE operations
+const PORTFOLIO_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+const HISTORY_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+const SOCIAL_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+
 /**
  * Store portfolio data for a signed-in user
  * @param {object} params - Parameters
@@ -47,9 +52,10 @@ async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid,
                 await bigqueryBatchManager.addPortfolioRow(row);
             } else {
                 // Direct write (fallback for when batch manager not available)
+                // Uses MERGE to prevent duplicates if user is processed twice on same day
                 await ensurePortfolioSnapshotsTable(logger);
-                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
-                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for signed-in user ${cid} (date: ${date})`);
+                await insertRowsWithMerge(datasetId, 'portfolio_snapshots', [row], PORTFOLIO_DEDUP_KEYS, logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for signed-in user ${cid} (date: ${date}) [MERGE]`);
             }
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
@@ -106,9 +112,10 @@ async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, c
                 await bigqueryBatchManager.addHistoryRow(row);
             } else {
                 // Direct write (fallback for when batch manager not available)
+                // Uses MERGE to prevent duplicates if user is processed twice on same day
                 await ensureTradeHistorySnapshotsTable(logger);
-                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
-                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for signed-in user ${cid} (date: ${date})`);
+                await insertRowsWithMerge(datasetId, 'trade_history_snapshots', [row], HISTORY_DEDUP_KEYS, logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for signed-in user ${cid} (date: ${date}) [MERGE]`);
             }
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
@@ -169,8 +176,9 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
                 fetched_at: new Date().toISOString()
             };
             
-            await insertRows(datasetId, 'social_post_snapshots', [row], logger);
-            logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for signed-in user ${cid} (date: ${date}, ${posts.length} posts)`);
+            // Uses MERGE to prevent duplicates if user is processed twice on same day
+            await insertRowsWithMerge(datasetId, 'social_post_snapshots', [row], SOCIAL_DEDUP_KEYS, logger);
+            logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for signed-in user ${cid} (date: ${date}, ${posts.length} posts) [MERGE]`);
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
             // Continue to Firestore write (fallback)
@@ -258,9 +266,10 @@ async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, c
                 await bigqueryBatchManager.addPortfolioRow(row);
             } else {
                 // Direct write (fallback for when batch manager not available)
+                // Uses MERGE to prevent duplicates if PI is processed twice on same day
                 await ensurePortfolioSnapshotsTable(logger);
-                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
-                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for PI ${cid} (date: ${date})`);
+                await insertRowsWithMerge(datasetId, 'portfolio_snapshots', [row], PORTFOLIO_DEDUP_KEYS, logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for PI ${cid} (date: ${date}) [MERGE]`);
             }
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
@@ -316,9 +325,10 @@ async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry
                 await bigqueryBatchManager.addHistoryRow(row);
             } else {
                 // Direct write (fallback for when batch manager not available)
+                // Uses MERGE to prevent duplicates if PI is processed twice on same day
                 await ensureTradeHistorySnapshotsTable(logger);
-                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
-                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for PI ${cid} (date: ${date})`);
+                await insertRowsWithMerge(datasetId, 'trade_history_snapshots', [row], HISTORY_DEDUP_KEYS, logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for PI ${cid} (date: ${date}) [MERGE]`);
             }
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
@@ -382,9 +392,10 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
                 await bigqueryBatchManager.addSocialRow(row);
             } else {
                 // Direct write (fallback for when batch manager not available)
+                // Uses MERGE to prevent duplicates if PI is processed twice on same day
                 await ensureSocialPostSnapshotsTable(logger);
-                await insertRows(datasetId, 'social_post_snapshots', [row], logger);
-                logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for PI ${cid} (date: ${date}, ${posts.length} posts)`);
+                await insertRowsWithMerge(datasetId, 'social_post_snapshots', [row], SOCIAL_DEDUP_KEYS, logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for PI ${cid} (date: ${date}, ${posts.length} posts) [MERGE]`);
             }
         } catch (bqError) {
             logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);

package/functions/task-engine/utils/bigquery_batch_manager.js

@@ -1,27 +1,30 @@
 /**
  * @fileoverview BigQuery Batch Manager for Task Engine
  * 
- * Batches BigQuery writes and flushes them using LOAD JOBS (FREE).
+ * Batches BigQuery writes and flushes them using MERGE operations.
  * Integrates with FirestoreBatchManager to flush together.
  * 
- * Lessons learned from backfill script:
- * - Use createLoadJob() not table.load()
- * - Use temp files for load jobs
- * - Wait for job completion with getMetadata() polling
- * - Handle errors gracefully
- * - Clean up temp files properly
+ * UPDATED: Now uses insertRowsWithMerge to prevent duplicate rows
+ * when a user is processed twice on the same day.
+ * 
+ * Deduplication keys:
+ * - portfolio_snapshots: ['date', 'user_id', 'user_type']
+ * - trade_history_snapshots: ['date', 'user_id', 'user_type']
+ * - social_post_snapshots: ['date', 'user_id', 'user_type']
  */
 
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
 const { 
-    getOrCreateDataset,
     ensurePortfolioSnapshotsTable,
     ensureTradeHistorySnapshotsTable,
-    ensureSocialPostSnapshotsTable
+    ensureSocialPostSnapshotsTable,
+    insertRowsWithMerge
 } = require('../../core/utils/bigquery_utils');
 
+// Deduplication keys for each table type
+const PORTFOLIO_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+const HISTORY_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+const SOCIAL_DEDUP_KEYS = ['date', 'user_id', 'user_type'];
+
 class BigQueryBatchManager {
     constructor(logger) {
         this.logger = logger;
@@ -74,83 +77,31 @@ class BigQueryBatchManager {
     }
     
     /**
-     * Flush a buffer to BigQuery using load job
-     * Uses lessons from backfill: createLoadJob, temp files, proper polling
+     * Flush a buffer to BigQuery using MERGE operation
+     * Uses insertRowsWithMerge to prevent duplicates when a user is processed twice on same day
      */
-    async _flushBuffer(buffer, tableId, tableName) {
+    async _flushBuffer(buffer, tableId, tableName, dedupKeys) {
         if (buffer.length === 0) return 0;
         
         const rows = [...buffer]; // Copy buffer
         buffer.length = 0; // Clear buffer
         
         try {
-            const dataset = await getOrCreateDataset(this.datasetId, this.logger);
-            const table = dataset.table(tableId);
-            
-            // Write to temporary file (load jobs require a file, not in-memory data)
-            const tempFile = path.join(os.tmpdir(), `bigquery_${tableId}_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
-            const ndjson = rows.map(r => JSON.stringify(r)).join('\n');
+            // Use insertRowsWithMerge for deduplication
+            // This prevents duplicate rows if a user is processed twice on the same day
+            const rowsInserted = await insertRowsWithMerge(
+                this.datasetId, 
+                tableId, 
+                rows, 
+                dedupKeys, 
+                this.logger
+            );
             
-            try {
-                fs.writeFileSync(tempFile, ndjson, 'utf8');
-                
-                // Create load job (FREE) - using createLoadJob as learned from backfill
-                const [job] = await table.createLoadJob(tempFile, {
-                    sourceFormat: 'NEWLINE_DELIMITED_JSON',
-                    writeDisposition: 'WRITE_APPEND',
-                    autodetect: false // Use existing table schema
-                });
-                
-                // Wait for job to complete using polling (as learned from backfill)
-                let jobMetadata;
-                const maxAttempts = 60; // 5 minutes max (5 second intervals)
-                const pollInterval = 5000; // 5 seconds
-                
-                for (let attempt = 0; attempt < maxAttempts; attempt++) {
-                    [jobMetadata] = await job.getMetadata();
-                    const state = jobMetadata.status?.state;
-                    
-                    if (state === 'DONE') {
-                        break;
-                    }
-                    
-                    if (state === 'PENDING' || state === 'RUNNING') {
-                        // Wait before next poll
-                        await new Promise(resolve => setTimeout(resolve, pollInterval));
-                    } else {
-                        throw new Error(`Unexpected job state: ${state}`);
-                    }
-                }
-                
-                // Check if we timed out
-                if (jobMetadata.status?.state !== 'DONE') {
-                    throw new Error(`Load job did not complete within ${maxAttempts * pollInterval / 1000} seconds`);
-                }
-                
-                // Check for errors
-                if (jobMetadata.status?.errorResult) {
-                    throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
-                }
-                
-                const rowsLoaded = jobMetadata.statistics?.load?.outputRows || rows.length;
-                
-                if (this.logger) {
-                    this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rowsLoaded} ${tableName} rows to BigQuery using LOAD JOB (free)`);
-                }
-                
-                return rowsLoaded;
-            } finally {
-                // Clean up temp file (as learned from backfill)
-                try {
-                    if (fs.existsSync(tempFile)) {
-                        fs.unlinkSync(tempFile);
-                    }
-                } catch (cleanupError) {
-                    if (this.logger) {
-                        this.logger.log('WARN', `[BigQueryBatch] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
-                    }
-                }
+            if (this.logger) {
+                this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rows.length} ${tableName} rows to BigQuery using MERGE (${rowsInserted} new, ${rows.length - rowsInserted} updated)`);
             }
+            
+            return rows.length;
         } catch (error) {
             // Log error but don't throw - allow Firestore writes to continue
             if (this.logger) {
@@ -172,9 +123,9 @@ class BigQueryBatchManager {
         }
         
         const results = await Promise.allSettled([
-            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio'),
-            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history'),
-            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social')
+            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio', PORTFOLIO_DEDUP_KEYS),
+            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history', HISTORY_DEDUP_KEYS),
+            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social', SOCIAL_DEDUP_KEYS)
         ]);
         
         const totalFlushed = results

package/index.js

@@ -11,7 +11,6 @@ const { FirestoreBatchManager }    = require('./functions/task-engine/utils/fire
 const firestoreUtils               = require('./functions/core/utils/firestore_utils');
 
 // Orchestrator
-// [UPDATED] Imported handleOrchestratorHttp
 const { 
     runDiscoveryOrchestrator, 
     runUpdateOrchestrator, 
@@ -47,19 +46,15 @@ const dataLoader                                          = require('./functions
 const computationUtils                                    = require('./functions/computation-system/utils/utils');
 
 
-// API v2 (CommonJS)
 const { createApiV2App } = require('./functions/api-v2/index');
 
-// Maintenance & Data Acquisition
 const { fetchAndStoreInsights }                           = require('./functions/fetch-insights/helpers/handler_helpers');
 const { fetchAndStorePrices }                             = require('./functions/etoro-price-fetcher/helpers/handler_helpers');
 const { runSocialOrchestrator }                           = require('./functions/social-orchestrator/helpers/orchestrator_helpers');
 const { handleSocialTask }                                = require('./functions/social-task-handler/helpers/handler_helpers');
 const { runBackfillAssetPrices }                          = require('./functions/price-backfill/helpers/handler_helpers');
 const { runRootDataIndexer }                              = require('./functions/root-data-indexer/index');
-// [NEW] Popular Investor Fetcher
 const { runPopularInvestorFetch }                         = require('./functions/fetch-popular-investors/index');
-// [NEW] Backfill Task Engine Data
 const { backfillTaskEngineData }                          = require('./functions/maintenance/backfill-task-engine-data/index');
 const { backfillPIMasterListRankings }                    = require('./functions/maintenance/backfill-pi-master-list-rankings/index');
 const { backfillInstrumentInsights }                      = require('./functions/maintenance/backfill-instrument-insights/index');
@@ -86,7 +81,6 @@ const core = {
 };
 
 const orchestrator = {
-  // [UPDATED] Exported handleOrchestratorHttp so it can be mapped in Cloud Functions
   handleOrchestratorHttp,
   runDiscoveryOrchestrator,
   runUpdateOrchestrator,
@@ -122,21 +116,17 @@ const computationSystem = {
 };
 
 const api = {
-  createApiV2App, // New API v2 entry point
+  createApiV2App, 
 };
 
 const maintenance = {
-  runSpeculatorCleanup: runCleanup,
-  handleInvalidSpeculator,
   runFetchInsights: fetchAndStoreInsights,
   runFetchPrices: fetchAndStorePrices,
   runSocialOrchestrator,
   handleSocialTask,
   runBackfillAssetPrices,
   runRootDataIndexer,
-  // [NEW] Added to maintenance pipe
   runPopularInvestorFetch,
-  // [NEW] BigQuery backfills
   backfillTaskEngineData,
   backfillPIMasterListRankings,
   backfillInstrumentInsights,
@@ -151,8 +141,8 @@ const maintenance = {
 const proxy = { handlePost };
 
 const alertSystem = {
-  handleAlertTrigger, // Pub/Sub handler (kept for backward compatibility, but not used)
-  handleComputationResultWrite, // Firestore trigger handler - main entry point
+  handleAlertTrigger, 
+  handleComputationResultWrite,
   checkAndSendAllClearNotifications
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.723",
+  "version": "1.0.725",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [