bulltrackers-module 1.0.167 → 1.0.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -82,30 +82,63 @@ class IntelligentProxyManager {
82
82
  }
83
83
 
84
84
  /**
85
- * --- MODIFIED: Makes a fetch request with exponential backoff for rate limits ---
85
+ * --- CORRECTED LOGIC: Makes a fetch request by trying different proxies ---
86
86
  * @param {string} targetUrl - The URL to fetch.
87
87
  * @param {object} options - Fetch options (e.g., headers).
88
88
  * @returns {Promise<object>} A mock Response object.
89
89
  */
90
90
  async fetch(targetUrl, options = {}) {
91
- let proxy = null;
92
- try { proxy = await this._selectProxy(); } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
93
- let backoff = this.INITIAL_BACKOFF_MS;
94
91
  let lastResponse = null;
92
+
93
+ // Use MAX_RETRIES to define the number of *different proxies* we will try
94
+ // before giving up on the request.
95
95
  for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
96
+ let proxy;
97
+ try {
98
+ // 1. Select a new, UNLOCKED proxy *inside* the loop.
99
+ proxy = await this._selectProxy();
100
+ } catch (error) {
101
+ // This fails if all proxies are locked.
102
+ this.logger.log('ERROR', '[ProxyManager] fetch failed: All proxies are locked.', { url: targetUrl });
103
+ return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
104
+ }
105
+
106
+ // 2. Make a SINGLE attempt with this selected proxy.
96
107
  const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
97
- lastResponse = response;
98
- // 1. Success
99
- if (response.ok) { return response; }
100
- // 2. Rate Limit Error (Retryable)
101
- if (response.isRateLimitError) { this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl }); await sleep(backoff); backoff *= 2; continue; }
102
- // 3. Other Fetch Error (Non-Retryable, Lock Proxy)
103
- if (response.isUrlFetchError) { this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status }); await this.lockProxy(proxy.owner); return response; }
104
- // 4. Standard Error (e.g., 404, 500 from *target* URL, not proxy)
105
- return response; }
106
- // If loop finishes, all retries failed (likely all were rate-limit errors)
107
- this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} rate-limit retries.`, { url: targetUrl });
108
- return lastResponse;
108
+ lastResponse = response; // Save this response in case it's the last one
109
+
110
+ // 3. Case 1: Success! Return immediately.
111
+ if (response.ok) {
112
+ return response;
113
+ }
114
+
115
+ // 4. Case 2: Proxy-level error (Rate limit OR other fetch error)
116
+ // This is a *proxy* failure, not a *target* failure.
117
+ if (response.isUrlFetchError) {
118
+ this.logger.log('WARN', `[ProxyManager] Proxy ${proxy.owner} failed (Attempt ${attempt}/${this.MAX_RETRIES}). Locking it and trying a new proxy.`, {
119
+ reason: response.isRateLimitError ? "Rate Limit" : "Other Fetch Error",
120
+ status: response.status,
121
+ url: targetUrl
122
+ });
123
+
124
+ // LOCK THE FAILED PROXY so _selectProxy() won't pick it again.
125
+ await this.lockProxy(proxy.owner);
126
+
127
+ // Back off slightly before trying the *next* proxy to avoid a thundering herd.
128
+ await sleep(this.INITIAL_BACKOFF_MS * attempt);
129
+
130
+ continue; // Go to the next loop iteration to select a *new* proxy.
131
+ }
132
+
133
+ // 5. Case 3: Standard error from *target* URL (e.g., 404, 500)
134
+ // This means the proxy *worked* but the target URL is bad.
135
+ // This is a "successful" fetch. Do not retry, just return the response.
136
+ return response;
137
+ }
138
+
139
+ // 6. If loop finishes, all (this.MAX_RETRIES) proxy attempts failed.
140
+ this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} proxy attempts.`, { url: targetUrl, lastStatus: lastResponse?.status });
141
+ return lastResponse; // Return the last failed response
109
142
  }
110
143
 
111
144
 
@@ -1,7 +1,14 @@
1
+ /*
2
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
3
+ * (REFACTORED: Now includes node-fetch fallback)
4
+ * (REFACTORED: Added verbose, traceable logging)
5
+ */
6
+
1
7
  /**
2
8
  * @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
3
9
  * REFACTORED: Now stateless and receives dependencies.
4
10
  * --- MODIFIED: Passes username to the 'verify' task. ---
11
+ * --- MODIFIED: Added node-fetch fallback and verbose logging. ---
5
12
  */
6
13
 
7
14
  /**
@@ -15,56 +22,164 @@ async function handleDiscover(task, taskId, dependencies, config) {
15
22
  const { logger, headerManager, proxyManager, pubsub, batchManager } = dependencies;
16
23
  const { cids, blockId, instrument, userType } = task;
17
24
  const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
25
+
18
26
  const selectedHeader = await headerManager.selectHeader();
19
- if (!selectedHeader) throw new Error("Could not select a header.");
27
+ if (!selectedHeader) {
28
+ logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
29
+ throw new Error("Could not select a header.");
30
+ }
31
+
20
32
  let wasSuccess = false;
33
+ let proxyUsed = true;
34
+ const logPrefix = `[DISCOVER/${taskId}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
35
+
36
+ try { // Outer try for the whole operation
37
+ let response;
38
+ const options = {
39
+ method: 'POST',
40
+ headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
41
+ body: JSON.stringify(cids),
42
+ };
43
+
44
+ logger.log('INFO', `${logPrefix} Starting discovery for ${cids.length} CIDs. Block: ${blockId}, Type: ${userType}`);
45
+
46
+ try {
47
+ // --- REFACTOR 3: ADD FALLBACK ---
48
+ logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`);
49
+ response = await proxyManager.fetch(url, options);
50
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
51
+
52
+ } catch (proxyError) {
53
+ logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
54
+ error: proxyError.message,
55
+ source: 'AppScript'
56
+ });
57
+ proxyUsed = false;
58
+
59
+ try {
60
+ response = await fetch(url, options); // Direct node-fetch
61
+ if (!response.ok) {
62
+ const errorText = await response.text();
63
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
64
+ }
65
+
66
+ } catch (fallbackError) {
67
+ logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
68
+ error: fallbackError.message,
69
+ source: 'eToro/Network'
70
+ });
71
+ throw fallbackError; // Throw to be caught by outer try
72
+ }
73
+ // --- END REFACTOR 3 ---
74
+ }
75
+
76
+ // --- If we are here, `response` is valid ---
77
+
78
+ // Step 1. Discover Speculators
79
+ if (userType === 'speculator') {
80
+ batchManager.addProcessedSpeculatorCids(cids);
81
+ logger.log('INFO', `${logPrefix} Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);
82
+ }
83
+
84
+ const body = await response.text();
85
+ let publicUsers;
86
+ try {
87
+ // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
88
+ publicUsers = JSON.parse(body);
89
+ } catch (parseError) {
90
+ logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
91
+ parseErrorMessage: parseError.message,
92
+ rawResponseText: body
93
+ });
94
+ throw new Error(`Failed to parse JSON response from discovery API. Body: ${body.substring(0, 200)}`);
95
+ }
96
+ // --- END REFACTOR 4 ---
97
+
98
+ if (!Array.isArray(publicUsers)) {
99
+ logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`);
100
+ wasSuccess = true; // API call worked, data was just empty/weird
101
+ return;
102
+ }
103
+
104
+ wasSuccess = true; // Mark as success *after* parsing
105
+ logger.log('INFO', `${logPrefix} API call successful, found ${publicUsers.length} public users.`);
21
106
 
22
- // Step 1. Discover Speculators
23
- try {if (userType === 'speculator') {batchManager.addProcessedSpeculatorCids(cids);logger.log('INFO', `[DISCOVER] Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);}
24
- const response = await proxyManager.fetch(url, {method: 'POST',headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },body: JSON.stringify(cids),});
25
- if (!response.ok) {throw new Error(`API status ${response.status}`);}
26
- wasSuccess = true;
27
- const publicUsers = await response.json();
28
- if (!Array.isArray(publicUsers)) return;
29
107
  const oneMonthAgo = new Date();
30
108
  oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
31
- // Step 2. We know a user is "active" if their risk score is NOT 0 and they logged in, in the last month. This covers the edge case that a user IS active but does NOT hold positions (cannot hold no positions and have 0 risk score)
32
- const preliminaryActiveUsers = publicUsers.filter(user => new Date(user.Value.LastActivity) > oneMonthAgo &&user.Value.DailyGain !== 0 &&user.Value.Exposure !== 0 &&user.Value.RiskScore !== 0);
109
+
110
+ // Step 2. Filter active users
111
+ const preliminaryActiveUsers = publicUsers.filter(user =>
112
+ new Date(user.Value.LastActivity) > oneMonthAgo &&
113
+ user.Value.DailyGain !== 0 &&
114
+ user.Value.Exposure !== 0 &&
115
+ user.Value.RiskScore !== 0
116
+ );
117
+ logger.log('INFO', `${logPrefix} Found ${preliminaryActiveUsers.length} preliminary active users.`);
118
+
33
119
  let finalActiveUsers = [];
34
120
  const invalidCidsToLog = [];
35
- // Step 3. Users that were passed INTO the discovery API but did not come OUT OF the discovery API, are private accounts, the API filters this for us, so mark them as such to avoid processing them again.
36
- if (userType === 'speculator') {const publicUserCids = new Set(publicUsers.map(u => u.CID));
37
- if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));invalidCidsToLog.push(...privateUserCids);}
121
+
122
+ // Step 3. Log private/inactive users
123
+ if (userType === 'speculator') {
124
+ const publicUserCids = new Set(publicUsers.map(u => u.CID));
125
+ if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {
126
+ const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));
127
+ invalidCidsToLog.push(...privateUserCids);
128
+ logger.log('INFO', `${logPrefix} Found ${privateUserCids.length} private users (in request but not response).`);
129
+ }
38
130
  const activeUserCids = new Set(preliminaryActiveUsers.map(u => u.CID));
39
131
  const inactiveUserCids = publicUsers.filter(u => !activeUserCids.has(u.CID)).map(u => u.CID);
40
132
  invalidCidsToLog.push(...inactiveUserCids);
41
- logger.log('INFO', `[DISCOVER] Applying new speculator pre-filter to ${preliminaryActiveUsers.length} active users.`);
133
+ logger.log('INFO', `${logPrefix} Found ${inactiveUserCids.length} inactive users.`);
134
+
135
+ // Step 4. Apply speculator heuristic
42
136
  const nonSpeculatorCids = [];
43
- // Step 4. For the remaining active users, apply heuristic filters to weed out non-speculators.
44
137
  for (const user of preliminaryActiveUsers) {
45
138
  const v = user.Value;
46
139
  const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
47
- const isLikelySpeculator = ((v.Trades || 0) > 500 ||(v.TotalTradedInstruments || 0) > 50 ||totalLeverage > 50 ||(v.WeeklyDD || 0) < -25);
48
- if (isLikelySpeculator) {finalActiveUsers.push(user);} else {nonSpeculatorCids.push(user.CID);}}
140
+ const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25);
141
+
142
+ if (isLikelySpeculator) {
143
+ finalActiveUsers.push(user);
144
+ } else {
145
+ nonSpeculatorCids.push(user.CID);
146
+ }
147
+ }
49
148
  invalidCidsToLog.push(...nonSpeculatorCids);
50
- logger.log('INFO', `[DISCOVER] Pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
149
+ logger.log('INFO', `${logPrefix} Speculator pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
150
+
51
151
  if (invalidCidsToLog.length > 0) {
52
152
  await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
53
- logger.log('INFO', `[DISCOVER] Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
153
+ logger.log('INFO', `${logPrefix} Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
54
154
  }
55
- // Step 5. Remaining users are public active non speculators
155
+ // Step 5. Non-speculators are just active users
56
156
  } else {
57
157
  finalActiveUsers = preliminaryActiveUsers;
58
158
  }
59
- // Step 6. Publish 'verify' task for all active users found
159
+
160
+ // Step 6. Publish 'verify' task
60
161
  if (finalActiveUsers.length > 0) {
61
- const verificationTask = {type: 'verify',users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze,username: u.Value.UserName})), blockId, instrument, userType};
162
+ const verificationTask = {
163
+ type: 'verify',
164
+ users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze, username: u.Value.UserName })),
165
+ blockId,
166
+ instrument,
167
+ userType
168
+ };
62
169
  await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
63
- logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify({ tasks: [verificationTask] })} `);
64
- logger.log('INFO', `[DISCOVER] Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
170
+ logger.log('SUCCESS', `${logPrefix} Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
171
+ } else {
172
+ logger.log('INFO', `${logPrefix} No active users found to verify.`);
65
173
  }
174
+
175
+ } catch (err) {
176
+ logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack });
177
+ wasSuccess = false; // Ensure it's marked as failure
66
178
  } finally {
67
- if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
179
+ if (selectedHeader && proxyUsed) {
180
+ // Only update performance if the proxy was used
181
+ headerManager.updatePerformance(selectedHeader.id, wasSuccess);
182
+ }
68
183
  }
69
184
  }
70
185
 
@@ -1,31 +1,24 @@
1
1
  /*
2
2
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
3
- * (MODIFIED: To conditionally fetch history API once per user per batch)
4
- * (MODIFIED: `lookupUsernames` runs batches in parallel)
5
- * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
6
- * (MODIFIED: `handleUpdate` now uses batchManager for history cache)
3
+ * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
4
+ * (REFACTORED: Added node-fetch fallback for all API calls)
5
+ * (REFACTORED: Added verbose, user-centric logging for all operations)
7
6
  */
8
7
 
9
- /**
10
- * @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
11
- * REFACTORED: Now stateless and receives dependencies.
12
- * OPTIMIZED: Removed immediate batch commit for speculator timestamp fix.
13
- * --- MODIFIED: Fetches portfolio AND trade history in parallel. ---
14
- * --- MODIFIED: Includes helper to look up usernames from CIDs. ---
15
- * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
16
- */
17
8
  const { FieldValue } = require('@google-cloud/firestore');
18
9
  const pLimit = require('p-limit');
19
10
 
20
11
  /**
21
- * (MODIFIED: Runs lookup batches in parallel)
12
+ * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
22
13
  */
23
14
  async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
24
15
  if (!cids?.length) return [];
25
16
  logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
26
17
 
27
- // Use a new config value, falling back to 5
28
- const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
18
+ // --- REFACTOR 1: REMOVE CONCURRENCY ---
19
+ const limit = pLimit(1);
20
+ // --- END REFACTOR 1 ---
21
+
29
22
  const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
30
23
 
31
24
  const batches = [];
@@ -33,26 +26,67 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
33
26
  batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
34
27
  }
35
28
 
36
- const batchPromises = batches.map(batch => limit(async () => {
29
+ const batchPromises = batches.map((batch, index) => limit(async () => {
30
+ const batchId = `batch-${index + 1}`;
31
+ logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
32
+
37
33
  const header = await headerManager.selectHeader();
38
34
  if (!header) {
39
- logger.log('ERROR', '[lookupUsernames] Could not select a header.');
40
- return null; // Return null to skip this batch
35
+ logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`);
36
+ return null;
41
37
  }
42
38
 
43
- let success = false;
39
+ let wasSuccess = false;
40
+ let proxyUsed = true;
41
+ let response;
42
+ const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
43
+ const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
44
+
44
45
  try {
45
- const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
46
- if (!res.ok) throw new Error(`API status ${res.status}`);
47
- const data = await res.json();
48
- success = true;
49
- logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); // Log only a few
50
- return data; // Return data on success
51
- } catch (err) {
52
- logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
53
- return null; // Return null on failure
46
+ // --- REFACTOR 3: ADD FALLBACK ---
47
+ logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
48
+ response = await proxyManager.fetch(url, options);
49
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
50
+
51
+ wasSuccess = true;
52
+ logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
53
+
54
+ } catch (proxyError) {
55
+ logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
56
+ error: proxyError.message,
57
+ source: 'AppScript'
58
+ });
59
+
60
+ proxyUsed = false; // Don't penalize header for proxy failure
61
+
62
+ try {
63
+ response = await fetch(url, options); // Direct node-fetch
64
+ if (!response.ok) {
65
+ const errorText = await response.text();
66
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
67
+ }
68
+ logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
69
+
70
+ } catch (fallbackError) {
71
+ logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, {
72
+ error: fallbackError.message,
73
+ source: 'eToro/Network'
74
+ });
75
+ return null; // Give up on this batch
76
+ }
77
+ // --- END REFACTOR 3 ---
54
78
  } finally {
55
- headerManager.updatePerformance(header.id, success);
79
+ if (proxyUsed) {
80
+ headerManager.updatePerformance(header.id, wasSuccess);
81
+ }
82
+ }
83
+
84
+ try {
85
+ const data = await response.json();
86
+ return data;
87
+ } catch (parseError) {
88
+ logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
89
+ return null;
56
90
  }
57
91
  }));
58
92
 
@@ -60,7 +94,7 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
60
94
 
61
95
  const allUsers = results
62
96
  .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
63
- .flatMap(r => r.value); // Flatten all successful batch results
97
+ .flatMap(r => r.value);
64
98
 
65
99
  logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
66
100
  return allUsers;
@@ -68,128 +102,197 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
68
102
 
69
103
 
70
104
  /**
71
- * (MODIFIED: Fetches history and all portfolios in parallel)
72
- * (MODIFIED: Uses batchManager for history cache)
105
+ * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
73
106
  */
74
- async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) { // <--- REMOVED historyFetchedForUser
107
+ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
75
108
  const { userId, instruments, instrumentId, userType } = task;
76
109
  const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined];
77
110
  const today = new Date().toISOString().slice(0, 10);
78
111
  const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
112
+ let isPrivate = false;
79
113
 
114
+ // --- REFACTOR 2: ADD VERBOSE LOGGING (with User ID) ---
115
+ logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
116
+
117
+ // --- 1. Process History Fetch (Sequentially) ---
80
118
  let historyHeader = null;
81
119
  let wasHistorySuccess = false;
82
- let historyFetchPromise = null;
83
- let isPrivate = false;
120
+ let proxyUsedForHistory = true;
84
121
 
85
122
  try {
86
- // --- 1. Prepare History Fetch (if needed) ---
87
- // (MODIFIED: Use batchManager's cross-invocation cache)
88
123
  if (!batchManager.checkAndSetHistoryFetched(userId)) {
89
- // This user has NOT been fetched in the last 10 mins (by this instance)
124
+ logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
90
125
  historyHeader = await headerManager.selectHeader();
91
- if (historyHeader) {
92
- // No need to add to a local set, batchManager did it.
93
- const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
94
- historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header });
126
+ if (!historyHeader) {
127
+ logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
95
128
  } else {
96
- logger.log('WARN', `[handleUpdate] Could not select history header for ${userId}. History will be skipped for this task.`);
129
+ const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
130
+ const options = { headers: historyHeader.header };
131
+ let response;
132
+
133
+ try {
134
+ // --- REFACTOR 3: ADD FALLBACK ---
135
+ logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
136
+ response = await proxyManager.fetch(historyUrl, options);
137
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
138
+ wasHistorySuccess = true;
139
+
140
+ } catch (proxyError) {
141
+ logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
142
+ error: proxyError.message,
143
+ source: 'AppScript'
144
+ });
145
+ proxyUsedForHistory = false;
146
+
147
+ try {
148
+ response = await fetch(historyUrl, options); // Direct node-fetch
149
+ if (!response.ok) {
150
+ const errorText = await response.text();
151
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
152
+ }
153
+ wasHistorySuccess = true; // Fallback succeeded
154
+
155
+ } catch (fallbackError) {
156
+ logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, {
157
+ error: fallbackError.message,
158
+ source: 'eToro/Network'
159
+ });
160
+ wasHistorySuccess = false;
161
+ }
162
+ // --- END REFACTOR 3 ---
163
+ }
164
+
165
+ if (wasHistorySuccess) {
166
+ logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
167
+ const data = await response.json();
168
+ await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
169
+ }
97
170
  }
98
171
  } else {
99
- logger.log('TRACE', `[handleUpdate] History fetch for ${userId} skipped (already fetched by this instance).`);
172
+ logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
173
+ }
174
+ } catch (err) {
175
+ logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
176
+ wasHistorySuccess = false;
177
+ } finally {
178
+ if (historyHeader && proxyUsedForHistory) {
179
+ headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
180
+ }
181
+ }
182
+
183
+ // --- 2. Process Portfolio Fetches (Sequentially) ---
184
+ logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
185
+
186
+ for (const instId of instrumentsToProcess) {
187
+ if (isPrivate) {
188
+ logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
189
+ break;
100
190
  }
101
191
 
102
- // --- 2. Prepare All Portfolio Fetches ---
103
- const portfolioRequests = [];
104
- for (const instId of instrumentsToProcess) {
105
- const portfolioHeader = await headerManager.selectHeader();
106
- if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
107
-
108
- const portfolioUrl = userType === 'speculator'
109
- ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
110
- : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
111
-
112
- portfolioRequests.push({
113
- instrumentId: instId,
114
- url: portfolioUrl,
115
- header: portfolioHeader,
116
- promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header })
117
- });
192
+ const portfolioHeader = await headerManager.selectHeader();
193
+ if (!portfolioHeader) {
194
+ logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
195
+ continue;
118
196
  }
197
+
198
+ const portfolioUrl = userType === 'speculator'
199
+ ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
200
+ : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
119
201
 
120
- // --- 3. Execute All API Calls in Parallel ---
121
- const allPromises = [
122
- ...(historyFetchPromise ? [historyFetchPromise] : []),
123
- ...portfolioRequests.map(r => r.promise)
124
- ];
125
- const allResults = await Promise.allSettled(allPromises);
126
-
127
- // --- 4. Process History Result ---
128
- let resultIndex = 0;
129
- if (historyFetchPromise) {
130
- const historyRes = allResults[resultIndex++];
131
- if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
132
- const data = await historyRes.value.json();
133
- wasHistorySuccess = true;
134
- await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
135
- } else {
136
- logger.log('WARN', `[handleUpdate] History fetch failed for ${userId}`, { error: historyRes.reason || `status ${historyRes.value?.status}` });
137
- }
138
- }
202
+ const options = { headers: portfolioHeader.header };
203
+ let response;
204
+ let wasPortfolioSuccess = false;
205
+ let proxyUsedForPortfolio = true;
139
206
 
140
- // --- 5. Process Portfolio Results ---
141
- for (let i = 0; i < portfolioRequests.length; i++) {
142
- const requestInfo = portfolioRequests[i];
143
- const portfolioRes = allResults[resultIndex++];
144
- let wasPortfolioSuccess = false;
145
-
146
- if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
147
- const body = await portfolioRes.value.text();
148
- if (body.includes("user is PRIVATE")) {
149
- isPrivate = true;
150
- logger.log('WARN', `User ${userId} is private. Removing from updates.`);
151
- break; // Stop processing more portfolios for this private user
152
- } else {
153
- wasPortfolioSuccess = true;
154
- await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId);
207
+ try {
208
+ // --- REFACTOR 3: ADD FALLBACK ---
209
+ logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
210
+ response = await proxyManager.fetch(portfolioUrl, options);
211
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
212
+ wasPortfolioSuccess = true;
213
+
214
+ } catch (proxyError) {
215
+ logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
216
+ error: proxyError.message,
217
+ source: 'AppScript'
218
+ });
219
+ proxyUsedForPortfolio = false;
220
+
221
+ try {
222
+ response = await fetch(portfolioUrl, options); // Direct node-fetch
223
+ if (!response.ok) {
224
+ const errorText = await response.text();
225
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
155
226
  }
156
- logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
157
- logger.log('DEBUG', 'Response returned ', { body } , 'for user' , { userId })
158
- } else {
159
- logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` });
227
+ wasPortfolioSuccess = true; // Fallback succeeded
228
+
229
+ } catch (fallbackError) {
230
+ logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, {
231
+ error: fallbackError.message,
232
+ source: 'eToro/Network'
233
+ });
234
+ wasPortfolioSuccess = false;
160
235
  }
161
- // Update performance for this specific header
162
- headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
236
+ // --- END REFACTOR 3 ---
163
237
  }
164
238
 
165
- // --- 6. Handle Private Users & Timestamps ---
166
- if (isPrivate) {
167
- logger.log('WARN', `User ${userId} is private. Removing from updates.`);
168
- for (const instrumentId of instrumentsToProcess) {
169
- await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
239
+ // --- 4. Process Portfolio Result (with verbose, raw logging) ---
240
+ if (wasPortfolioSuccess) {
241
+ const body = await response.text();
242
+ if (body.includes("user is PRIVATE")) {
243
+ isPrivate = true;
244
+ logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
245
+ break; // Stop processing more portfolios for this private user
170
246
  }
171
- const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
172
- for (const instrumentId of instrumentsToProcess) {
173
- const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
174
- // This is not batched, but it's a rare event.
175
- await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
247
+
248
+ try {
249
+ const portfolioJson = JSON.parse(body);
250
+ await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
251
+ logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`);
252
+
253
+ } catch (parseError) {
254
+ // --- REFACTOR 4: RETURN EXACT PAGE RESPONSE ---
255
+ wasPortfolioSuccess = false; // Mark as failure
256
+ logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
257
+ url: portfolioUrl,
258
+ parseErrorMessage: parseError.message,
259
+ rawResponseText: body // <--- THIS LOGS THE FULL HTML/ERROR RESPONSE
260
+ });
261
+ // --- END REFACTOR 4 ---
176
262
  }
177
- return; // Don't update timestamps
263
+ } else {
264
+ logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`);
265
+ }
266
+
267
+ if (proxyUsedForPortfolio) {
268
+ headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
178
269
  }
270
+ } // --- End of sequential portfolio loop ---
179
271
 
180
- // If not private, update all timestamps
272
+ // --- 5. Handle Private Users & Timestamps ---
273
+ if (isPrivate) {
274
+ logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
181
275
  for (const instrumentId of instrumentsToProcess) {
182
- await batchManager.updateUserTimestamp(userId, userType, instrumentId);
276
+ await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
183
277
  }
184
- if (userType === 'speculator') {
185
- await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
278
+ const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
279
+ for (const instrumentId of instrumentsToProcess) {
280
+ const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
281
+ await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
186
282
  }
283
+ return;
284
+ }
187
285
 
188
- } finally {
189
- if (historyHeader) { // historyHeader is only set if a fetch was attempted
190
- headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
191
- }
286
+ // If not private, update all timestamps
287
+ for (const instrumentId of instrumentsToProcess) {
288
+ await batchManager.updateUserTimestamp(userId, userType, instrumentId);
289
+ }
290
+ if (userType === 'speculator') {
291
+ await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
192
292
  }
293
+
294
+ logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
295
+ // 'finally' block for header flushing is handled by the main handler_creator.js
193
296
  }
194
297
 
195
298
  module.exports = { handleUpdate, lookupUsernames };
@@ -1,28 +1,142 @@
1
+ /*
2
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/verify_helpers.js
3
+ * (REFACTORED: `handleVerify` now runs sequentially to prevent throttling)
4
+ * (REFACTORED: `fetchAndVerifyUser` now includes node-fetch fallback)
5
+ * (REFACTORED: Added verbose, user-centric logging for all operations)
6
+ */
1
7
  const { FieldValue } = require('@google-cloud/firestore');
2
8
 
3
- async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }) {
9
+ /**
10
+ * (REFACTORED: Now includes taskId for logging, and full fallback logic)
11
+ */
12
+ async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }, taskId) {
13
+ const cid = user.cid;
14
+ const logPrefix = `[VERIFY/${taskId}/${cid}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
15
+
4
16
  const selectedHeader = await headerManager.selectHeader();
5
- if (!selectedHeader) return null;
17
+ if (!selectedHeader) {
18
+ logger.log('WARN', `${logPrefix} Could not select a header. Skipping user.`);
19
+ return null;
20
+ }
21
+
6
22
  let wasSuccess = false;
7
- try { const res = await proxyManager.fetch(`${process.env.ETORO_API_PORTFOLIO_URL}?cid=${user.cid}`, { headers: selectedHeader.header }); if (!res.ok) return null;
23
+ let proxyUsed = true;
24
+
25
+ try { // Outer try for the whole operation
26
+ let response;
27
+ const url = `${process.env.ETORO_API_PORTFOLIO_URL}?cid=${cid}`;
28
+ const options = { headers: selectedHeader.header };
29
+
30
+ try {
31
+ // --- REFACTOR 3: ADD FALLBACK ---
32
+ logger.log('TRACE', `${logPrefix} Attempting portfolio fetch via AppScript proxy...`);
33
+ response = await proxyManager.fetch(url, options);
34
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
35
+
36
+ } catch (proxyError) {
37
+ logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
38
+ error: proxyError.message,
39
+ source: 'AppScript'
40
+ });
41
+ proxyUsed = false;
42
+
43
+ try {
44
+ response = await fetch(url, options); // Direct node-fetch
45
+ if (!response.ok) {
46
+ const errorText = await response.text();
47
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
48
+ }
49
+ // Fallback succeeded, but we don't set wasSuccess yet,
50
+ // as we still need to parse the body.
51
+
52
+ } catch (fallbackError) {
53
+ logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
54
+ error: fallbackError.message,
55
+ source: 'eToro/Network'
56
+ });
57
+ throw fallbackError; // Throw to be caught by outer try
58
+ }
59
+ // --- END REFACTOR 3 ---
60
+ }
61
+
62
+ // --- If we are here, `response` is valid ---
63
+ let portfolioData;
64
+ const body = await response.text();
65
+
66
+ try {
67
+ // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
68
+ portfolioData = JSON.parse(body);
69
+ } catch (parseError) {
70
+ logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
71
+ parseErrorMessage: parseError.message,
72
+ rawResponseText: body
73
+ });
74
+ throw new Error(`Failed to parse JSON for user ${cid}.`);
75
+ }
76
+ // --- END REFACTOR 4 ---
77
+
78
+ // --- Original logic ---
79
+ if (userType === 'speculator') {
80
+ const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID).filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
81
+ if (!instruments.length) {
82
+ logger.log('TRACE', `${logPrefix} Verified user, but not a speculator (no matching assets).`);
83
+ wasSuccess = true; // API call *worked*
84
+ return null;
85
+ }
86
+ logger.log('INFO', `${logPrefix} Verified as SPECULATOR.`);
87
+ wasSuccess = true;
88
+ return { type: 'speculator', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } };
89
+ }
90
+
91
+ logger.log('INFO', `${logPrefix} Verified as NORMAL user.`);
8
92
  wasSuccess = true;
9
- const portfolioData = await res.json();
10
- if (userType === 'speculator') { const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID) .filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
11
- if (!instruments.length) return null; return { type: 'speculator', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } }; }
12
- return { type: 'normal', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
13
- } catch (err) { logger.log('WARN', `[VERIFY] Error processing user ${user.cid}`, { errorMessage: err.message }); return null;
14
- } finally { if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess); }
93
+ return { type: 'normal', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
94
+
95
+ } catch (err) {
96
+ // This catches proxy, fallback, or parse errors
97
+ logger.log('WARN', `${logPrefix} Error processing user.`, { errorMessage: err.message });
98
+ wasSuccess = false; // Ensure it's marked as failure
99
+ return null;
100
+ } finally {
101
+ if (selectedHeader && proxyUsed) {
102
+ // Only update performance if the proxy was used
103
+ headerManager.updatePerformance(selectedHeader.id, wasSuccess);
104
+ }
105
+ }
15
106
  }
16
107
 
108
+ /**
109
+ * (REFACTORED: Now runs fetches sequentially)
110
+ */
17
111
  async function handleVerify(task, taskId, { db, logger, ...dependencies }, config) {
18
112
  const { users, blockId, instrument, userType } = task;
19
113
  const batch = db.batch();
20
114
  const speculatorUpdates = {}, normalUpdates = {}, bronzeStates = {}, usernameMap = {};
21
115
  const specSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
22
- const results = await Promise.allSettled(users.map(u => fetchAndVerifyUser(u, dependencies, { ...config, userType })));
116
+
117
+ // --- REFACTOR 1: REMOVE CONCURRENCY ---
118
+ logger.log('INFO', `[VERIFY/${taskId}] Starting sequential verification for ${users.length} users...`);
119
+ const results = [];
120
+ for (const user of users) {
121
+ // Await each user one by one
122
+ const result = await fetchAndVerifyUser(user, { db, logger, ...dependencies }, { ...config, userType }, taskId);
123
+ results.push(result); // Push the actual result (or null)
124
+ }
125
+ logger.log('INFO', `[VERIFY/${taskId}] Sequential verification complete.`);
126
+ // --- END REFACTOR 1 ---
127
+
23
128
  let validUserCount = 0;
24
129
  results.forEach(r => {
25
- if (r.status === 'fulfilled' && r.value) { const d = r.value; usernameMap[d.userId] = { username: d.username }; bronzeStates[d.userId] = d.isBronze; validUserCount++; if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData; else normalUpdates[`users.${d.userId}`] = d.updateData; } });
130
+ if (r) { // Only process non-null results
131
+ const d = r;
132
+ usernameMap[d.userId] = { username: d.username };
133
+ bronzeStates[d.userId] = d.isBronze;
134
+ validUserCount++;
135
+ if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData;
136
+ else normalUpdates[`users.${d.userId}`] = d.updateData;
137
+ }
138
+ });
139
+
26
140
  if (Object.keys(speculatorUpdates).length || Object.keys(normalUpdates).length) {
27
141
  const blockRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS : config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS).doc(String(blockId));
28
142
  batch.set(blockRef, userType === 'speculator' ? speculatorUpdates : normalUpdates, { merge: true });
@@ -35,9 +149,10 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
35
149
  for (const cid in usernameMap) { const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
36
150
  if (!shardedUpdates[shardId]) { shardedUpdates[shardId] = {}; } shardedUpdates[shardId][cid] = usernameMap[cid]; }
37
151
  for (const shardId in shardedUpdates) { const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId); batch.set(mapRef, shardedUpdates[shardId], { merge: true }); }
38
- logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
152
+ logger.log('INFO', `[VERIFY/${taskId}] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
153
+
39
154
  await batch.commit();
40
- if (validUserCount) logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);
155
+ if (validUserCount) logger.log('SUCCESS', `[VERIFY/${taskId}] Verified and stored ${validUserCount} new ${userType} users.`);
41
156
  }
42
157
 
43
158
  module.exports = { handleVerify };
@@ -1,9 +1,6 @@
1
1
  /*
2
2
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
3
- * (MODIFIED: To pass down a Set to track history fetches)
4
- * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
5
- * (MODIFIED: To use a SINGLE parallel work pool for ALL tasks)
6
- * (MODIFIED: To remove local history cache set)
3
+ * (REFACTORED: Concurrency limit set to 1 to prevent API throttling)
7
4
  */
8
5
 
9
6
  /**
@@ -15,7 +12,7 @@
15
12
  const { handleDiscover } = require('../helpers/discover_helpers');
16
13
  const { handleVerify } = require('../helpers/verify_helpers');
17
14
  const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
18
- const pLimit = require('p-limit'); // <--- IMPORT p-limit
15
+ const pLimit = require('p-limit');
19
16
 
20
17
  /**
21
18
  * Parses Pub/Sub message into task array.
@@ -55,15 +52,16 @@ async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config
55
52
 
56
53
  /**
57
54
  * Executes all tasks.
58
- * (MODIFIED: Runs ALL tasks in a single parallel pool)
55
+ * (REFACTORED: Concurrency limit set to 1)
59
56
  */
60
57
  async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
61
- const { logger, batchManager } = dependencies; // <--- Get batchManager
58
+ const { logger, batchManager } = dependencies;
62
59
 
63
- // REMOVED: const historyFetchedForUser = new Set();
60
+ // --- REFACTOR 1: REMOVE CONCURRENCY ---
61
+ // Set limit to 1 to serialize all tasks and prevent AppScript throttling.
62
+ const limit = pLimit(1);
63
+ // --- END REFACTOR 1 ---
64
64
 
65
- // Create one unified parallel pool
66
- const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 3); // TODO Work out what the optimal concurrency is
67
65
  const allTaskPromises = [];
68
66
  let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
69
67
 
@@ -94,8 +92,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
94
92
  const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
95
93
  allTaskPromises.push(
96
94
  limit(() =>
97
- // Pass batchManager instead of the local set
98
- handleUpdate(task, subTaskId, dependencies, config, username) // <--- REMOVED historyFetchedForUser
95
+ handleUpdate(task, subTaskId, dependencies, config, username)
99
96
  .then(() => taskCounters.update++)
100
97
  .catch(err => {
101
98
  logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
@@ -105,7 +102,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
105
102
  );
106
103
  }
107
104
 
108
- // 3. Wait for ALL tasks to complete
105
+ // 3. Wait for ALL tasks to complete (now sequentially)
109
106
  await Promise.all(allTaskPromises);
110
107
 
111
108
  // 4. Log final summary
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.167",
3
+ "version": "1.0.169",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [