bulltrackers-module 1.0.178 → 1.0.180

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/functions/task-engine/helpers/update_helpers.js +72 -129
  2. package/package.json +1 -1
package/functions/task-engine/helpers/update_helpers.js CHANGED
@@ -3,7 +3,7 @@
3
3
  * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
4
4
  * (REFACTORED: Added node-fetch fallback for all API calls)
5
5
  * (REFACTORED: Added verbose, user-centric logging for all operations)
6
- * (FIXED: Resolved ReferenceError 'instId is not defined' in final timestamp loops)
6
+ * (FIXED: Corrected variable name 'instId' to 'instrumentId' in final timestamp loops)
7
7
  */
8
8
 
9
9
  const { FieldValue } = require('@google-cloud/firestore');
@@ -16,83 +16,54 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
16
16
  if (!cids?.length) return [];
17
17
  logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
18
18
 
19
- // --- Set concurrency to 1 to prevent AppScript rate limits. DO NOT CHANGE THIS.
19
+ // --- Set concurrency to 1 because appscript gets really fucked up with undocumented rate limits if we try spam it concurrently, a shame but that's life. DO NOT CHANGE THIS
20
20
  const limit = pLimit(1);
21
21
  const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
22
22
  const batches = [];
23
- for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
24
- batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
25
- }
26
-
27
- const batchPromises = batches.map((batch, index) => limit(async () => {
28
- const batchId = `batch-${index + 1}`;
23
+ for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
24
+ const batchPromises = batches.map((batch, index) => limit(async () => { const batchId = `batch-${index + 1}`;
29
25
  logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
30
26
  const header = await headerManager.selectHeader();
31
27
  if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
32
-
33
28
  let wasSuccess = false;
34
29
  let proxyUsed = true;
35
30
  let response;
36
31
  const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
37
32
  const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
38
-
39
33
  try {
40
34
  logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
41
35
  response = await proxyManager.fetch(url, options);
42
36
  if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
43
- wasSuccess = true;
37
+ wasSuccess = true; // Yay we win
44
38
  logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
45
- } catch (proxyError) {
46
- logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
47
- proxyUsed = false;
48
- try {
49
- response = await fetch(url, options);
50
- if (!response.ok) {
51
- const errorText = await response.text();
52
- throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
53
- }
54
- logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
55
- } catch (fallbackError) {
56
- logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' });
57
- return null;
39
+ } catch (proxyError) { logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed...
40
+ proxyUsed = false; // Don't penalize header for proxy failure
41
+ try { response = await fetch(url, options); // Ok let's try again with node, using GCP IP pools
42
+ if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
43
+ logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // Yay we win
44
+ } catch (fallbackError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); // SHIT, we failed here too
45
+ return null; // Give up on this batch
58
46
  }
59
- } finally {
60
- if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); }
61
- }
62
-
63
- try {
64
- const data = await response.json();
65
- return data;
66
- } catch (parseError) {
67
- logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
68
- return null;
69
- }
70
- }));
47
+ } finally { if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } } // If we used Appscript IP Pool and not GCP IP Pool, record performance
48
+ try { const data = await response.json(); return data;
49
+ } catch (parseError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; } }));
71
50
 
72
51
  const results = await Promise.allSettled(batchPromises);
73
- const allUsers = results
74
- .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
75
- .flatMap(r => r.value);
76
-
52
+ const allUsers = results .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)) .flatMap(r => r.value);
77
53
  logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
78
54
  return allUsers;
79
55
  }
80
56
 
81
57
 
82
58
  /**
83
- * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback, FIXED SCOPING)
59
+ * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
84
60
  */
85
61
  async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
86
62
  const { userId, instruments, instrumentId, userType } = task;
87
-
88
- // For normal users, we pass [undefined] so the loop runs exactly once.
89
- // For speculators, we pass the list of instruments to fetch individually.
90
- const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined];
91
-
63
+ const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined];
92
64
  const today = new Date().toISOString().slice(0, 10);
93
65
  const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
94
66
  let isPrivate = false;
95
-
96
67
  logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
97
68
 
98
69
  // --- 1. Process History Fetch (Sequentially) ---
@@ -104,70 +75,52 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
104
75
  if (!batchManager.checkAndSetHistoryFetched(userId)) {
105
76
  logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
106
77
  historyHeader = await headerManager.selectHeader();
107
- if (!historyHeader) {
108
- logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
78
+ if (!historyHeader) { logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
109
79
  } else {
110
80
  const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
111
81
  const options = { headers: historyHeader.header };
112
82
  let response;
113
-
114
- try {
115
- logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
83
+ try { logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
116
84
  response = await proxyManager.fetch(historyUrl, options);
117
- if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
118
- wasHistorySuccess = true;
85
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
86
+ wasHistorySuccess = true; // Appscript worked, we are very smart
87
+
119
88
  } catch (proxyError) {
120
- logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
89
+ logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed here
121
90
  proxyUsedForHistory = false;
122
- try {
123
- response = await fetch(historyUrl, options);
124
- if (!response.ok) {
125
- const errorText = await response.text();
126
- throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
127
- }
128
- wasHistorySuccess = true;
129
- } catch (fallbackError) {
130
- logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
131
- wasHistorySuccess = false;
91
+ try { response = await fetch(historyUrl, options);
92
+ if (!response.ok) { const errorText = await response.text();
93
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } // SHIT we failed here too
94
+ wasHistorySuccess = true; // Fallback succeeded, we are so smart
95
+
96
+ } catch (fallbackError) { logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); // We are dumb, everything failed
97
+ wasHistorySuccess = false; // Nope we are dumb....
132
98
  }
133
99
  }
134
100
 
135
- if (wasHistorySuccess) {
136
- logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
101
+ if (wasHistorySuccess) { logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`); // Some method worked, we are very smart
137
102
  const data = await response.json();
138
- await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
139
- }
103
+ await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); }
140
104
  }
141
- } else {
142
- logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
143
- }
144
- } catch (err) {
145
- logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
146
- wasHistorySuccess = false;
147
- } finally {
148
- if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); }
149
- }
105
+ } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); }
106
+ } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; // We fucked up.
107
+ } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } // If we used appscript proxy, record performance, otherwise fuck off.
150
108
 
151
109
  // --- 2. Process Portfolio Fetches (Sequentially) ---
152
110
  logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
153
111
 
154
- // Renamed loop variable to currentInstrumentId to prevent 'instId' reference errors later
155
- for (const currentInstrumentId of instrumentsToProcess) {
112
+ for (const instId of instrumentsToProcess) {
156
113
  if (isPrivate) {
157
114
  logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
158
115
  break;
159
116
  }
160
117
 
161
118
  const portfolioHeader = await headerManager.selectHeader();
162
- if (!portfolioHeader) {
163
- logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header. Skipping instrument.`);
119
+ if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
164
120
  continue;
165
121
  }
166
122
 
167
- // Only append InstrumentID if it is defined (i.e., for speculators)
168
- const portfolioUrl = userType === 'speculator' && currentInstrumentId
169
- ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${currentInstrumentId}`
170
- : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
123
+ const portfolioUrl = userType === 'speculator' ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
171
124
 
172
125
  const options = { headers: portfolioHeader.header };
173
126
  let response;
@@ -175,87 +128,77 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
175
128
  let proxyUsedForPortfolio = true;
176
129
 
177
130
  try {
178
- logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
131
+ // --- REFACTOR 3: ADD FALLBACK ---
132
+ logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
179
133
  response = await proxyManager.fetch(portfolioUrl, options);
180
- if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
181
- wasPortfolioSuccess = true;
134
+ if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
135
+ wasPortfolioSuccess = true; // Oh we are smart, worked first time.
182
136
 
183
- } catch (proxyError) {
184
- logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
185
- proxyUsedForPortfolio = false;
137
+ } catch (proxyError) { // try fallback with local node fetch using GCP IP Pools
138
+ logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
139
+ proxyUsedForPortfolio = false; // We are not using Appscript proxy here as fallback is GCP based, so false
186
140
 
187
141
  try {
188
- response = await fetch(portfolioUrl, options);
142
+ response = await fetch(portfolioUrl, options); // Direct node-fetch
189
143
  if (!response.ok) {
190
144
  const errorText = await response.text();
191
- throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
145
+ throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // SHIT we failed here
192
146
  }
193
- wasPortfolioSuccess = true;
147
+ wasPortfolioSuccess = true; // Fallback succeeded we are so smart
148
+
194
149
  } catch (fallbackError) {
195
- logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
150
+ logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
196
151
  wasPortfolioSuccess = false;
197
152
  }
198
153
  }
199
154
 
155
+ // --- 4. Process Portfolio Result (with verbose, raw logging) ---
200
156
  if (wasPortfolioSuccess) {
201
157
  const body = await response.text();
202
- if (body.includes("user is PRIVATE")) {
203
- isPrivate = true;
204
- logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
205
- break;
158
+ if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
159
+ break; // Stop processing more portfolios for this private user
206
160
  }
207
161
 
208
162
  try {
209
163
  const portfolioJson = JSON.parse(body);
210
- // Pass currentInstrumentId to batchManager. For normal users it's undefined, which is fine.
211
- await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, currentInstrumentId);
212
-
213
- if (userType === 'speculator') {
214
- logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${currentInstrumentId}.`);
215
- } else {
216
- logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`);
217
- }
164
+ await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
165
+ if (userType === 'speculator') { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`); // Only speculators have an instid, so this is conditional
166
+ } else { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); } // Normal users
218
167
 
219
- } catch (parseError) {
220
- wasPortfolioSuccess = false;
221
- logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body });
168
+ } catch (parseError) { // Idk why this would happen, but if it does....log.
169
+ wasPortfolioSuccess = false; // Mark as failure
170
+ logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); // Return full response
222
171
  }
223
- } else {
224
- logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed. No response to process.`);
225
- }
172
+ } else { logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`); }
226
173
 
227
174
  if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
228
175
  }
229
176
 
230
177
  // --- 5. Handle Private Users & Timestamps ---
178
+ // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
231
179
  if (isPrivate) {
232
180
  logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
233
- // Iterate again using correct scoping
234
- for (const currentInstrumentId of instrumentsToProcess) {
235
- await batchManager.deleteFromTimestampBatch(userId, userType, currentInstrumentId);
181
+ for (const instrumentId of instrumentsToProcess) {
182
+ await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
236
183
  }
237
-
238
184
  const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
239
- for (const currentInstrumentId of instrumentsToProcess) {
240
- // Only update counts if we have a valid instrument ID (speculators)
241
- if (currentInstrumentId) {
242
- const incrementField = `counts.${currentInstrumentId}_${Math.floor(userId/1e6)*1e6}`;
243
- await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
244
- }
185
+ for (const instrumentId of instrumentsToProcess) {
186
+ const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
187
+ await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
245
188
  }
246
189
  return;
247
190
  }
248
191
 
249
192
  // If not private, update all timestamps
250
- for (const currentInstrumentId of instrumentsToProcess) {
251
- await batchManager.updateUserTimestamp(userId, userType, currentInstrumentId);
193
+ // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
194
+ for (const instrumentId of instrumentsToProcess) {
195
+ await batchManager.updateUserTimestamp(userId, userType, instrumentId);
252
196
  }
253
197
 
254
- if (userType === 'speculator') {
255
- await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
256
- }
198
+ if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
257
199
 
258
200
  logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
201
+ // 'finally' block for header flushing is handled by the main handler_creator.js
259
202
  }
260
203
 
261
204
  module.exports = { handleUpdate, lookupUsernames };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.178",
3
+ "version": "1.0.180",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [