bulltrackers-module 1.0.177 → 1.0.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
|
|
4
4
|
* (REFACTORED: Added node-fetch fallback for all API calls)
|
|
5
5
|
* (REFACTORED: Added verbose, user-centric logging for all operations)
|
|
6
|
-
* (FIXED:
|
|
6
|
+
* (FIXED: Resolved ReferenceError 'instId is not defined' in final timestamp loops)
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const { FieldValue } = require('@google-cloud/firestore');
|
|
@@ -16,54 +16,83 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
|
|
|
16
16
|
if (!cids?.length) return [];
|
|
17
17
|
logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
|
|
18
18
|
|
|
19
|
-
// --- Set concurrency to 1
|
|
19
|
+
// --- Set concurrency to 1 to prevent AppScript rate limits. DO NOT CHANGE THIS.
|
|
20
20
|
const limit = pLimit(1);
|
|
21
21
|
const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
|
|
22
22
|
const batches = [];
|
|
23
|
-
for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
|
|
24
|
-
|
|
23
|
+
for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
|
|
24
|
+
batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const batchPromises = batches.map((batch, index) => limit(async () => {
|
|
28
|
+
const batchId = `batch-${index + 1}`;
|
|
25
29
|
logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
|
|
26
30
|
const header = await headerManager.selectHeader();
|
|
27
31
|
if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
|
|
32
|
+
|
|
28
33
|
let wasSuccess = false;
|
|
29
34
|
let proxyUsed = true;
|
|
30
35
|
let response;
|
|
31
36
|
const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
|
|
32
37
|
const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
|
|
38
|
+
|
|
33
39
|
try {
|
|
34
40
|
logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
|
|
35
41
|
response = await proxyManager.fetch(url, options);
|
|
36
42
|
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
37
|
-
wasSuccess = true;
|
|
43
|
+
wasSuccess = true;
|
|
38
44
|
logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
|
|
39
|
-
} catch (proxyError) {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
} catch (proxyError) {
|
|
46
|
+
logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
|
|
47
|
+
proxyUsed = false;
|
|
48
|
+
try {
|
|
49
|
+
response = await fetch(url, options);
|
|
50
|
+
if (!response.ok) {
|
|
51
|
+
const errorText = await response.text();
|
|
52
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
53
|
+
}
|
|
54
|
+
logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
|
|
55
|
+
} catch (fallbackError) {
|
|
56
|
+
logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' });
|
|
57
|
+
return null;
|
|
46
58
|
}
|
|
47
|
-
} finally {
|
|
48
|
-
|
|
49
|
-
}
|
|
59
|
+
} finally {
|
|
60
|
+
if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
try {
|
|
64
|
+
const data = await response.json();
|
|
65
|
+
return data;
|
|
66
|
+
} catch (parseError) {
|
|
67
|
+
logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}));
|
|
50
71
|
|
|
51
72
|
const results = await Promise.allSettled(batchPromises);
|
|
52
|
-
const allUsers = results
|
|
73
|
+
const allUsers = results
|
|
74
|
+
.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
|
|
75
|
+
.flatMap(r => r.value);
|
|
76
|
+
|
|
53
77
|
logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
|
|
54
78
|
return allUsers;
|
|
55
79
|
}
|
|
56
80
|
|
|
57
81
|
|
|
58
82
|
/**
|
|
59
|
-
* (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
|
|
83
|
+
* (REFACTORED: Fully sequential, verbose logging, node-fetch fallback, FIXED SCOPING)
|
|
60
84
|
*/
|
|
61
85
|
async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
|
|
62
86
|
const { userId, instruments, instrumentId, userType } = task;
|
|
63
|
-
|
|
87
|
+
|
|
88
|
+
// For normal users, we pass [undefined] so the loop runs exactly once.
|
|
89
|
+
// For speculators, we pass the list of instruments to fetch individually.
|
|
90
|
+
const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined];
|
|
91
|
+
|
|
64
92
|
const today = new Date().toISOString().slice(0, 10);
|
|
65
93
|
const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
|
|
66
94
|
let isPrivate = false;
|
|
95
|
+
|
|
67
96
|
logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
|
|
68
97
|
|
|
69
98
|
// --- 1. Process History Fetch (Sequentially) ---
|
|
@@ -75,52 +104,70 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
|
|
|
75
104
|
if (!batchManager.checkAndSetHistoryFetched(userId)) {
|
|
76
105
|
logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
|
|
77
106
|
historyHeader = await headerManager.selectHeader();
|
|
78
|
-
if (!historyHeader) {
|
|
107
|
+
if (!historyHeader) {
|
|
108
|
+
logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
|
|
79
109
|
} else {
|
|
80
110
|
const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
|
|
81
111
|
const options = { headers: historyHeader.header };
|
|
82
112
|
let response;
|
|
83
|
-
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
|
|
84
116
|
response = await proxyManager.fetch(historyUrl, options);
|
|
85
|
-
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
86
|
-
wasHistorySuccess = true;
|
|
87
|
-
|
|
117
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
118
|
+
wasHistorySuccess = true;
|
|
88
119
|
} catch (proxyError) {
|
|
89
|
-
logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
|
|
120
|
+
logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
|
|
90
121
|
proxyUsedForHistory = false;
|
|
91
|
-
try {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
wasHistorySuccess =
|
|
122
|
+
try {
|
|
123
|
+
response = await fetch(historyUrl, options);
|
|
124
|
+
if (!response.ok) {
|
|
125
|
+
const errorText = await response.text();
|
|
126
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
127
|
+
}
|
|
128
|
+
wasHistorySuccess = true;
|
|
129
|
+
} catch (fallbackError) {
|
|
130
|
+
logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
|
|
131
|
+
wasHistorySuccess = false;
|
|
98
132
|
}
|
|
99
133
|
}
|
|
100
134
|
|
|
101
|
-
if (wasHistorySuccess) {
|
|
135
|
+
if (wasHistorySuccess) {
|
|
136
|
+
logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
|
|
102
137
|
const data = await response.json();
|
|
103
|
-
await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
|
|
138
|
+
await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
|
|
139
|
+
}
|
|
104
140
|
}
|
|
105
|
-
} else {
|
|
106
|
-
|
|
107
|
-
|
|
141
|
+
} else {
|
|
142
|
+
logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
|
|
143
|
+
}
|
|
144
|
+
} catch (err) {
|
|
145
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
|
|
146
|
+
wasHistorySuccess = false;
|
|
147
|
+
} finally {
|
|
148
|
+
if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); }
|
|
149
|
+
}
|
|
108
150
|
|
|
109
151
|
// --- 2. Process Portfolio Fetches (Sequentially) ---
|
|
110
152
|
logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
|
|
111
153
|
|
|
112
|
-
|
|
154
|
+
// Renamed loop variable to currentInstrumentId to prevent 'instId' reference errors later
|
|
155
|
+
for (const currentInstrumentId of instrumentsToProcess) {
|
|
113
156
|
if (isPrivate) {
|
|
114
157
|
logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
|
|
115
158
|
break;
|
|
116
159
|
}
|
|
117
160
|
|
|
118
161
|
const portfolioHeader = await headerManager.selectHeader();
|
|
119
|
-
if (!portfolioHeader) {
|
|
162
|
+
if (!portfolioHeader) {
|
|
163
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header. Skipping instrument.`);
|
|
120
164
|
continue;
|
|
121
165
|
}
|
|
122
166
|
|
|
123
|
-
|
|
167
|
+
// Only append InstrumentID if it is defined (i.e., for speculators)
|
|
168
|
+
const portfolioUrl = userType === 'speculator' && currentInstrumentId
|
|
169
|
+
? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${currentInstrumentId}`
|
|
170
|
+
: `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
|
|
124
171
|
|
|
125
172
|
const options = { headers: portfolioHeader.header };
|
|
126
173
|
let response;
|
|
@@ -128,77 +175,87 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
|
|
|
128
175
|
let proxyUsedForPortfolio = true;
|
|
129
176
|
|
|
130
177
|
try {
|
|
131
|
-
|
|
132
|
-
logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
|
|
178
|
+
logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
|
|
133
179
|
response = await proxyManager.fetch(portfolioUrl, options);
|
|
134
|
-
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
135
|
-
wasPortfolioSuccess = true;
|
|
180
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
181
|
+
wasPortfolioSuccess = true;
|
|
136
182
|
|
|
137
|
-
} catch (proxyError) {
|
|
138
|
-
logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch
|
|
139
|
-
proxyUsedForPortfolio = false;
|
|
183
|
+
} catch (proxyError) {
|
|
184
|
+
logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
|
|
185
|
+
proxyUsedForPortfolio = false;
|
|
140
186
|
|
|
141
187
|
try {
|
|
142
|
-
response = await fetch(portfolioUrl, options);
|
|
188
|
+
response = await fetch(portfolioUrl, options);
|
|
143
189
|
if (!response.ok) {
|
|
144
190
|
const errorText = await response.text();
|
|
145
|
-
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
191
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
146
192
|
}
|
|
147
|
-
wasPortfolioSuccess = true;
|
|
148
|
-
|
|
193
|
+
wasPortfolioSuccess = true;
|
|
149
194
|
} catch (fallbackError) {
|
|
150
|
-
logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch
|
|
195
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
|
|
151
196
|
wasPortfolioSuccess = false;
|
|
152
197
|
}
|
|
153
198
|
}
|
|
154
199
|
|
|
155
|
-
// --- 4. Process Portfolio Result (with verbose, raw logging) ---
|
|
156
200
|
if (wasPortfolioSuccess) {
|
|
157
201
|
const body = await response.text();
|
|
158
|
-
if (body.includes("user is PRIVATE")) {
|
|
159
|
-
|
|
202
|
+
if (body.includes("user is PRIVATE")) {
|
|
203
|
+
isPrivate = true;
|
|
204
|
+
logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
|
|
205
|
+
break;
|
|
160
206
|
}
|
|
161
207
|
|
|
162
208
|
try {
|
|
163
209
|
const portfolioJson = JSON.parse(body);
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
210
|
+
// Pass currentInstrumentId to batchManager. For normal users it's undefined, which is fine.
|
|
211
|
+
await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, currentInstrumentId);
|
|
212
|
+
|
|
213
|
+
if (userType === 'speculator') {
|
|
214
|
+
logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${currentInstrumentId}.`);
|
|
215
|
+
} else {
|
|
216
|
+
logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`);
|
|
217
|
+
}
|
|
167
218
|
|
|
168
|
-
} catch (parseError) {
|
|
169
|
-
wasPortfolioSuccess = false;
|
|
170
|
-
logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body });
|
|
219
|
+
} catch (parseError) {
|
|
220
|
+
wasPortfolioSuccess = false;
|
|
221
|
+
logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body });
|
|
171
222
|
}
|
|
172
|
-
} else {
|
|
223
|
+
} else {
|
|
224
|
+
logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed. No response to process.`);
|
|
225
|
+
}
|
|
173
226
|
|
|
174
227
|
if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
|
|
175
228
|
}
|
|
176
229
|
|
|
177
230
|
// --- 5. Handle Private Users & Timestamps ---
|
|
178
|
-
// FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
|
|
179
231
|
if (isPrivate) {
|
|
180
232
|
logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
|
|
181
|
-
|
|
182
|
-
|
|
233
|
+
// Iterate again using correct scoping
|
|
234
|
+
for (const currentInstrumentId of instrumentsToProcess) {
|
|
235
|
+
await batchManager.deleteFromTimestampBatch(userId, userType, currentInstrumentId);
|
|
183
236
|
}
|
|
237
|
+
|
|
184
238
|
const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
|
|
185
|
-
for (const
|
|
186
|
-
|
|
187
|
-
|
|
239
|
+
for (const currentInstrumentId of instrumentsToProcess) {
|
|
240
|
+
// Only update counts if we have a valid instrument ID (speculators)
|
|
241
|
+
if (currentInstrumentId) {
|
|
242
|
+
const incrementField = `counts.${currentInstrumentId}_${Math.floor(userId/1e6)*1e6}`;
|
|
243
|
+
await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
|
|
244
|
+
}
|
|
188
245
|
}
|
|
189
246
|
return;
|
|
190
247
|
}
|
|
191
248
|
|
|
192
249
|
// If not private, update all timestamps
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
await batchManager.updateUserTimestamp(userId, userType, instrumentId);
|
|
250
|
+
for (const currentInstrumentId of instrumentsToProcess) {
|
|
251
|
+
await batchManager.updateUserTimestamp(userId, userType, currentInstrumentId);
|
|
196
252
|
}
|
|
197
253
|
|
|
198
|
-
if (userType === 'speculator') {
|
|
254
|
+
if (userType === 'speculator') {
|
|
255
|
+
await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
|
|
256
|
+
}
|
|
199
257
|
|
|
200
258
|
logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
|
|
201
|
-
// 'finally' block for header flushing is handled by the main handler_creator.js
|
|
202
259
|
}
|
|
203
260
|
|
|
204
261
|
module.exports = { handleUpdate, lookupUsernames };
|