bulltrackers-module 1.0.167 → 1.0.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/core/utils/intelligent_proxy_manager.js +49 -16
- package/functions/task-engine/helpers/discover_helpers.js +140 -25
- package/functions/task-engine/helpers/update_helpers.js +222 -119
- package/functions/task-engine/helpers/verify_helpers.js +128 -13
- package/functions/task-engine/utils/task_engine_utils.js +10 -13
- package/package.json +1 -1
|
@@ -82,30 +82,63 @@ class IntelligentProxyManager {
|
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
* ---
|
|
85
|
+
* --- CORRECTED LOGIC: Makes a fetch request by trying different proxies ---
|
|
86
86
|
* @param {string} targetUrl - The URL to fetch.
|
|
87
87
|
* @param {object} options - Fetch options (e.g., headers).
|
|
88
88
|
* @returns {Promise<object>} A mock Response object.
|
|
89
89
|
*/
|
|
90
90
|
async fetch(targetUrl, options = {}) {
|
|
91
|
-
let proxy = null;
|
|
92
|
-
try { proxy = await this._selectProxy(); } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
|
|
93
|
-
let backoff = this.INITIAL_BACKOFF_MS;
|
|
94
91
|
let lastResponse = null;
|
|
92
|
+
|
|
93
|
+
// Use MAX_RETRIES to define the number of *different proxies* we will try
|
|
94
|
+
// before giving up on the request.
|
|
95
95
|
for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
|
|
96
|
+
let proxy;
|
|
97
|
+
try {
|
|
98
|
+
// 1. Select a new, UNLOCKED proxy *inside* the loop.
|
|
99
|
+
proxy = await this._selectProxy();
|
|
100
|
+
} catch (error) {
|
|
101
|
+
// This fails if all proxies are locked.
|
|
102
|
+
this.logger.log('ERROR', '[ProxyManager] fetch failed: All proxies are locked.', { url: targetUrl });
|
|
103
|
+
return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 2. Make a SINGLE attempt with this selected proxy.
|
|
96
107
|
const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
|
|
97
|
-
lastResponse = response;
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// 4.
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
108
|
+
lastResponse = response; // Save this response in case it's the last one
|
|
109
|
+
|
|
110
|
+
// 3. Case 1: Success! Return immediately.
|
|
111
|
+
if (response.ok) {
|
|
112
|
+
return response;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// 4. Case 2: Proxy-level error (Rate limit OR other fetch error)
|
|
116
|
+
// This is a *proxy* failure, not a *target* failure.
|
|
117
|
+
if (response.isUrlFetchError) {
|
|
118
|
+
this.logger.log('WARN', `[ProxyManager] Proxy ${proxy.owner} failed (Attempt ${attempt}/${this.MAX_RETRIES}). Locking it and trying a new proxy.`, {
|
|
119
|
+
reason: response.isRateLimitError ? "Rate Limit" : "Other Fetch Error",
|
|
120
|
+
status: response.status,
|
|
121
|
+
url: targetUrl
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// LOCK THE FAILED PROXY so _selectProxy() won't pick it again.
|
|
125
|
+
await this.lockProxy(proxy.owner);
|
|
126
|
+
|
|
127
|
+
// Back off slightly before trying the *next* proxy to avoid a thundering herd.
|
|
128
|
+
await sleep(this.INITIAL_BACKOFF_MS * attempt);
|
|
129
|
+
|
|
130
|
+
continue; // Go to the next loop iteration to select a *new* proxy.
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// 5. Case 3: Standard error from *target* URL (e.g., 404, 500)
|
|
134
|
+
// This means the proxy *worked* but the target URL is bad.
|
|
135
|
+
// This is a "successful" fetch. Do not retry, just return the response.
|
|
136
|
+
return response;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// 6. If loop finishes, all (this.MAX_RETRIES) proxy attempts failed.
|
|
140
|
+
this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} proxy attempts.`, { url: targetUrl, lastStatus: lastResponse?.status });
|
|
141
|
+
return lastResponse; // Return the last failed response
|
|
109
142
|
}
|
|
110
143
|
|
|
111
144
|
|
|
@@ -1,7 +1,14 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
|
|
3
|
+
* (REFACTORED: Now includes node-fetch fallback)
|
|
4
|
+
* (REFACTORED: Added verbose, traceable logging)
|
|
5
|
+
*/
|
|
6
|
+
|
|
1
7
|
/**
|
|
2
8
|
* @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
|
|
3
9
|
* REFACTORED: Now stateless and receives dependencies.
|
|
4
10
|
* --- MODIFIED: Passes username to the 'verify' task. ---
|
|
11
|
+
* --- MODIFIED: Added node-fetch fallback and verbose logging. ---
|
|
5
12
|
*/
|
|
6
13
|
|
|
7
14
|
/**
|
|
@@ -15,56 +22,164 @@ async function handleDiscover(task, taskId, dependencies, config) {
|
|
|
15
22
|
const { logger, headerManager, proxyManager, pubsub, batchManager } = dependencies;
|
|
16
23
|
const { cids, blockId, instrument, userType } = task;
|
|
17
24
|
const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
|
|
25
|
+
|
|
18
26
|
const selectedHeader = await headerManager.selectHeader();
|
|
19
|
-
if (!selectedHeader)
|
|
27
|
+
if (!selectedHeader) {
|
|
28
|
+
logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
|
|
29
|
+
throw new Error("Could not select a header.");
|
|
30
|
+
}
|
|
31
|
+
|
|
20
32
|
let wasSuccess = false;
|
|
33
|
+
let proxyUsed = true;
|
|
34
|
+
const logPrefix = `[DISCOVER/${taskId}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
|
|
35
|
+
|
|
36
|
+
try { // Outer try for the whole operation
|
|
37
|
+
let response;
|
|
38
|
+
const options = {
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
|
|
41
|
+
body: JSON.stringify(cids),
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
logger.log('INFO', `${logPrefix} Starting discovery for ${cids.length} CIDs. Block: ${blockId}, Type: ${userType}`);
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
// --- REFACTOR 3: ADD FALLBACK ---
|
|
48
|
+
logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`);
|
|
49
|
+
response = await proxyManager.fetch(url, options);
|
|
50
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
51
|
+
|
|
52
|
+
} catch (proxyError) {
|
|
53
|
+
logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
|
|
54
|
+
error: proxyError.message,
|
|
55
|
+
source: 'AppScript'
|
|
56
|
+
});
|
|
57
|
+
proxyUsed = false;
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
response = await fetch(url, options); // Direct node-fetch
|
|
61
|
+
if (!response.ok) {
|
|
62
|
+
const errorText = await response.text();
|
|
63
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
} catch (fallbackError) {
|
|
67
|
+
logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
|
|
68
|
+
error: fallbackError.message,
|
|
69
|
+
source: 'eToro/Network'
|
|
70
|
+
});
|
|
71
|
+
throw fallbackError; // Throw to be caught by outer try
|
|
72
|
+
}
|
|
73
|
+
// --- END REFACTOR 3 ---
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// --- If we are here, `response` is valid ---
|
|
77
|
+
|
|
78
|
+
// Step 1. Discover Speculators
|
|
79
|
+
if (userType === 'speculator') {
|
|
80
|
+
batchManager.addProcessedSpeculatorCids(cids);
|
|
81
|
+
logger.log('INFO', `${logPrefix} Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const body = await response.text();
|
|
85
|
+
let publicUsers;
|
|
86
|
+
try {
|
|
87
|
+
// --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
|
|
88
|
+
publicUsers = JSON.parse(body);
|
|
89
|
+
} catch (parseError) {
|
|
90
|
+
logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
|
|
91
|
+
parseErrorMessage: parseError.message,
|
|
92
|
+
rawResponseText: body
|
|
93
|
+
});
|
|
94
|
+
throw new Error(`Failed to parse JSON response from discovery API. Body: ${body.substring(0, 200)}`);
|
|
95
|
+
}
|
|
96
|
+
// --- END REFACTOR 4 ---
|
|
97
|
+
|
|
98
|
+
if (!Array.isArray(publicUsers)) {
|
|
99
|
+
logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`);
|
|
100
|
+
wasSuccess = true; // API call worked, data was just empty/weird
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
wasSuccess = true; // Mark as success *after* parsing
|
|
105
|
+
logger.log('INFO', `${logPrefix} API call successful, found ${publicUsers.length} public users.`);
|
|
21
106
|
|
|
22
|
-
// Step 1. Discover Speculators
|
|
23
|
-
try {if (userType === 'speculator') {batchManager.addProcessedSpeculatorCids(cids);logger.log('INFO', `[DISCOVER] Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);}
|
|
24
|
-
const response = await proxyManager.fetch(url, {method: 'POST',headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },body: JSON.stringify(cids),});
|
|
25
|
-
if (!response.ok) {throw new Error(`API status ${response.status}`);}
|
|
26
|
-
wasSuccess = true;
|
|
27
|
-
const publicUsers = await response.json();
|
|
28
|
-
if (!Array.isArray(publicUsers)) return;
|
|
29
107
|
const oneMonthAgo = new Date();
|
|
30
108
|
oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
|
|
31
|
-
|
|
32
|
-
|
|
109
|
+
|
|
110
|
+
// Step 2. Filter active users
|
|
111
|
+
const preliminaryActiveUsers = publicUsers.filter(user =>
|
|
112
|
+
new Date(user.Value.LastActivity) > oneMonthAgo &&
|
|
113
|
+
user.Value.DailyGain !== 0 &&
|
|
114
|
+
user.Value.Exposure !== 0 &&
|
|
115
|
+
user.Value.RiskScore !== 0
|
|
116
|
+
);
|
|
117
|
+
logger.log('INFO', `${logPrefix} Found ${preliminaryActiveUsers.length} preliminary active users.`);
|
|
118
|
+
|
|
33
119
|
let finalActiveUsers = [];
|
|
34
120
|
const invalidCidsToLog = [];
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
121
|
+
|
|
122
|
+
// Step 3. Log private/inactive users
|
|
123
|
+
if (userType === 'speculator') {
|
|
124
|
+
const publicUserCids = new Set(publicUsers.map(u => u.CID));
|
|
125
|
+
if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {
|
|
126
|
+
const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));
|
|
127
|
+
invalidCidsToLog.push(...privateUserCids);
|
|
128
|
+
logger.log('INFO', `${logPrefix} Found ${privateUserCids.length} private users (in request but not response).`);
|
|
129
|
+
}
|
|
38
130
|
const activeUserCids = new Set(preliminaryActiveUsers.map(u => u.CID));
|
|
39
131
|
const inactiveUserCids = publicUsers.filter(u => !activeUserCids.has(u.CID)).map(u => u.CID);
|
|
40
132
|
invalidCidsToLog.push(...inactiveUserCids);
|
|
41
|
-
logger.log('INFO',
|
|
133
|
+
logger.log('INFO', `${logPrefix} Found ${inactiveUserCids.length} inactive users.`);
|
|
134
|
+
|
|
135
|
+
// Step 4. Apply speculator heuristic
|
|
42
136
|
const nonSpeculatorCids = [];
|
|
43
|
-
// Step 4. For the remaining active users, apply heuristic filters to weed out non-speculators.
|
|
44
137
|
for (const user of preliminaryActiveUsers) {
|
|
45
138
|
const v = user.Value;
|
|
46
139
|
const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
|
|
47
|
-
const isLikelySpeculator = ((v.Trades || 0) > 500 ||(v.TotalTradedInstruments || 0) > 50 ||totalLeverage > 50 ||(v.WeeklyDD || 0) < -25);
|
|
48
|
-
|
|
140
|
+
const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25);
|
|
141
|
+
|
|
142
|
+
if (isLikelySpeculator) {
|
|
143
|
+
finalActiveUsers.push(user);
|
|
144
|
+
} else {
|
|
145
|
+
nonSpeculatorCids.push(user.CID);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
49
148
|
invalidCidsToLog.push(...nonSpeculatorCids);
|
|
50
|
-
logger.log('INFO',
|
|
149
|
+
logger.log('INFO', `${logPrefix} Speculator pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
|
|
150
|
+
|
|
51
151
|
if (invalidCidsToLog.length > 0) {
|
|
52
152
|
await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
|
|
53
|
-
logger.log('INFO',
|
|
153
|
+
logger.log('INFO', `${logPrefix} Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
|
|
54
154
|
}
|
|
55
|
-
// Step 5.
|
|
155
|
+
// Step 5. Non-speculators are just active users
|
|
56
156
|
} else {
|
|
57
157
|
finalActiveUsers = preliminaryActiveUsers;
|
|
58
158
|
}
|
|
59
|
-
|
|
159
|
+
|
|
160
|
+
// Step 6. Publish 'verify' task
|
|
60
161
|
if (finalActiveUsers.length > 0) {
|
|
61
|
-
const verificationTask = {
|
|
162
|
+
const verificationTask = {
|
|
163
|
+
type: 'verify',
|
|
164
|
+
users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze, username: u.Value.UserName })),
|
|
165
|
+
blockId,
|
|
166
|
+
instrument,
|
|
167
|
+
userType
|
|
168
|
+
};
|
|
62
169
|
await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
|
|
63
|
-
logger.log('
|
|
64
|
-
|
|
170
|
+
logger.log('SUCCESS', `${logPrefix} Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
|
|
171
|
+
} else {
|
|
172
|
+
logger.log('INFO', `${logPrefix} No active users found to verify.`);
|
|
65
173
|
}
|
|
174
|
+
|
|
175
|
+
} catch (err) {
|
|
176
|
+
logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack });
|
|
177
|
+
wasSuccess = false; // Ensure it's marked as failure
|
|
66
178
|
} finally {
|
|
67
|
-
if (selectedHeader
|
|
179
|
+
if (selectedHeader && proxyUsed) {
|
|
180
|
+
// Only update performance if the proxy was used
|
|
181
|
+
headerManager.updatePerformance(selectedHeader.id, wasSuccess);
|
|
182
|
+
}
|
|
68
183
|
}
|
|
69
184
|
}
|
|
70
185
|
|
|
@@ -1,31 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
|
|
3
|
-
* (
|
|
4
|
-
* (
|
|
5
|
-
* (
|
|
6
|
-
* (MODIFIED: `handleUpdate` now uses batchManager for history cache)
|
|
3
|
+
* (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
|
|
4
|
+
* (REFACTORED: Added node-fetch fallback for all API calls)
|
|
5
|
+
* (REFACTORED: Added verbose, user-centric logging for all operations)
|
|
7
6
|
*/
|
|
8
7
|
|
|
9
|
-
/**
|
|
10
|
-
* @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
|
|
11
|
-
* REFACTORED: Now stateless and receives dependencies.
|
|
12
|
-
* OPTIMIZED: Removed immediate batch commit for speculator timestamp fix.
|
|
13
|
-
* --- MODIFIED: Fetches portfolio AND trade history in parallel. ---
|
|
14
|
-
* --- MODIFIED: Includes helper to look up usernames from CIDs. ---
|
|
15
|
-
* --- MODIFIED: Conditionally fetches history only once per user per batch. ---
|
|
16
|
-
*/
|
|
17
8
|
const { FieldValue } = require('@google-cloud/firestore');
|
|
18
9
|
const pLimit = require('p-limit');
|
|
19
10
|
|
|
20
11
|
/**
|
|
21
|
-
* (
|
|
12
|
+
* (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
|
|
22
13
|
*/
|
|
23
14
|
async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
|
|
24
15
|
if (!cids?.length) return [];
|
|
25
16
|
logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
|
|
26
17
|
|
|
27
|
-
//
|
|
28
|
-
const limit = pLimit(
|
|
18
|
+
// --- REFACTOR 1: REMOVE CONCURRENCY ---
|
|
19
|
+
const limit = pLimit(1);
|
|
20
|
+
// --- END REFACTOR 1 ---
|
|
21
|
+
|
|
29
22
|
const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
|
|
30
23
|
|
|
31
24
|
const batches = [];
|
|
@@ -33,26 +26,67 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
|
|
|
33
26
|
batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
|
|
34
27
|
}
|
|
35
28
|
|
|
36
|
-
const batchPromises = batches.map(batch => limit(async () => {
|
|
29
|
+
const batchPromises = batches.map((batch, index) => limit(async () => {
|
|
30
|
+
const batchId = `batch-${index + 1}`;
|
|
31
|
+
logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
|
|
32
|
+
|
|
37
33
|
const header = await headerManager.selectHeader();
|
|
38
34
|
if (!header) {
|
|
39
|
-
logger.log('ERROR',
|
|
40
|
-
return null;
|
|
35
|
+
logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`);
|
|
36
|
+
return null;
|
|
41
37
|
}
|
|
42
38
|
|
|
43
|
-
let
|
|
39
|
+
let wasSuccess = false;
|
|
40
|
+
let proxyUsed = true;
|
|
41
|
+
let response;
|
|
42
|
+
const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
|
|
43
|
+
const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
|
|
44
|
+
|
|
44
45
|
try {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
46
|
+
// --- REFACTOR 3: ADD FALLBACK ---
|
|
47
|
+
logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
|
|
48
|
+
response = await proxyManager.fetch(url, options);
|
|
49
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
50
|
+
|
|
51
|
+
wasSuccess = true;
|
|
52
|
+
logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
|
|
53
|
+
|
|
54
|
+
} catch (proxyError) {
|
|
55
|
+
logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
|
|
56
|
+
error: proxyError.message,
|
|
57
|
+
source: 'AppScript'
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
proxyUsed = false; // Don't penalize header for proxy failure
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
response = await fetch(url, options); // Direct node-fetch
|
|
64
|
+
if (!response.ok) {
|
|
65
|
+
const errorText = await response.text();
|
|
66
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
67
|
+
}
|
|
68
|
+
logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
|
|
69
|
+
|
|
70
|
+
} catch (fallbackError) {
|
|
71
|
+
logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, {
|
|
72
|
+
error: fallbackError.message,
|
|
73
|
+
source: 'eToro/Network'
|
|
74
|
+
});
|
|
75
|
+
return null; // Give up on this batch
|
|
76
|
+
}
|
|
77
|
+
// --- END REFACTOR 3 ---
|
|
54
78
|
} finally {
|
|
55
|
-
|
|
79
|
+
if (proxyUsed) {
|
|
80
|
+
headerManager.updatePerformance(header.id, wasSuccess);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
const data = await response.json();
|
|
86
|
+
return data;
|
|
87
|
+
} catch (parseError) {
|
|
88
|
+
logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
|
|
89
|
+
return null;
|
|
56
90
|
}
|
|
57
91
|
}));
|
|
58
92
|
|
|
@@ -60,7 +94,7 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
|
|
|
60
94
|
|
|
61
95
|
const allUsers = results
|
|
62
96
|
.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
|
|
63
|
-
.flatMap(r => r.value);
|
|
97
|
+
.flatMap(r => r.value);
|
|
64
98
|
|
|
65
99
|
logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
|
|
66
100
|
return allUsers;
|
|
@@ -68,128 +102,197 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
|
|
|
68
102
|
|
|
69
103
|
|
|
70
104
|
/**
|
|
71
|
-
* (
|
|
72
|
-
* (MODIFIED: Uses batchManager for history cache)
|
|
105
|
+
* (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
|
|
73
106
|
*/
|
|
74
|
-
async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
|
|
107
|
+
async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
|
|
75
108
|
const { userId, instruments, instrumentId, userType } = task;
|
|
76
109
|
const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined];
|
|
77
110
|
const today = new Date().toISOString().slice(0, 10);
|
|
78
111
|
const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
|
|
112
|
+
let isPrivate = false;
|
|
79
113
|
|
|
114
|
+
// --- REFACTOR 2: ADD VERBOSE LOGGING (with User ID) ---
|
|
115
|
+
logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
|
|
116
|
+
|
|
117
|
+
// --- 1. Process History Fetch (Sequentially) ---
|
|
80
118
|
let historyHeader = null;
|
|
81
119
|
let wasHistorySuccess = false;
|
|
82
|
-
let
|
|
83
|
-
let isPrivate = false;
|
|
120
|
+
let proxyUsedForHistory = true;
|
|
84
121
|
|
|
85
122
|
try {
|
|
86
|
-
// --- 1. Prepare History Fetch (if needed) ---
|
|
87
|
-
// (MODIFIED: Use batchManager's cross-invocation cache)
|
|
88
123
|
if (!batchManager.checkAndSetHistoryFetched(userId)) {
|
|
89
|
-
|
|
124
|
+
logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
|
|
90
125
|
historyHeader = await headerManager.selectHeader();
|
|
91
|
-
if (historyHeader) {
|
|
92
|
-
|
|
93
|
-
const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
|
|
94
|
-
historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header });
|
|
126
|
+
if (!historyHeader) {
|
|
127
|
+
logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
|
|
95
128
|
} else {
|
|
96
|
-
|
|
129
|
+
const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
|
|
130
|
+
const options = { headers: historyHeader.header };
|
|
131
|
+
let response;
|
|
132
|
+
|
|
133
|
+
try {
|
|
134
|
+
// --- REFACTOR 3: ADD FALLBACK ---
|
|
135
|
+
logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
|
|
136
|
+
response = await proxyManager.fetch(historyUrl, options);
|
|
137
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
138
|
+
wasHistorySuccess = true;
|
|
139
|
+
|
|
140
|
+
} catch (proxyError) {
|
|
141
|
+
logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
|
|
142
|
+
error: proxyError.message,
|
|
143
|
+
source: 'AppScript'
|
|
144
|
+
});
|
|
145
|
+
proxyUsedForHistory = false;
|
|
146
|
+
|
|
147
|
+
try {
|
|
148
|
+
response = await fetch(historyUrl, options); // Direct node-fetch
|
|
149
|
+
if (!response.ok) {
|
|
150
|
+
const errorText = await response.text();
|
|
151
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
152
|
+
}
|
|
153
|
+
wasHistorySuccess = true; // Fallback succeeded
|
|
154
|
+
|
|
155
|
+
} catch (fallbackError) {
|
|
156
|
+
logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, {
|
|
157
|
+
error: fallbackError.message,
|
|
158
|
+
source: 'eToro/Network'
|
|
159
|
+
});
|
|
160
|
+
wasHistorySuccess = false;
|
|
161
|
+
}
|
|
162
|
+
// --- END REFACTOR 3 ---
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (wasHistorySuccess) {
|
|
166
|
+
logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
|
|
167
|
+
const data = await response.json();
|
|
168
|
+
await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
|
|
169
|
+
}
|
|
97
170
|
}
|
|
98
171
|
} else {
|
|
99
|
-
logger.log('TRACE', `[handleUpdate] History fetch
|
|
172
|
+
logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
|
|
173
|
+
}
|
|
174
|
+
} catch (err) {
|
|
175
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
|
|
176
|
+
wasHistorySuccess = false;
|
|
177
|
+
} finally {
|
|
178
|
+
if (historyHeader && proxyUsedForHistory) {
|
|
179
|
+
headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// --- 2. Process Portfolio Fetches (Sequentially) ---
|
|
184
|
+
logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
|
|
185
|
+
|
|
186
|
+
for (const instId of instrumentsToProcess) {
|
|
187
|
+
if (isPrivate) {
|
|
188
|
+
logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
|
|
189
|
+
break;
|
|
100
190
|
}
|
|
101
191
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
|
|
107
|
-
|
|
108
|
-
const portfolioUrl = userType === 'speculator'
|
|
109
|
-
? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
|
|
110
|
-
: `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
|
|
111
|
-
|
|
112
|
-
portfolioRequests.push({
|
|
113
|
-
instrumentId: instId,
|
|
114
|
-
url: portfolioUrl,
|
|
115
|
-
header: portfolioHeader,
|
|
116
|
-
promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header })
|
|
117
|
-
});
|
|
192
|
+
const portfolioHeader = await headerManager.selectHeader();
|
|
193
|
+
if (!portfolioHeader) {
|
|
194
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
|
|
195
|
+
continue;
|
|
118
196
|
}
|
|
197
|
+
|
|
198
|
+
const portfolioUrl = userType === 'speculator'
|
|
199
|
+
? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
|
|
200
|
+
: `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
|
|
119
201
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
];
|
|
125
|
-
const allResults = await Promise.allSettled(allPromises);
|
|
126
|
-
|
|
127
|
-
// --- 4. Process History Result ---
|
|
128
|
-
let resultIndex = 0;
|
|
129
|
-
if (historyFetchPromise) {
|
|
130
|
-
const historyRes = allResults[resultIndex++];
|
|
131
|
-
if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
|
|
132
|
-
const data = await historyRes.value.json();
|
|
133
|
-
wasHistorySuccess = true;
|
|
134
|
-
await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
|
|
135
|
-
} else {
|
|
136
|
-
logger.log('WARN', `[handleUpdate] History fetch failed for ${userId}`, { error: historyRes.reason || `status ${historyRes.value?.status}` });
|
|
137
|
-
}
|
|
138
|
-
}
|
|
202
|
+
const options = { headers: portfolioHeader.header };
|
|
203
|
+
let response;
|
|
204
|
+
let wasPortfolioSuccess = false;
|
|
205
|
+
let proxyUsedForPortfolio = true;
|
|
139
206
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
207
|
+
try {
|
|
208
|
+
// --- REFACTOR 3: ADD FALLBACK ---
|
|
209
|
+
logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
|
|
210
|
+
response = await proxyManager.fetch(portfolioUrl, options);
|
|
211
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
212
|
+
wasPortfolioSuccess = true;
|
|
213
|
+
|
|
214
|
+
} catch (proxyError) {
|
|
215
|
+
logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
|
|
216
|
+
error: proxyError.message,
|
|
217
|
+
source: 'AppScript'
|
|
218
|
+
});
|
|
219
|
+
proxyUsedForPortfolio = false;
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
response = await fetch(portfolioUrl, options); // Direct node-fetch
|
|
223
|
+
if (!response.ok) {
|
|
224
|
+
const errorText = await response.text();
|
|
225
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
155
226
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
}
|
|
159
|
-
logger.log('
|
|
227
|
+
wasPortfolioSuccess = true; // Fallback succeeded
|
|
228
|
+
|
|
229
|
+
} catch (fallbackError) {
|
|
230
|
+
logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, {
|
|
231
|
+
error: fallbackError.message,
|
|
232
|
+
source: 'eToro/Network'
|
|
233
|
+
});
|
|
234
|
+
wasPortfolioSuccess = false;
|
|
160
235
|
}
|
|
161
|
-
//
|
|
162
|
-
headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
|
|
236
|
+
// --- END REFACTOR 3 ---
|
|
163
237
|
}
|
|
164
238
|
|
|
165
|
-
// ---
|
|
166
|
-
if (
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
239
|
+
// --- 4. Process Portfolio Result (with verbose, raw logging) ---
|
|
240
|
+
if (wasPortfolioSuccess) {
|
|
241
|
+
const body = await response.text();
|
|
242
|
+
if (body.includes("user is PRIVATE")) {
|
|
243
|
+
isPrivate = true;
|
|
244
|
+
logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
|
|
245
|
+
break; // Stop processing more portfolios for this private user
|
|
170
246
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
const portfolioJson = JSON.parse(body);
|
|
250
|
+
await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
|
|
251
|
+
logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`);
|
|
252
|
+
|
|
253
|
+
} catch (parseError) {
|
|
254
|
+
// --- REFACTOR 4: RETURN EXACT PAGE RESPONSE ---
|
|
255
|
+
wasPortfolioSuccess = false; // Mark as failure
|
|
256
|
+
logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
|
|
257
|
+
url: portfolioUrl,
|
|
258
|
+
parseErrorMessage: parseError.message,
|
|
259
|
+
rawResponseText: body // <--- THIS LOGS THE FULL HTML/ERROR RESPONSE
|
|
260
|
+
});
|
|
261
|
+
// --- END REFACTOR 4 ---
|
|
176
262
|
}
|
|
177
|
-
|
|
263
|
+
} else {
|
|
264
|
+
logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (proxyUsedForPortfolio) {
|
|
268
|
+
headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
|
|
178
269
|
}
|
|
270
|
+
} // --- End of sequential portfolio loop ---
|
|
179
271
|
|
|
180
|
-
|
|
272
|
+
// --- 5. Handle Private Users & Timestamps ---
|
|
273
|
+
if (isPrivate) {
|
|
274
|
+
logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
|
|
181
275
|
for (const instrumentId of instrumentsToProcess) {
|
|
182
|
-
await batchManager.
|
|
276
|
+
await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
|
|
183
277
|
}
|
|
184
|
-
|
|
185
|
-
|
|
278
|
+
const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
|
|
279
|
+
for (const instrumentId of instrumentsToProcess) {
|
|
280
|
+
const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
|
|
281
|
+
await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
|
|
186
282
|
}
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
187
285
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
286
|
+
// If not private, update all timestamps
|
|
287
|
+
for (const instrumentId of instrumentsToProcess) {
|
|
288
|
+
await batchManager.updateUserTimestamp(userId, userType, instrumentId);
|
|
289
|
+
}
|
|
290
|
+
if (userType === 'speculator') {
|
|
291
|
+
await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
|
|
192
292
|
}
|
|
293
|
+
|
|
294
|
+
logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
|
|
295
|
+
// 'finally' block for header flushing is handled by the main handler_creator.js
|
|
193
296
|
}
|
|
194
297
|
|
|
195
298
|
module.exports = { handleUpdate, lookupUsernames };
|
|
@@ -1,28 +1,142 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/verify_helpers.js
|
|
3
|
+
* (REFACTORED: `handleVerify` now runs sequentially to prevent throttling)
|
|
4
|
+
* (REFACTORED: `fetchAndVerifyUser` now includes node-fetch fallback)
|
|
5
|
+
* (REFACTORED: Added verbose, user-centric logging for all operations)
|
|
6
|
+
*/
|
|
1
7
|
const { FieldValue } = require('@google-cloud/firestore');
|
|
2
8
|
|
|
3
|
-
|
|
9
|
+
/**
|
|
10
|
+
* (REFACTORED: Now includes taskId for logging, and full fallback logic)
|
|
11
|
+
*/
|
|
12
|
+
async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }, taskId) {
|
|
13
|
+
const cid = user.cid;
|
|
14
|
+
const logPrefix = `[VERIFY/${taskId}/${cid}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
|
|
15
|
+
|
|
4
16
|
const selectedHeader = await headerManager.selectHeader();
|
|
5
|
-
if (!selectedHeader)
|
|
17
|
+
if (!selectedHeader) {
|
|
18
|
+
logger.log('WARN', `${logPrefix} Could not select a header. Skipping user.`);
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
|
|
6
22
|
let wasSuccess = false;
|
|
7
|
-
|
|
23
|
+
let proxyUsed = true;
|
|
24
|
+
|
|
25
|
+
try { // Outer try for the whole operation
|
|
26
|
+
let response;
|
|
27
|
+
const url = `${process.env.ETORO_API_PORTFOLIO_URL}?cid=${cid}`;
|
|
28
|
+
const options = { headers: selectedHeader.header };
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
// --- REFACTOR 3: ADD FALLBACK ---
|
|
32
|
+
logger.log('TRACE', `${logPrefix} Attempting portfolio fetch via AppScript proxy...`);
|
|
33
|
+
response = await proxyManager.fetch(url, options);
|
|
34
|
+
if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
|
|
35
|
+
|
|
36
|
+
} catch (proxyError) {
|
|
37
|
+
logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
|
|
38
|
+
error: proxyError.message,
|
|
39
|
+
source: 'AppScript'
|
|
40
|
+
});
|
|
41
|
+
proxyUsed = false;
|
|
42
|
+
|
|
43
|
+
try {
|
|
44
|
+
response = await fetch(url, options); // Direct node-fetch
|
|
45
|
+
if (!response.ok) {
|
|
46
|
+
const errorText = await response.text();
|
|
47
|
+
throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
|
|
48
|
+
}
|
|
49
|
+
// Fallback succeeded, but we don't set wasSuccess yet,
|
|
50
|
+
// as we still need to parse the body.
|
|
51
|
+
|
|
52
|
+
} catch (fallbackError) {
|
|
53
|
+
logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
|
|
54
|
+
error: fallbackError.message,
|
|
55
|
+
source: 'eToro/Network'
|
|
56
|
+
});
|
|
57
|
+
throw fallbackError; // Throw to be caught by outer try
|
|
58
|
+
}
|
|
59
|
+
// --- END REFACTOR 3 ---
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// --- If we are here, `response` is valid ---
|
|
63
|
+
let portfolioData;
|
|
64
|
+
const body = await response.text();
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
// --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
|
|
68
|
+
portfolioData = JSON.parse(body);
|
|
69
|
+
} catch (parseError) {
|
|
70
|
+
logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
|
|
71
|
+
parseErrorMessage: parseError.message,
|
|
72
|
+
rawResponseText: body
|
|
73
|
+
});
|
|
74
|
+
throw new Error(`Failed to parse JSON for user ${cid}.`);
|
|
75
|
+
}
|
|
76
|
+
// --- END REFACTOR 4 ---
|
|
77
|
+
|
|
78
|
+
// --- Original logic ---
|
|
79
|
+
if (userType === 'speculator') {
|
|
80
|
+
const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID).filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
|
|
81
|
+
if (!instruments.length) {
|
|
82
|
+
logger.log('TRACE', `${logPrefix} Verified user, but not a speculator (no matching assets).`);
|
|
83
|
+
wasSuccess = true; // API call *worked*
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
logger.log('INFO', `${logPrefix} Verified as SPECULATOR.`);
|
|
87
|
+
wasSuccess = true;
|
|
88
|
+
return { type: 'speculator', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
logger.log('INFO', `${logPrefix} Verified as NORMAL user.`);
|
|
8
92
|
wasSuccess = true;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
93
|
+
return { type: 'normal', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
|
|
94
|
+
|
|
95
|
+
} catch (err) {
|
|
96
|
+
// This catches proxy, fallback, or parse errors
|
|
97
|
+
logger.log('WARN', `${logPrefix} Error processing user.`, { errorMessage: err.message });
|
|
98
|
+
wasSuccess = false; // Ensure it's marked as failure
|
|
99
|
+
return null;
|
|
100
|
+
} finally {
|
|
101
|
+
if (selectedHeader && proxyUsed) {
|
|
102
|
+
// Only update performance if the proxy was used
|
|
103
|
+
headerManager.updatePerformance(selectedHeader.id, wasSuccess);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
15
106
|
}
|
|
16
107
|
|
|
108
|
+
/**
|
|
109
|
+
* (REFACTORED: Now runs fetches sequentially)
|
|
110
|
+
*/
|
|
17
111
|
async function handleVerify(task, taskId, { db, logger, ...dependencies }, config) {
|
|
18
112
|
const { users, blockId, instrument, userType } = task;
|
|
19
113
|
const batch = db.batch();
|
|
20
114
|
const speculatorUpdates = {}, normalUpdates = {}, bronzeStates = {}, usernameMap = {};
|
|
21
115
|
const specSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
|
|
22
|
-
|
|
116
|
+
|
|
117
|
+
// --- REFACTOR 1: REMOVE CONCURRENCY ---
|
|
118
|
+
logger.log('INFO', `[VERIFY/${taskId}] Starting sequential verification for ${users.length} users...`);
|
|
119
|
+
const results = [];
|
|
120
|
+
for (const user of users) {
|
|
121
|
+
// Await each user one by one
|
|
122
|
+
const result = await fetchAndVerifyUser(user, { db, logger, ...dependencies }, { ...config, userType }, taskId);
|
|
123
|
+
results.push(result); // Push the actual result (or null)
|
|
124
|
+
}
|
|
125
|
+
logger.log('INFO', `[VERIFY/${taskId}] Sequential verification complete.`);
|
|
126
|
+
// --- END REFACTOR 1 ---
|
|
127
|
+
|
|
23
128
|
let validUserCount = 0;
|
|
24
129
|
results.forEach(r => {
|
|
25
|
-
if (r
|
|
130
|
+
if (r) { // Only process non-null results
|
|
131
|
+
const d = r;
|
|
132
|
+
usernameMap[d.userId] = { username: d.username };
|
|
133
|
+
bronzeStates[d.userId] = d.isBronze;
|
|
134
|
+
validUserCount++;
|
|
135
|
+
if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData;
|
|
136
|
+
else normalUpdates[`users.${d.userId}`] = d.updateData;
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
26
140
|
if (Object.keys(speculatorUpdates).length || Object.keys(normalUpdates).length) {
|
|
27
141
|
const blockRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS : config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS).doc(String(blockId));
|
|
28
142
|
batch.set(blockRef, userType === 'speculator' ? speculatorUpdates : normalUpdates, { merge: true });
|
|
@@ -35,9 +149,10 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
|
|
|
35
149
|
for (const cid in usernameMap) { const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
|
|
36
150
|
if (!shardedUpdates[shardId]) { shardedUpdates[shardId] = {}; } shardedUpdates[shardId][cid] = usernameMap[cid]; }
|
|
37
151
|
for (const shardId in shardedUpdates) { const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId); batch.set(mapRef, shardedUpdates[shardId], { merge: true }); }
|
|
38
|
-
logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
|
|
152
|
+
logger.log('INFO', `[VERIFY/${taskId}] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
|
|
153
|
+
|
|
39
154
|
await batch.commit();
|
|
40
|
-
if (validUserCount) logger.log('
|
|
155
|
+
if (validUserCount) logger.log('SUCCESS', `[VERIFY/${taskId}] Verified and stored ${validUserCount} new ${userType} users.`);
|
|
41
156
|
}
|
|
42
157
|
|
|
43
158
|
module.exports = { handleVerify };
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
|
|
3
|
-
* (
|
|
4
|
-
* (MODIFIED: To run all update tasks in parallel with a concurrency limit)
|
|
5
|
-
* (MODIFIED: To use a SINGLE parallel work pool for ALL tasks)
|
|
6
|
-
* (MODIFIED: To remove local history cache set)
|
|
3
|
+
* (REFACTORED: Concurrency limit set to 1 to prevent API throttling)
|
|
7
4
|
*/
|
|
8
5
|
|
|
9
6
|
/**
|
|
@@ -15,7 +12,7 @@
|
|
|
15
12
|
const { handleDiscover } = require('../helpers/discover_helpers');
|
|
16
13
|
const { handleVerify } = require('../helpers/verify_helpers');
|
|
17
14
|
const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
|
|
18
|
-
const pLimit = require('p-limit');
|
|
15
|
+
const pLimit = require('p-limit');
|
|
19
16
|
|
|
20
17
|
/**
|
|
21
18
|
* Parses Pub/Sub message into task array.
|
|
@@ -55,15 +52,16 @@ async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config
|
|
|
55
52
|
|
|
56
53
|
/**
|
|
57
54
|
* Executes all tasks.
|
|
58
|
-
* (
|
|
55
|
+
* (REFACTORED: Concurrency limit set to 1)
|
|
59
56
|
*/
|
|
60
57
|
async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
|
|
61
|
-
const { logger, batchManager } = dependencies;
|
|
58
|
+
const { logger, batchManager } = dependencies;
|
|
62
59
|
|
|
63
|
-
//
|
|
60
|
+
// --- REFACTOR 1: REMOVE CONCURRENCY ---
|
|
61
|
+
// Set limit to 1 to serialize all tasks and prevent AppScript throttling.
|
|
62
|
+
const limit = pLimit(1);
|
|
63
|
+
// --- END REFACTOR 1 ---
|
|
64
64
|
|
|
65
|
-
// Create one unified parallel pool
|
|
66
|
-
const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 3); // TODO Work out what the optimal concurrency is
|
|
67
65
|
const allTaskPromises = [];
|
|
68
66
|
let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
|
|
69
67
|
|
|
@@ -94,8 +92,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
|
|
|
94
92
|
const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
|
|
95
93
|
allTaskPromises.push(
|
|
96
94
|
limit(() =>
|
|
97
|
-
|
|
98
|
-
handleUpdate(task, subTaskId, dependencies, config, username) // <--- REMOVED historyFetchedForUser
|
|
95
|
+
handleUpdate(task, subTaskId, dependencies, config, username)
|
|
99
96
|
.then(() => taskCounters.update++)
|
|
100
97
|
.catch(err => {
|
|
101
98
|
logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
|
|
@@ -105,7 +102,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
|
|
|
105
102
|
);
|
|
106
103
|
}
|
|
107
104
|
|
|
108
|
-
// 3. Wait for ALL tasks to complete
|
|
105
|
+
// 3. Wait for ALL tasks to complete (now sequentially)
|
|
109
106
|
await Promise.all(allTaskPromises);
|
|
110
107
|
|
|
111
108
|
// 4. Log final summary
|