bulltrackers-module 1.0.629 → 1.0.631
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/alert-system/helpers/alert_helpers.js +69 -77
- package/functions/alert-system/index.js +19 -29
- package/functions/api-v2/helpers/notification_helpers.js +187 -0
- package/functions/computation-system/helpers/computation_worker.js +1 -1
- package/functions/task-engine/helpers/popular_investor_helpers.js +11 -7
- package/index.js +0 -5
- package/package.json +1 -2
- package/functions/old-generic-api/admin-api/index.js +0 -895
- package/functions/old-generic-api/helpers/api_helpers.js +0 -457
- package/functions/old-generic-api/index.js +0 -204
- package/functions/old-generic-api/user-api/helpers/alerts/alert_helpers.js +0 -355
- package/functions/old-generic-api/user-api/helpers/alerts/subscription_helpers.js +0 -327
- package/functions/old-generic-api/user-api/helpers/alerts/test_alert_helpers.js +0 -212
- package/functions/old-generic-api/user-api/helpers/collection_helpers.js +0 -193
- package/functions/old-generic-api/user-api/helpers/core/compression_helpers.js +0 -68
- package/functions/old-generic-api/user-api/helpers/core/data_lookup_helpers.js +0 -256
- package/functions/old-generic-api/user-api/helpers/core/path_resolution_helpers.js +0 -640
- package/functions/old-generic-api/user-api/helpers/core/user_status_helpers.js +0 -195
- package/functions/old-generic-api/user-api/helpers/data/computation_helpers.js +0 -503
- package/functions/old-generic-api/user-api/helpers/data/instrument_helpers.js +0 -55
- package/functions/old-generic-api/user-api/helpers/data/portfolio_helpers.js +0 -245
- package/functions/old-generic-api/user-api/helpers/data/social_helpers.js +0 -174
- package/functions/old-generic-api/user-api/helpers/data_helpers.js +0 -87
- package/functions/old-generic-api/user-api/helpers/dev/dev_helpers.js +0 -336
- package/functions/old-generic-api/user-api/helpers/fetch/on_demand_fetch_helpers.js +0 -615
- package/functions/old-generic-api/user-api/helpers/metrics/personalized_metrics_helpers.js +0 -231
- package/functions/old-generic-api/user-api/helpers/notifications/notification_helpers.js +0 -641
- package/functions/old-generic-api/user-api/helpers/profile/pi_profile_helpers.js +0 -182
- package/functions/old-generic-api/user-api/helpers/profile/profile_view_helpers.js +0 -137
- package/functions/old-generic-api/user-api/helpers/profile/user_profile_helpers.js +0 -190
- package/functions/old-generic-api/user-api/helpers/recommendations/recommendation_helpers.js +0 -66
- package/functions/old-generic-api/user-api/helpers/reviews/review_helpers.js +0 -550
- package/functions/old-generic-api/user-api/helpers/rootdata/rootdata_aggregation_helpers.js +0 -378
- package/functions/old-generic-api/user-api/helpers/search/pi_request_helpers.js +0 -295
- package/functions/old-generic-api/user-api/helpers/search/pi_search_helpers.js +0 -162
- package/functions/old-generic-api/user-api/helpers/sync/user_sync_helpers.js +0 -677
- package/functions/old-generic-api/user-api/helpers/verification/verification_helpers.js +0 -323
- package/functions/old-generic-api/user-api/helpers/watchlist/watchlist_analytics_helpers.js +0 -96
- package/functions/old-generic-api/user-api/helpers/watchlist/watchlist_data_helpers.js +0 -141
- package/functions/old-generic-api/user-api/helpers/watchlist/watchlist_generation_helpers.js +0 -310
- package/functions/old-generic-api/user-api/helpers/watchlist/watchlist_management_helpers.js +0 -829
- package/functions/old-generic-api/user-api/index.js +0 -109
|
@@ -1,677 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview User Sync Request Helpers
|
|
3
|
-
* Allows users to request on-demand sync for their own profile or Popular Investor profiles
|
|
4
|
-
* Rate limiting: 1 update per user per 6 hours (global, not per-requester)
|
|
5
|
-
* Developer accounts bypass rate limits
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
const { FieldValue } = require('@google-cloud/firestore');
|
|
9
|
-
const crypto = require('crypto');
|
|
10
|
-
|
|
11
|
-
const RATE_LIMIT_HOURS = 6;
|
|
12
|
-
const RATE_LIMIT_MS = RATE_LIMIT_HOURS * 60 * 60 * 1000;
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Request on-demand sync for a user (signed-in user or Popular Investor)
|
|
16
|
-
* POST /user/:userCid/sync
|
|
17
|
-
*/
|
|
18
|
-
async function requestUserSync(req, res, dependencies, config) {
|
|
19
|
-
const { db, logger, pubsub } = dependencies;
|
|
20
|
-
const { userCid: targetUserCid } = req.params;
|
|
21
|
-
|
|
22
|
-
// Get requesting userCid from query
|
|
23
|
-
const requestingUserCid = req.query?.userCid;
|
|
24
|
-
|
|
25
|
-
if (!requestingUserCid) {
|
|
26
|
-
return res.status(400).json({
|
|
27
|
-
success: false,
|
|
28
|
-
error: "Missing userCid",
|
|
29
|
-
message: "Please provide userCid as a query parameter"
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
// Check the referrer or source header to determine if this is from PI page or profile page
|
|
34
|
-
const referrer = req.headers.referer || req.headers.referrer || '';
|
|
35
|
-
const sourcePage = req.query?.sourcePage || req.body?.sourcePage || '';
|
|
36
|
-
|
|
37
|
-
// Determine user type based on where the request came from
|
|
38
|
-
// If from /popular-investors/{cid}, it's a PI
|
|
39
|
-
// If from /profile, it's a signed-in user
|
|
40
|
-
const isPI = referrer.includes('/popular-investors/') || sourcePage === 'popular-investor';
|
|
41
|
-
|
|
42
|
-
// Check for dev override impersonation
|
|
43
|
-
const { getEffectiveCid, getDevOverride } = require('../dev/dev_helpers');
|
|
44
|
-
const effectiveCid = await getEffectiveCid(db, requestingUserCid, config, logger);
|
|
45
|
-
const devOverride = await getDevOverride(db, requestingUserCid, config, logger);
|
|
46
|
-
const isImpersonating = devOverride && devOverride.enabled && devOverride.impersonateCid && effectiveCid !== Number(requestingUserCid);
|
|
47
|
-
|
|
48
|
-
const targetCidNum = Number(targetUserCid);
|
|
49
|
-
if (isNaN(targetCidNum) || targetCidNum <= 0) {
|
|
50
|
-
return res.status(400).json({
|
|
51
|
-
success: false,
|
|
52
|
-
error: "Invalid user CID",
|
|
53
|
-
message: "Please provide a valid user CID"
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
try {
|
|
58
|
-
// Check if this is a developer account (bypass rate limits for developers)
|
|
59
|
-
const { isDeveloperAccount } = require('../dev/dev_helpers');
|
|
60
|
-
const isDeveloper = isDeveloperAccount(requestingUserCid);
|
|
61
|
-
|
|
62
|
-
let rateLimitCheck = { allowed: true }; // Default to allowed for developers
|
|
63
|
-
|
|
64
|
-
if (!isDeveloper) {
|
|
65
|
-
// Check global rate limit (applies to target user, not requester)
|
|
66
|
-
rateLimitCheck = await checkRateLimits(db, targetCidNum, logger);
|
|
67
|
-
|
|
68
|
-
if (!rateLimitCheck.allowed) {
|
|
69
|
-
return res.status(429).json({
|
|
70
|
-
success: false,
|
|
71
|
-
error: "Rate limit exceeded",
|
|
72
|
-
message: rateLimitCheck.message,
|
|
73
|
-
rateLimit: {
|
|
74
|
-
canRequestAgainAt: rateLimitCheck.canRequestAgainAt,
|
|
75
|
-
lastRequestedAt: rateLimitCheck.lastRequestedAt
|
|
76
|
-
}
|
|
77
|
-
});
|
|
78
|
-
}
|
|
79
|
-
} else {
|
|
80
|
-
logger.log('INFO', `[requestUserSync] Developer account ${requestingUserCid} bypassing rate limits${isImpersonating ? ` (impersonating ${effectiveCid})` : ''}`);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// Get username based on user type
|
|
84
|
-
let username = null;
|
|
85
|
-
if (isPI) {
|
|
86
|
-
// For PIs, get username from rankings
|
|
87
|
-
const { checkIfUserIsPI } = require('../core/user_status_helpers');
|
|
88
|
-
const rankEntry = await checkIfUserIsPI(db, targetCidNum, config, logger);
|
|
89
|
-
if (rankEntry && (rankEntry.UserName || rankEntry.username)) {
|
|
90
|
-
username = rankEntry.UserName || rankEntry.username;
|
|
91
|
-
}
|
|
92
|
-
} else {
|
|
93
|
-
// For signed-in users, try verification data
|
|
94
|
-
const signedInUsersCollection = config.signedInUsersCollection || 'signed_in_users';
|
|
95
|
-
const userDoc = await db.collection(signedInUsersCollection).doc(String(targetCidNum)).get();
|
|
96
|
-
if (userDoc.exists) {
|
|
97
|
-
const userData = userDoc.data();
|
|
98
|
-
username = userData.username || userData.verification?.username || null;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
if (!username) {
|
|
103
|
-
username = String(targetCidNum); // Fallback to CID
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// Create request document
|
|
107
|
-
const requestId = `sync_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
|
|
108
|
-
const now = new Date();
|
|
109
|
-
|
|
110
|
-
// Store request
|
|
111
|
-
const requestRef = db.collection('user_sync_requests')
|
|
112
|
-
.doc(String(targetCidNum))
|
|
113
|
-
.collection('requests')
|
|
114
|
-
.doc(requestId);
|
|
115
|
-
|
|
116
|
-
await requestRef.set({
|
|
117
|
-
requestId,
|
|
118
|
-
targetUserCid: targetCidNum,
|
|
119
|
-
requestedBy: Number(requestingUserCid),
|
|
120
|
-
effectiveRequestedBy: effectiveCid,
|
|
121
|
-
username,
|
|
122
|
-
userType: isPI ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER',
|
|
123
|
-
status: 'queued',
|
|
124
|
-
isImpersonating: isImpersonating || false,
|
|
125
|
-
createdAt: FieldValue.serverTimestamp(),
|
|
126
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
// Update global rate limit (for target user)
|
|
130
|
-
const globalRef = db.collection('user_sync_requests')
|
|
131
|
-
.doc(String(targetCidNum))
|
|
132
|
-
.collection('global')
|
|
133
|
-
.doc('latest');
|
|
134
|
-
|
|
135
|
-
await globalRef.set({
|
|
136
|
-
targetUserCid: targetCidNum,
|
|
137
|
-
lastRequestedAt: FieldValue.serverTimestamp(),
|
|
138
|
-
lastRequestedBy: Number(requestingUserCid),
|
|
139
|
-
totalRequests: FieldValue.increment(1),
|
|
140
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
141
|
-
}, { merge: true });
|
|
142
|
-
|
|
143
|
-
// Publish to task engine - use on-demand topic for API requests
|
|
144
|
-
const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH_ONDEMAND ||
|
|
145
|
-
config.pubsubTopicUserFetchOnDemand ||
|
|
146
|
-
'etoro-user-fetch-topic-ondemand';
|
|
147
|
-
const topic = pubsub.topic(topicName);
|
|
148
|
-
|
|
149
|
-
const message = {
|
|
150
|
-
type: isPI ? 'POPULAR_INVESTOR_UPDATE' : 'ON_DEMAND_USER_UPDATE',
|
|
151
|
-
cid: targetCidNum,
|
|
152
|
-
username: username,
|
|
153
|
-
priority: 'high',
|
|
154
|
-
source: 'on_demand_sync',
|
|
155
|
-
requestId,
|
|
156
|
-
requestedBy: Number(requestingUserCid),
|
|
157
|
-
effectiveRequestedBy: effectiveCid,
|
|
158
|
-
data: {
|
|
159
|
-
includeSocial: true, // Always include social data for on-demand syncs
|
|
160
|
-
since: new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString() // Last 7 days
|
|
161
|
-
},
|
|
162
|
-
metadata: {
|
|
163
|
-
onDemand: true,
|
|
164
|
-
targetCid: targetCidNum, // Target specific user for optimization
|
|
165
|
-
requestedAt: now.toISOString(),
|
|
166
|
-
isImpersonating: isImpersonating || false,
|
|
167
|
-
requestingUserCid: Number(requestingUserCid), // Store for notifications
|
|
168
|
-
userType: isPI ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER' // Explicitly set userType in metadata
|
|
169
|
-
}
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
const messageId = await topic.publishMessage({
|
|
173
|
-
data: Buffer.from(JSON.stringify(message))
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
// Update request with message ID
|
|
177
|
-
await requestRef.update({
|
|
178
|
-
taskMessageId: messageId,
|
|
179
|
-
status: 'dispatched',
|
|
180
|
-
dispatchedAt: FieldValue.serverTimestamp(),
|
|
181
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
logger.log('INFO', `[requestUserSync] User ${requestingUserCid}${isImpersonating ? ` (impersonating ${effectiveCid})` : ''} requested sync for ${isPI ? 'PI' : 'signed-in user'} ${targetCidNum} (${username}). Request ID: ${requestId}`);
|
|
185
|
-
|
|
186
|
-
return res.status(200).json({
|
|
187
|
-
success: true,
|
|
188
|
-
requestId,
|
|
189
|
-
status: 'dispatched',
|
|
190
|
-
message: 'Sync request queued successfully. Data will be updated shortly.',
|
|
191
|
-
estimatedTime: '2-5 minutes',
|
|
192
|
-
rateLimit: {
|
|
193
|
-
canRequestAgainAt: new Date(Date.now() + RATE_LIMIT_MS).toISOString()
|
|
194
|
-
}
|
|
195
|
-
});
|
|
196
|
-
|
|
197
|
-
} catch (error) {
|
|
198
|
-
logger.log('ERROR', `[requestUserSync] Error requesting sync for user ${targetCidNum}`, error);
|
|
199
|
-
return res.status(500).json({
|
|
200
|
-
success: false,
|
|
201
|
-
error: "Internal server error",
|
|
202
|
-
message: error.message
|
|
203
|
-
});
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/**
|
|
208
|
-
* Get sync status for a user
|
|
209
|
-
* GET /user/:userCid/sync-status
|
|
210
|
-
*/
|
|
211
|
-
async function getUserSyncStatus(req, res, dependencies, config) {
|
|
212
|
-
const { db, logger } = dependencies;
|
|
213
|
-
const { userCid: targetUserCid } = req.params;
|
|
214
|
-
const requestingUserCid = req.query?.userCid; // Optional - for rate limit info
|
|
215
|
-
|
|
216
|
-
const targetCidNum = Number(targetUserCid);
|
|
217
|
-
if (isNaN(targetCidNum) || targetCidNum <= 0) {
|
|
218
|
-
return res.status(400).json({
|
|
219
|
-
success: false,
|
|
220
|
-
error: "Invalid user CID"
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
try {
|
|
225
|
-
// Check for latest request
|
|
226
|
-
const requestsRef = db.collection('user_sync_requests')
|
|
227
|
-
.doc(String(targetCidNum))
|
|
228
|
-
.collection('requests')
|
|
229
|
-
.orderBy('createdAt', 'desc')
|
|
230
|
-
.limit(1);
|
|
231
|
-
|
|
232
|
-
const requestsSnapshot = await requestsRef.get();
|
|
233
|
-
|
|
234
|
-
if (!requestsSnapshot.empty) {
|
|
235
|
-
const latestRequest = requestsSnapshot.docs[0].data();
|
|
236
|
-
let status = latestRequest.status || 'queued';
|
|
237
|
-
const userType = latestRequest.userType || 'SIGNED_IN_USER'; // Default to signed-in user
|
|
238
|
-
|
|
239
|
-
// If status is 'indexing' or 'computing', check if computation results are now available
|
|
240
|
-
if (status === 'indexing' || status === 'computing') {
|
|
241
|
-
const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
|
|
242
|
-
const resultsSub = config.resultsSubcollection || 'results';
|
|
243
|
-
const compsSub = config.computationsSubcollection || 'computations';
|
|
244
|
-
|
|
245
|
-
// Determine category and computation name based on user type
|
|
246
|
-
// NOTE: ResultCommitter ignores category metadata if computation is in a non-core folder
|
|
247
|
-
// SignedInUserProfileMetrics is in popular-investor folder, so it's stored in popular-investor category
|
|
248
|
-
let category, computationName;
|
|
249
|
-
if (userType === 'POPULAR_INVESTOR') {
|
|
250
|
-
category = 'popular-investor';
|
|
251
|
-
computationName = 'PopularInvestorProfileMetrics';
|
|
252
|
-
} else {
|
|
253
|
-
// SignedInUserProfileMetrics is in popular-investor folder, so stored in popular-investor category
|
|
254
|
-
category = 'popular-investor';
|
|
255
|
-
computationName = 'SignedInUserProfileMetrics';
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// Check today and yesterday for computation results
|
|
259
|
-
const checkDate = new Date();
|
|
260
|
-
for (let i = 0; i < 2; i++) {
|
|
261
|
-
const dateStr = new Date(checkDate);
|
|
262
|
-
dateStr.setDate(checkDate.getDate() - i);
|
|
263
|
-
const dateStrFormatted = dateStr.toISOString().split('T')[0];
|
|
264
|
-
|
|
265
|
-
const docRef = db.collection(insightsCollection)
|
|
266
|
-
.doc(dateStrFormatted)
|
|
267
|
-
.collection(resultsSub)
|
|
268
|
-
.doc(category)
|
|
269
|
-
.collection(compsSub)
|
|
270
|
-
.doc(computationName);
|
|
271
|
-
|
|
272
|
-
const doc = await docRef.get();
|
|
273
|
-
if (doc.exists) {
|
|
274
|
-
const docData = doc.data();
|
|
275
|
-
let mergedData = null;
|
|
276
|
-
|
|
277
|
-
// Check if data is sharded
|
|
278
|
-
if (docData._sharded === true && docData._shardCount) {
|
|
279
|
-
// Data is stored in shards - read all shards and merge
|
|
280
|
-
const shardsCol = docRef.collection('_shards');
|
|
281
|
-
const shardsSnapshot = await shardsCol.get();
|
|
282
|
-
|
|
283
|
-
if (!shardsSnapshot.empty) {
|
|
284
|
-
mergedData = {};
|
|
285
|
-
for (const shardDoc of shardsSnapshot.docs) {
|
|
286
|
-
const shardData = shardDoc.data();
|
|
287
|
-
Object.assign(mergedData, shardData);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
} else {
|
|
291
|
-
// Data is in the main document (compressed or not)
|
|
292
|
-
const { tryDecompress } = require('../data_helpers');
|
|
293
|
-
mergedData = tryDecompress(docData);
|
|
294
|
-
|
|
295
|
-
// Handle string decompression result
|
|
296
|
-
if (typeof mergedData === 'string') {
|
|
297
|
-
try {
|
|
298
|
-
mergedData = JSON.parse(mergedData);
|
|
299
|
-
} catch (e) {
|
|
300
|
-
logger.log('WARN', `[getUserSyncStatus] Failed to parse decompressed string for date ${dateStrFormatted}:`, e.message);
|
|
301
|
-
mergedData = null;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
if (mergedData && typeof mergedData === 'object' && mergedData[String(targetCidNum)]) {
|
|
307
|
-
// Computation completed! Update status
|
|
308
|
-
status = 'completed';
|
|
309
|
-
await requestsSnapshot.docs[0].ref.update({
|
|
310
|
-
status: 'completed',
|
|
311
|
-
completedAt: FieldValue.serverTimestamp(),
|
|
312
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
313
|
-
});
|
|
314
|
-
logger.log('INFO', `[getUserSyncStatus] Computation completed for user ${targetCidNum} (${userType}). Found results in date ${dateStrFormatted}`);
|
|
315
|
-
break;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
// Check if request is stale (stuck in processing state for too long)
|
|
322
|
-
// Set to 2 minutes to prevent indefinite polling when computation system crashes
|
|
323
|
-
const STALE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
|
|
324
|
-
const processingStates = ['processing', 'dispatched', 'indexing', 'computing', 'queued'];
|
|
325
|
-
const isProcessingState = processingStates.includes(status);
|
|
326
|
-
|
|
327
|
-
let isStale = false;
|
|
328
|
-
if (isProcessingState) {
|
|
329
|
-
const now = Date.now();
|
|
330
|
-
const createdAt = latestRequest.createdAt?.toDate?.()?.getTime() ||
|
|
331
|
-
latestRequest.createdAt?.toMillis?.() || null;
|
|
332
|
-
const dispatchedAt = latestRequest.dispatchedAt?.toDate?.()?.getTime() ||
|
|
333
|
-
latestRequest.dispatchedAt?.toMillis?.() || null;
|
|
334
|
-
const updatedAt = latestRequest.updatedAt?.toDate?.()?.getTime() ||
|
|
335
|
-
latestRequest.updatedAt?.toMillis?.() || null;
|
|
336
|
-
|
|
337
|
-
// Use the most recent timestamp to determine age
|
|
338
|
-
const referenceTime = dispatchedAt || createdAt || updatedAt;
|
|
339
|
-
|
|
340
|
-
if (referenceTime && (now - referenceTime) > STALE_THRESHOLD_MS) {
|
|
341
|
-
// Before marking as stale, do one final check for computation results
|
|
342
|
-
// Sometimes computation completes but status wasn't updated
|
|
343
|
-
const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
|
|
344
|
-
const resultsSub = config.resultsSubcollection || 'results';
|
|
345
|
-
const compsSub = config.computationsSubcollection || 'computations';
|
|
346
|
-
const finalCheck = await checkComputationResults(db, targetCidNum, userType, insightsCollection, resultsSub, compsSub, logger);
|
|
347
|
-
if (!finalCheck) {
|
|
348
|
-
isStale = true;
|
|
349
|
-
logger.log('WARN', `[getUserSyncStatus] Detected stale request ${latestRequest.requestId} for user ${targetCidNum}. Status: ${status}, Age: ${Math.round((now - referenceTime) / 60000)} minutes`);
|
|
350
|
-
} else {
|
|
351
|
-
// Found results, update status to completed
|
|
352
|
-
status = 'completed';
|
|
353
|
-
await requestsSnapshot.docs[0].ref.update({
|
|
354
|
-
status: 'completed',
|
|
355
|
-
completedAt: FieldValue.serverTimestamp(),
|
|
356
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
357
|
-
});
|
|
358
|
-
logger.log('INFO', `[getUserSyncStatus] Found computation results on stale check, marked as completed`);
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
// If stale, mark as failed to stop polling
|
|
364
|
-
if (isStale) {
|
|
365
|
-
status = 'failed';
|
|
366
|
-
const requestDocRef = requestsSnapshot.docs[0].ref;
|
|
367
|
-
try {
|
|
368
|
-
await requestDocRef.update({
|
|
369
|
-
status: 'failed',
|
|
370
|
-
error: 'Request timed out - task may have failed to process. Please try again.',
|
|
371
|
-
failedAt: FieldValue.serverTimestamp(),
|
|
372
|
-
updatedAt: FieldValue.serverTimestamp()
|
|
373
|
-
});
|
|
374
|
-
logger.log('INFO', `[getUserSyncStatus] Marked stale request ${latestRequest.requestId} as failed`);
|
|
375
|
-
} catch (updateErr) {
|
|
376
|
-
logger.log('WARN', `[getUserSyncStatus] Failed to update stale request status`, updateErr);
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// Re-fetch request if we updated it to get the latest completedAt
|
|
381
|
-
let completedAt = latestRequest.completedAt?.toDate?.()?.toISOString() || null;
|
|
382
|
-
if (status === 'completed' && !completedAt) {
|
|
383
|
-
// If we just updated to completed, re-fetch to get the server timestamp
|
|
384
|
-
const updatedDoc = await requestsSnapshot.docs[0].ref.get();
|
|
385
|
-
if (updatedDoc.exists) {
|
|
386
|
-
const updatedData = updatedDoc.data();
|
|
387
|
-
completedAt = updatedData.completedAt?.toDate?.()?.toISOString() || null;
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
const response = {
|
|
392
|
-
success: true,
|
|
393
|
-
status,
|
|
394
|
-
requestId: latestRequest.requestId,
|
|
395
|
-
startedAt: latestRequest.startedAt?.toDate?.()?.toISOString() || null,
|
|
396
|
-
createdAt: latestRequest.createdAt?.toDate?.()?.toISOString() || null,
|
|
397
|
-
dispatchedAt: latestRequest.dispatchedAt?.toDate?.()?.toISOString() || null,
|
|
398
|
-
completedAt: completedAt,
|
|
399
|
-
estimatedCompletion: latestRequest.dispatchedAt
|
|
400
|
-
? new Date(new Date(latestRequest.dispatchedAt.toDate()).getTime() + 5 * 60 * 1000).toISOString() // 5 min estimate
|
|
401
|
-
: null
|
|
402
|
-
};
|
|
403
|
-
|
|
404
|
-
// Include error details if status is failed
|
|
405
|
-
if (status === 'failed') {
|
|
406
|
-
response.error = isStale
|
|
407
|
-
? 'Request timed out - task may have failed to process. Please try again.'
|
|
408
|
-
: (latestRequest.error || 'Unknown error occurred');
|
|
409
|
-
response.failedAt = latestRequest.failedAt?.toDate?.()?.toISOString() ||
|
|
410
|
-
(isStale ? new Date().toISOString() : null);
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
// Include raw data status if processing
|
|
414
|
-
if (status === 'processing' || status === 'indexing' || status === 'computing') {
|
|
415
|
-
response.rawDataStoredAt = latestRequest.rawDataStoredAt?.toDate?.()?.toISOString() || null;
|
|
416
|
-
response.message = status === 'indexing' ? 'Raw data stored, indexing data...' :
|
|
417
|
-
status === 'computing' ? 'Raw data stored, computation in progress...' :
|
|
418
|
-
'Processing data...';
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
return res.status(200).json(response);
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
// No request found, check if user can request
|
|
425
|
-
let canRequest = true;
|
|
426
|
-
let rateLimitInfo = null;
|
|
427
|
-
let lastUpdatedInfo = null;
|
|
428
|
-
|
|
429
|
-
// Check last updated times for the target user
|
|
430
|
-
try {
|
|
431
|
-
const { collectionRegistry } = dependencies;
|
|
432
|
-
const HOURS_THRESHOLD = 24;
|
|
433
|
-
const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
|
|
434
|
-
const now = Date.now();
|
|
435
|
-
|
|
436
|
-
// Determine user type (PI or signed-in user)
|
|
437
|
-
const referrer = req.headers.referer || req.headers.referrer || '';
|
|
438
|
-
const sourcePage = req.query?.sourcePage || '';
|
|
439
|
-
const isPI = referrer.includes('/popular-investors/') || sourcePage === 'popular-investor';
|
|
440
|
-
|
|
441
|
-
// [FIX] Get user document path (not lastUpdated subpath)
|
|
442
|
-
let userDocPath;
|
|
443
|
-
if (collectionRegistry && collectionRegistry.getCollectionPath) {
|
|
444
|
-
try {
|
|
445
|
-
if (isPI) {
|
|
446
|
-
// Use PI document path: PopularInvestors/{cid}
|
|
447
|
-
userDocPath = `PopularInvestors/${targetCidNum}`;
|
|
448
|
-
} else {
|
|
449
|
-
// Use signed-in user document path: SignedInUsers/{cid}
|
|
450
|
-
userDocPath = `SignedInUsers/${targetCidNum}`;
|
|
451
|
-
}
|
|
452
|
-
} catch (err) {
|
|
453
|
-
logger.log('WARN', `[getUserSyncStatus] Failed to get user document path: ${err.message}`);
|
|
454
|
-
}
|
|
455
|
-
} else {
|
|
456
|
-
// Fallback
|
|
457
|
-
userDocPath = isPI ? `PopularInvestors/${targetCidNum}` : `SignedInUsers/${targetCidNum}`;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
if (userDocPath) {
|
|
461
|
-
const userDocRef = db.doc(userDocPath);
|
|
462
|
-
const userDoc = await userDocRef.get();
|
|
463
|
-
|
|
464
|
-
if (userDoc.exists) {
|
|
465
|
-
const userData = userDoc.data();
|
|
466
|
-
// [FIX] Access nested lastUpdated fields
|
|
467
|
-
const lastUpdatedData = userData?.lastUpdated || {};
|
|
468
|
-
const dataTypes = ['portfolio', 'tradeHistory', 'socialPosts'];
|
|
469
|
-
const allRecent = dataTypes.every(dataType => {
|
|
470
|
-
const lastUpdated = lastUpdatedData[dataType];
|
|
471
|
-
if (!lastUpdated) return false;
|
|
472
|
-
|
|
473
|
-
const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
|
|
474
|
-
return lastUpdatedMs && (now - lastUpdatedMs) < thresholdMs;
|
|
475
|
-
});
|
|
476
|
-
|
|
477
|
-
if (allRecent) {
|
|
478
|
-
// All data types were updated recently - disable sync
|
|
479
|
-
canRequest = false;
|
|
480
|
-
const oldestUpdate = Math.min(...dataTypes.map(dataType => {
|
|
481
|
-
const lastUpdated = lastUpdatedData[dataType];
|
|
482
|
-
const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
|
|
483
|
-
return lastUpdatedMs || 0;
|
|
484
|
-
}).filter(ms => ms > 0));
|
|
485
|
-
|
|
486
|
-
const canRequestAgainAt = new Date(oldestUpdate + thresholdMs).toISOString();
|
|
487
|
-
lastUpdatedInfo = {
|
|
488
|
-
allDataTypesRecent: true,
|
|
489
|
-
canRequestAgainAt,
|
|
490
|
-
lastUpdated: {
|
|
491
|
-
portfolio: lastUpdatedData.portfolio?.toDate?.()?.toISOString() || null,
|
|
492
|
-
tradeHistory: lastUpdatedData.tradeHistory?.toDate?.()?.toISOString() || null,
|
|
493
|
-
socialPosts: lastUpdatedData.socialPosts?.toDate?.()?.toISOString() || null
|
|
494
|
-
}
|
|
495
|
-
};
|
|
496
|
-
} else {
|
|
497
|
-
// Some data types need updating - allow sync
|
|
498
|
-
lastUpdatedInfo = {
|
|
499
|
-
allDataTypesRecent: false,
|
|
500
|
-
lastUpdated: {
|
|
501
|
-
portfolio: lastUpdatedData.portfolio?.toDate?.()?.toISOString() || null,
|
|
502
|
-
tradeHistory: lastUpdatedData.tradeHistory?.toDate?.()?.toISOString() || null,
|
|
503
|
-
socialPosts: lastUpdatedData.socialPosts?.toDate?.()?.toISOString() || null
|
|
504
|
-
}
|
|
505
|
-
};
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
} catch (lastUpdatedError) {
|
|
510
|
-
logger.log('WARN', `[getUserSyncStatus] Error checking last updated times: ${lastUpdatedError.message}`);
|
|
511
|
-
// Non-critical, continue with rate limit check
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
if (requestingUserCid) {
|
|
515
|
-
// Check if this is a developer account (bypass rate limits for developers)
|
|
516
|
-
const { isDeveloperAccount } = require('../dev/dev_helpers');
|
|
517
|
-
const isDeveloper = isDeveloperAccount(requestingUserCid);
|
|
518
|
-
|
|
519
|
-
if (isDeveloper) {
|
|
520
|
-
// Developer accounts bypass rate limits (but still respect last updated check)
|
|
521
|
-
rateLimitInfo = {
|
|
522
|
-
bypassed: true,
|
|
523
|
-
reason: 'developer_account'
|
|
524
|
-
};
|
|
525
|
-
} else {
|
|
526
|
-
const rateLimitCheck = await checkRateLimits(db, targetCidNum, logger);
|
|
527
|
-
// Combine rate limit and last updated checks - both must pass
|
|
528
|
-
if (!rateLimitCheck.allowed) {
|
|
529
|
-
canRequest = false;
|
|
530
|
-
}
|
|
531
|
-
rateLimitInfo = {
|
|
532
|
-
canRequestAgainAt: rateLimitCheck.canRequestAgainAt
|
|
533
|
-
};
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
return res.status(200).json({
|
|
538
|
-
success: true,
|
|
539
|
-
status: 'not_requested',
|
|
540
|
-
canRequest,
|
|
541
|
-
rateLimit: rateLimitInfo,
|
|
542
|
-
lastUpdated: lastUpdatedInfo
|
|
543
|
-
});
|
|
544
|
-
|
|
545
|
-
} catch (error) {
|
|
546
|
-
logger.log('ERROR', `[getUserSyncStatus] Error checking sync status for user ${targetCidNum}`, error);
|
|
547
|
-
return res.status(500).json({
|
|
548
|
-
success: false,
|
|
549
|
-
error: "Internal server error",
|
|
550
|
-
message: error.message
|
|
551
|
-
});
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
/**
|
|
556
|
-
* Check global rate limits for a user
|
|
557
|
-
* NOTE: Developer accounts should bypass this check (check isDeveloperAccount before calling)
|
|
558
|
-
*/
|
|
559
|
-
async function checkRateLimits(db, targetUserCid, logger) {
|
|
560
|
-
const now = Date.now();
|
|
561
|
-
|
|
562
|
-
// Check global rate limit (applies to target user, not requester)
|
|
563
|
-
const globalRef = db.collection('user_sync_requests')
|
|
564
|
-
.doc(String(targetUserCid))
|
|
565
|
-
.collection('global')
|
|
566
|
-
.doc('latest');
|
|
567
|
-
|
|
568
|
-
const globalDoc = await globalRef.get();
|
|
569
|
-
let canRequestAgainAt = null;
|
|
570
|
-
let blocked = false;
|
|
571
|
-
let lastRequestedAt = null;
|
|
572
|
-
|
|
573
|
-
if (globalDoc.exists) {
|
|
574
|
-
const globalData = globalDoc.data();
|
|
575
|
-
const globalLastRequestedAt = globalData.lastRequestedAt?.toDate?.()?.getTime() || globalData.lastRequestedAt?.toMillis?.() || null;
|
|
576
|
-
|
|
577
|
-
if (globalLastRequestedAt && (now - globalLastRequestedAt) < RATE_LIMIT_MS) {
|
|
578
|
-
blocked = true;
|
|
579
|
-
canRequestAgainAt = new Date(globalLastRequestedAt + RATE_LIMIT_MS).toISOString();
|
|
580
|
-
lastRequestedAt = new Date(globalLastRequestedAt).toISOString();
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
if (blocked) {
|
|
585
|
-
const hoursRemaining = Math.ceil((new Date(canRequestAgainAt).getTime() - now) / (60 * 60 * 1000));
|
|
586
|
-
return {
|
|
587
|
-
allowed: false,
|
|
588
|
-
message: `This user was recently synced. Please try again in ${hoursRemaining} hour${hoursRemaining !== 1 ? 's' : ''}.`,
|
|
589
|
-
canRequestAgainAt,
|
|
590
|
-
lastRequestedAt
|
|
591
|
-
};
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
return {
|
|
595
|
-
allowed: true,
|
|
596
|
-
canRequestAgainAt: new Date(now + RATE_LIMIT_MS).toISOString()
|
|
597
|
-
};
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
/**
|
|
601
|
-
* Helper function to check for computation results
|
|
602
|
-
*/
|
|
603
|
-
async function checkComputationResults(db, targetCidNum, userType, insightsCollection, resultsSub, compsSub, logger) {
|
|
604
|
-
try {
|
|
605
|
-
// Determine category and computation name based on user type
|
|
606
|
-
// NOTE: ResultCommitter ignores category metadata if computation is in a non-core folder
|
|
607
|
-
// SignedInUserProfileMetrics is in popular-investor folder, so it's stored in popular-investor category
|
|
608
|
-
let category, computationName;
|
|
609
|
-
if (userType === 'POPULAR_INVESTOR') {
|
|
610
|
-
category = 'popular-investor';
|
|
611
|
-
computationName = 'PopularInvestorProfileMetrics';
|
|
612
|
-
} else {
|
|
613
|
-
// SignedInUserProfileMetrics is in popular-investor folder, so stored in popular-investor category
|
|
614
|
-
category = 'popular-investor';
|
|
615
|
-
computationName = 'SignedInUserProfileMetrics';
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
// Check today and yesterday for computation results
|
|
619
|
-
const checkDate = new Date();
|
|
620
|
-
for (let i = 0; i < 2; i++) {
|
|
621
|
-
const dateStr = new Date(checkDate);
|
|
622
|
-
dateStr.setDate(checkDate.getDate() - i);
|
|
623
|
-
const dateStrFormatted = dateStr.toISOString().split('T')[0];
|
|
624
|
-
|
|
625
|
-
const docRef = db.collection(insightsCollection)
|
|
626
|
-
.doc(dateStrFormatted)
|
|
627
|
-
.collection(resultsSub)
|
|
628
|
-
.doc(category)
|
|
629
|
-
.collection(compsSub)
|
|
630
|
-
.doc(computationName);
|
|
631
|
-
|
|
632
|
-
const doc = await docRef.get();
|
|
633
|
-
if (doc.exists) {
|
|
634
|
-
const docData = doc.data();
|
|
635
|
-
let mergedData = null;
|
|
636
|
-
|
|
637
|
-
// Check if data is sharded
|
|
638
|
-
if (docData._sharded === true && docData._shardCount) {
|
|
639
|
-
const shardsCol = docRef.collection('_shards');
|
|
640
|
-
const shardsSnapshot = await shardsCol.get();
|
|
641
|
-
|
|
642
|
-
if (!shardsSnapshot.empty) {
|
|
643
|
-
mergedData = {};
|
|
644
|
-
for (const shardDoc of shardsSnapshot.docs) {
|
|
645
|
-
const shardData = shardDoc.data();
|
|
646
|
-
Object.assign(mergedData, shardData);
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
} else {
|
|
650
|
-
const { tryDecompress } = require('../data_helpers');
|
|
651
|
-
mergedData = tryDecompress(docData);
|
|
652
|
-
|
|
653
|
-
if (typeof mergedData === 'string') {
|
|
654
|
-
try {
|
|
655
|
-
mergedData = JSON.parse(mergedData);
|
|
656
|
-
} catch (e) {
|
|
657
|
-
mergedData = null;
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
if (mergedData && typeof mergedData === 'object' && mergedData[String(targetCidNum)]) {
|
|
663
|
-
return true; // Found results
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
return false; // No results found
|
|
668
|
-
} catch (error) {
|
|
669
|
-
logger.log('WARN', `[checkComputationResults] Error checking computation results`, error);
|
|
670
|
-
return false;
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
module.exports = {
|
|
675
|
-
requestUserSync,
|
|
676
|
-
getUserSyncStatus
|
|
677
|
-
};
|