bulltrackers-module 1.0.477 → 1.0.479
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/workflows/bulltrackers_pipeline.yaml +4 -4
- package/functions/etoro-price-fetcher/helpers/handler_helpers.js +22 -15
- package/functions/fetch-insights/helpers/handler_helpers.js +22 -15
- package/functions/fetch-popular-investors/helpers/fetch_helpers.js +23 -16
- package/functions/root-data-indexer/index.js +43 -17
- package/package.json +1 -1
|
@@ -27,7 +27,7 @@ main:
|
|
|
27
27
|
- call_dispatcher:
|
|
28
28
|
call: http.post
|
|
29
29
|
args:
|
|
30
|
-
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-
|
|
30
|
+
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
|
|
31
31
|
body:
|
|
32
32
|
pass: '${pass_id}'
|
|
33
33
|
cursorIndex: '${n_cursor}'
|
|
@@ -84,7 +84,7 @@ main:
|
|
|
84
84
|
- verify_pass_completion:
|
|
85
85
|
call: http.post
|
|
86
86
|
args:
|
|
87
|
-
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-
|
|
87
|
+
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
|
|
88
88
|
body:
|
|
89
89
|
action: 'VERIFY'
|
|
90
90
|
pass: '${pass_id}'
|
|
@@ -104,7 +104,7 @@ main:
|
|
|
104
104
|
- dispatch_force_sweep:
|
|
105
105
|
call: http.post
|
|
106
106
|
args:
|
|
107
|
-
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-
|
|
107
|
+
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
|
|
108
108
|
body:
|
|
109
109
|
action: 'SWEEP'
|
|
110
110
|
pass: '${pass_id}'
|
|
@@ -126,7 +126,7 @@ main:
|
|
|
126
126
|
- generate_final_report:
|
|
127
127
|
call: http.post
|
|
128
128
|
args:
|
|
129
|
-
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-
|
|
129
|
+
url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
|
|
130
130
|
body:
|
|
131
131
|
action: 'REPORT'
|
|
132
132
|
pass: '${pass_id}'
|
|
@@ -97,22 +97,29 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
|
|
|
97
97
|
// Update root data indexer for today's date after price data is stored
|
|
98
98
|
try {
|
|
99
99
|
const { runRootDataIndexer } = require('../../root-data-indexer/index');
|
|
100
|
-
const rootDataIndexerConfig = config.rootDataIndexer || {
|
|
101
|
-
availabilityCollection: 'system_root_data_index',
|
|
102
|
-
earliestDate: '2025-08-01',
|
|
103
|
-
collections: {
|
|
104
|
-
prices: priceCollectionName
|
|
105
|
-
}
|
|
106
|
-
};
|
|
107
|
-
|
|
108
|
-
const indexerConfig = {
|
|
109
|
-
...rootDataIndexerConfig,
|
|
110
|
-
targetDate: today // Index only today's date for speed
|
|
111
|
-
};
|
|
112
100
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
101
|
+
if (!config.rootDataIndexer) {
|
|
102
|
+
logger.log('WARN', `[PriceFetcherHelpers] Root data indexer config not provided. Skipping index update.`);
|
|
103
|
+
} else {
|
|
104
|
+
// Merge price collection name into the full config
|
|
105
|
+
const indexerConfig = {
|
|
106
|
+
...config.rootDataIndexer,
|
|
107
|
+
collections: {
|
|
108
|
+
...config.rootDataIndexer.collections,
|
|
109
|
+
prices: priceCollectionName // Override with actual collection name used
|
|
110
|
+
},
|
|
111
|
+
targetDate: today // Index only today's date for speed
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
logger.log('INFO', `[PriceFetcherHelpers] Triggering root data indexer for date ${today} after price data storage...`);
|
|
115
|
+
const result = await runRootDataIndexer(indexerConfig, dependencies);
|
|
116
|
+
|
|
117
|
+
if (result.success && result.count > 0) {
|
|
118
|
+
logger.log('INFO', `[PriceFetcherHelpers] Root data indexer completed successfully for date ${today} (updated ${result.count} dates)`);
|
|
119
|
+
} else {
|
|
120
|
+
logger.log('WARN', `[PriceFetcherHelpers] Root data indexer completed but no dates were updated for ${today}`);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
116
123
|
} catch (indexerError) {
|
|
117
124
|
logger.log('ERROR', `[PriceFetcherHelpers] Failed to run root data indexer for ${today}`, indexerError);
|
|
118
125
|
// Continue - price data is stored, indexer failure is non-critical
|
|
@@ -121,22 +121,29 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
|
|
|
121
121
|
// Update root data indexer for today's date after insights data is stored
|
|
122
122
|
try {
|
|
123
123
|
const { runRootDataIndexer } = require('../../root-data-indexer/index');
|
|
124
|
-
const rootDataIndexerConfig = config.rootDataIndexer || {
|
|
125
|
-
availabilityCollection: 'system_root_data_index',
|
|
126
|
-
earliestDate: '2025-08-01',
|
|
127
|
-
collections: {
|
|
128
|
-
insights: config.insightsCollectionName
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
|
|
132
|
-
const indexerConfig = {
|
|
133
|
-
...rootDataIndexerConfig,
|
|
134
|
-
targetDate: today // Index only today's date for speed
|
|
135
|
-
};
|
|
136
124
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
125
|
+
if (!config.rootDataIndexer) {
|
|
126
|
+
logger.log('WARN', `[FetchInsightsHelpers] Root data indexer config not provided. Skipping index update.`);
|
|
127
|
+
} else {
|
|
128
|
+
// Merge insights collection name into the full config
|
|
129
|
+
const indexerConfig = {
|
|
130
|
+
...config.rootDataIndexer,
|
|
131
|
+
collections: {
|
|
132
|
+
...config.rootDataIndexer.collections,
|
|
133
|
+
insights: config.insightsCollectionName // Override with actual collection name used
|
|
134
|
+
},
|
|
135
|
+
targetDate: today // Index only today's date for speed
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
logger.log('INFO', `[FetchInsightsHelpers] Triggering root data indexer for date ${today} after insights data storage...`);
|
|
139
|
+
const result = await runRootDataIndexer(indexerConfig, dependencies);
|
|
140
|
+
|
|
141
|
+
if (result.success && result.count > 0) {
|
|
142
|
+
logger.log('INFO', `[FetchInsightsHelpers] Root data indexer completed successfully for date ${today} (updated ${result.count} dates)`);
|
|
143
|
+
} else {
|
|
144
|
+
logger.log('WARN', `[FetchInsightsHelpers] Root data indexer completed but no dates were updated for ${today}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
140
147
|
} catch (indexerError) {
|
|
141
148
|
logger.log('ERROR', `[FetchInsightsHelpers] Failed to run root data indexer for ${today}`, indexerError);
|
|
142
149
|
// Continue - insights data is stored, indexer failure is non-critical
|
|
@@ -117,28 +117,35 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
|
|
|
117
117
|
// Update root data indexer for today's date after rankings data is stored
|
|
118
118
|
try {
|
|
119
119
|
const { runRootDataIndexer } = require('../../root-data-indexer/index');
|
|
120
|
+
|
|
120
121
|
// Access rootDataIndexer from config (passed from index.js) or use defaults
|
|
121
122
|
// Using bracket notation to avoid TypeScript errors
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
123
|
+
const rootDataIndexerConfig = (config && typeof config === 'object' && config['rootDataIndexer'])
|
|
124
|
+
? config['rootDataIndexer']
|
|
125
|
+
: null;
|
|
126
|
+
|
|
127
|
+
if (!rootDataIndexerConfig) {
|
|
128
|
+
logger.log('WARN', `[PopularInvestorFetch] Root data indexer config not provided. Skipping index update.`);
|
|
125
129
|
} else {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
130
|
+
// Merge rankings collection name into the full config
|
|
131
|
+
const indexerConfig = {
|
|
132
|
+
...rootDataIndexerConfig,
|
|
129
133
|
collections: {
|
|
130
|
-
|
|
131
|
-
|
|
134
|
+
...rootDataIndexerConfig.collections,
|
|
135
|
+
piRankings: rankingsCollectionName // Override with actual collection name used
|
|
136
|
+
},
|
|
137
|
+
targetDate: today // Index only today's date for speed
|
|
132
138
|
};
|
|
139
|
+
|
|
140
|
+
logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
|
|
141
|
+
const result = await runRootDataIndexer(indexerConfig, dependencies);
|
|
142
|
+
|
|
143
|
+
if (result.success && result.count > 0) {
|
|
144
|
+
logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed successfully for date ${today} (updated ${result.count} dates)`);
|
|
145
|
+
} else {
|
|
146
|
+
logger.log('WARN', `[PopularInvestorFetch] Root data indexer completed but no dates were updated for ${today}`);
|
|
147
|
+
}
|
|
133
148
|
}
|
|
134
|
-
|
|
135
|
-
const indexerConfig = Object.assign({}, rootDataIndexerConfig, {
|
|
136
|
-
targetDate: today // Index only today's date for speed
|
|
137
|
-
});
|
|
138
|
-
|
|
139
|
-
logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
|
|
140
|
-
await runRootDataIndexer(indexerConfig, dependencies);
|
|
141
|
-
logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed for date ${today}`);
|
|
142
149
|
} catch (indexerError) {
|
|
143
150
|
logger.log('ERROR', `[PopularInvestorFetch] Failed to run root data indexer for ${today}`, indexerError);
|
|
144
151
|
// Continue - rankings data is stored, indexer failure is non-critical
|
|
@@ -62,7 +62,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
|
|
|
62
62
|
const {
|
|
63
63
|
availabilityCollection,
|
|
64
64
|
earliestDate,
|
|
65
|
-
collections,
|
|
65
|
+
collections = {},
|
|
66
66
|
targetDate // [NEW] Optional parameter to scan a single specific date
|
|
67
67
|
} = config;
|
|
68
68
|
|
|
@@ -71,6 +71,25 @@ exports.runRootDataIndexer = async (config, dependencies) => {
|
|
|
71
71
|
// Collection Names (Fail-safe defaults)
|
|
72
72
|
const PI_SOCIAL_COLL_NAME = collections.piSocial || 'pi_social_posts';
|
|
73
73
|
const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users_social';
|
|
74
|
+
|
|
75
|
+
// Ensure all required collections have defaults to prevent "collectionPath is not valid" errors
|
|
76
|
+
const safeCollections = {
|
|
77
|
+
normalPortfolios: collections.normalPortfolios || 'NormalUserPortfolios',
|
|
78
|
+
speculatorPortfolios: collections.speculatorPortfolios || 'SpeculatorPortfolios',
|
|
79
|
+
normalHistory: collections.normalHistory || 'NormalUserTradeHistory',
|
|
80
|
+
speculatorHistory: collections.speculatorHistory || 'SpeculatorTradeHistory',
|
|
81
|
+
insights: collections.insights || 'daily_instrument_insights',
|
|
82
|
+
social: collections.social || 'daily_social_insights',
|
|
83
|
+
prices: collections.prices || PRICE_COLLECTION_NAME,
|
|
84
|
+
piRankings: collections.piRankings || 'popular_investor_rankings',
|
|
85
|
+
piPortfolios: collections.piPortfolios || 'pi_portfolios_overall',
|
|
86
|
+
piDeepPortfolios: collections.piDeepPortfolios || 'pi_portfolios_deep',
|
|
87
|
+
piHistory: collections.piHistory || 'pi_trade_history',
|
|
88
|
+
signedInUsers: collections.signedInUsers || 'signed_in_users',
|
|
89
|
+
signedInHistory: collections.signedInHistory || 'signed_in_user_history',
|
|
90
|
+
verifications: collections.verifications || 'user_verifications',
|
|
91
|
+
...collections // Allow overrides
|
|
92
|
+
};
|
|
74
93
|
|
|
75
94
|
const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
|
|
76
95
|
logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
|
|
@@ -189,51 +208,51 @@ exports.runRootDataIndexer = async (config, dependencies) => {
|
|
|
189
208
|
|
|
190
209
|
// 1. Standard Retail
|
|
191
210
|
// Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
192
|
-
const normPortPartsRef = db.collection(
|
|
211
|
+
const normPortPartsRef = db.collection(safeCollections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
193
212
|
|
|
194
213
|
// Path: {speculatorPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
195
|
-
const specPortPartsRef = db.collection(
|
|
214
|
+
const specPortPartsRef = db.collection(safeCollections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
196
215
|
|
|
197
216
|
// Path: {normalHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
198
|
-
const normHistPartsRef = db.collection(
|
|
217
|
+
const normHistPartsRef = db.collection(safeCollections.normalHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
199
218
|
|
|
200
219
|
// Path: {speculatorHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
201
|
-
const specHistPartsRef = db.collection(
|
|
220
|
+
const specHistPartsRef = db.collection(safeCollections.speculatorHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
202
221
|
|
|
203
222
|
// Path: {insights}/{YYYY-MM-DD}
|
|
204
|
-
const insightsRef = db.collection(
|
|
223
|
+
const insightsRef = db.collection(safeCollections.insights).doc(dateStr);
|
|
205
224
|
|
|
206
225
|
// Generic Asset Posts
|
|
207
226
|
// Path: {social}/{YYYY-MM-DD}/posts (Limit 1)
|
|
208
|
-
const socialPostsRef = db.collection(
|
|
227
|
+
const socialPostsRef = db.collection(safeCollections.social).doc(dateStr).collection('posts');
|
|
209
228
|
|
|
210
229
|
// 2. Popular Investors
|
|
211
230
|
// Path: {piRankings}/{YYYY-MM-DD}
|
|
212
|
-
const piRankingsRef = db.collection(
|
|
231
|
+
const piRankingsRef = db.collection(safeCollections.piRankings).doc(dateStr);
|
|
213
232
|
|
|
214
233
|
// Path: {piPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
215
|
-
const piPortfoliosPartsRef = db.collection(
|
|
234
|
+
const piPortfoliosPartsRef = db.collection(safeCollections.piPortfolios)
|
|
216
235
|
.doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
217
236
|
|
|
218
237
|
// Path: {piDeepPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
219
|
-
const piDeepPartsRef = db.collection(
|
|
238
|
+
const piDeepPartsRef = db.collection(safeCollections.piDeepPortfolios)
|
|
220
239
|
.doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
221
240
|
|
|
222
241
|
// Path: {piHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
223
|
-
const piHistoryPartsRef = db.collection(
|
|
242
|
+
const piHistoryPartsRef = db.collection(safeCollections.piHistory)
|
|
224
243
|
.doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
225
244
|
|
|
226
245
|
// 3. Signed-In Users
|
|
227
246
|
// Path: {signedInUsers}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
228
|
-
const signedInPortPartsRef = db.collection(
|
|
247
|
+
const signedInPortPartsRef = db.collection(safeCollections.signedInUsers)
|
|
229
248
|
.doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
230
249
|
|
|
231
250
|
// Path: {signedInHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
|
|
232
|
-
const signedInHistPartsRef = db.collection(
|
|
251
|
+
const signedInHistPartsRef = db.collection(safeCollections.signedInHistory)
|
|
233
252
|
.doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
|
|
234
|
-
|
|
253
|
+
|
|
235
254
|
// Path: {verifications} (Limit 1) - Checks if collection is non-empty generally
|
|
236
|
-
const verificationsRef = db.collection(
|
|
255
|
+
const verificationsRef = db.collection(safeCollections.verifications);
|
|
237
256
|
|
|
238
257
|
// 4. Social Data Checks - Use date tracking documents
|
|
239
258
|
// Single tracking documents at root level:
|
|
@@ -353,6 +372,13 @@ exports.runRootDataIndexer = async (config, dependencies) => {
|
|
|
353
372
|
}));
|
|
354
373
|
|
|
355
374
|
await Promise.all(promises);
|
|
356
|
-
|
|
357
|
-
|
|
375
|
+
|
|
376
|
+
// Log appropriately based on results
|
|
377
|
+
if (updatesCount === 0) {
|
|
378
|
+
logger.log('WARN', `[RootDataIndexer] Indexing complete but NO dates were updated. This may indicate a failure. Mode: ${scanMode}`, { targetDate });
|
|
379
|
+
} else {
|
|
380
|
+
logger.log('SUCCESS', `[RootDataIndexer] Indexing complete. Updated ${updatesCount} dates. Mode: ${scanMode}`, { targetDate, updatesCount });
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
return { success: updatesCount > 0, count: updatesCount, mode: scanMode };
|
|
358
384
|
};
|