bulltrackers-module 1.0.403 → 1.0.404

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/functions/root-data-indexer/index.js +123 -65
  2. package/package.json +1 -1
package/functions/root-data-indexer/index.js CHANGED
@@ -10,8 +10,53 @@ const { FieldValue } = require('@google-cloud/firestore');
10
10
  const pLimit = require('p-limit');
11
11
 
12
12
  const CANARY_BLOCK_ID = '19M';
13
- const CANARY_PART_ID = 'part_0';
14
- const PRICE_SHARD_ID = 'shard_0';
13
+ const PRICE_SHARD_ID = 'shard_0';
14
+
15
+ /**
16
+ * Helper function to check if any part document exists in a parts collection
17
+ * @param {Firestore.CollectionReference} partsCollectionRef - Reference to the parts collection
18
+ * @returns {Promise<boolean>} - True if any part_* document exists
19
+ */
20
+ async function checkAnyPartExists(partsCollectionRef) {
21
+ try {
22
+ // List all documents in the parts collection
23
+ const snapshot = await partsCollectionRef.limit(10).get();
24
+ if (snapshot.empty) return false;
25
+
26
+ // Check if any document ID starts with 'part_'
27
+ for (const doc of snapshot.docs) {
28
+ if (doc.id.startsWith('part_')) {
29
+ return true;
30
+ }
31
+ }
32
+ return false;
33
+ } catch (error) {
34
+ return false;
35
+ }
36
+ }
37
+
38
+ /**
39
+ * Helper function to check if any shard document exists in a collection
40
+ * @param {Firestore.CollectionReference} collectionRef - Reference to the collection
41
+ * @returns {Promise<boolean>} - True if any shard_* document exists
42
+ */
43
+ async function checkAnyShardExists(collectionRef) {
44
+ try {
45
+ // List all documents in the collection
46
+ const snapshot = await collectionRef.limit(10).get();
47
+ if (snapshot.empty) return false;
48
+
49
+ // Check if any document ID starts with 'shard_'
50
+ for (const doc of snapshot.docs) {
51
+ if (doc.id.startsWith('shard_')) {
52
+ return true;
53
+ }
54
+ }
55
+ return false;
56
+ } catch (error) {
57
+ return false;
58
+ }
59
+ }
15
60
 
16
61
  exports.runRootDataIndexer = async (config, dependencies) => {
17
62
  const { db, logger } = dependencies;
@@ -30,23 +75,33 @@ exports.runRootDataIndexer = async (config, dependencies) => {
30
75
  logger.log('INFO', '[RootDataIndexer] Starting Root Data Availability Scan...');
31
76
 
32
77
  // 1. Price Availability
78
+ // Sample up to 10 shards to extract date keys (efficient - doesn't read all shards)
33
79
  const priceAvailabilitySet = new Set();
34
80
  try {
35
- // Path: asset_prices/shard_0
36
- const priceShardRef = db.collection(PRICE_COLLECTION_NAME).doc(PRICE_SHARD_ID);
37
- const priceSnap = await priceShardRef.get();
38
- if (priceSnap.exists) {
39
- const data = priceSnap.data();
40
- Object.values(data).forEach(instrument => {
41
- if (instrument.prices) {
42
- Object.keys(instrument.prices).forEach(dateKey => {
43
- if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { priceAvailabilitySet.add(dateKey); }
44
- });
45
- }
46
- });
81
+ // Path: asset_prices/shard_*
82
+ const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
83
+ const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
84
+
85
+ if (!priceShardsSnapshot.empty) {
86
+ // Sample up to 10 shards and extract date keys from them
87
+ // This gives us a representative sample without reading hundreds of shards
88
+ for (const shardDoc of priceShardsSnapshot.docs) {
89
+ if (shardDoc.id.startsWith('shard_')) {
90
+ const data = shardDoc.data();
91
+ Object.values(data).forEach(instrument => {
92
+ if (instrument && instrument.prices) {
93
+ Object.keys(instrument.prices).forEach(dateKey => {
94
+ if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) {
95
+ priceAvailabilitySet.add(dateKey);
96
+ }
97
+ });
98
+ }
99
+ });
100
+ }
101
+ }
47
102
  }
48
103
  } catch (e) {
49
- logger.log('ERROR', '[RootDataIndexer] Failed to load price shard.', { error: e.message });
104
+ logger.log('ERROR', '[RootDataIndexer] Failed to sample price shards.', { error: e.message });
50
105
  }
51
106
 
52
107
  // 2. Determine Date Range
@@ -96,17 +151,17 @@ exports.runRootDataIndexer = async (config, dependencies) => {
96
151
  // --- Define Refs & Check Paths ---
97
152
 
98
153
  // 1. Standard Retail
99
- // Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
100
- const normPortRef = db.collection(collections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
154
+ // Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
155
+ const normPortPartsRef = db.collection(collections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
101
156
 
102
- // Path: {speculatorPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
103
- const specPortRef = db.collection(collections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
157
+ // Path: {speculatorPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
158
+ const specPortPartsRef = db.collection(collections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
104
159
 
105
- // Path: {normalHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
106
- const normHistRef = db.collection(collections.normalHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
160
+ // Path: {normalHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
161
+ const normHistPartsRef = db.collection(collections.normalHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
107
162
 
108
- // Path: {speculatorHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
109
- const specHistRef = db.collection(collections.speculatorHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
163
+ // Path: {speculatorHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
164
+ const specHistPartsRef = db.collection(collections.speculatorHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
110
165
 
111
166
  // Path: {insights}/{YYYY-MM-DD}
112
167
  const insightsRef = db.collection(collections.insights).doc(dateStr);
@@ -119,26 +174,26 @@ exports.runRootDataIndexer = async (config, dependencies) => {
119
174
  // Path: {piRankings}/{YYYY-MM-DD}
120
175
  const piRankingsRef = db.collection(collections.piRankings || 'popular_investor_rankings').doc(dateStr);
121
176
 
122
- // Path: {piPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
123
- const piPortfoliosRef = db.collection(collections.piPortfolios || 'pi_portfolios_overall')
124
- .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
177
+ // Path: {piPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
178
+ const piPortfoliosPartsRef = db.collection(collections.piPortfolios || 'pi_portfolios_overall')
179
+ .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
125
180
 
126
- // Path: {piDeepPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
127
- const piDeepRef = db.collection(collections.piDeepPortfolios || 'pi_portfolios_deep')
128
- .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
181
+ // Path: {piDeepPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
182
+ const piDeepPartsRef = db.collection(collections.piDeepPortfolios || 'pi_portfolios_deep')
183
+ .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
129
184
 
130
- // Path: {piHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
131
- const piHistoryRef = db.collection(collections.piHistory || 'pi_trade_history')
132
- .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
185
+ // Path: {piHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
186
+ const piHistoryPartsRef = db.collection(collections.piHistory || 'pi_trade_history')
187
+ .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
133
188
 
134
189
  // 3. Signed-In Users
135
- // Path: {signedInUsers}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
136
- const signedInPortRef = db.collection(collections.signedInUsers || 'signed_in_user_portfolios')
137
- .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
190
+ // Path: {signedInUsers}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
191
+ const signedInPortPartsRef = db.collection(collections.signedInUsers || 'signed_in_users')
192
+ .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
138
193
 
139
- // Path: {signedInHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_0
140
- const signedInHistRef = db.collection(collections.signedInHistory || 'signed_in_user_history')
141
- .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
194
+ // Path: {signedInHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
195
+ const signedInHistPartsRef = db.collection(collections.signedInHistory || 'signed_in_user_history')
196
+ .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
142
197
 
143
198
  // Path: {verifications} (Limit 1) - Checks if collection is non-empty generally
144
199
  const verificationsRef = db.collection(collections.verifications || 'user_verifications');
@@ -152,26 +207,29 @@ exports.runRootDataIndexer = async (config, dependencies) => {
152
207
 
153
208
  // --- Execute Checks ---
154
209
  const [
155
- normPortSnap, specPortSnap,
156
- normHistSnap, specHistSnap,
210
+ normPortExists, specPortExists,
211
+ normHistExists, specHistExists,
157
212
  insightsSnap, socialQuerySnap,
158
213
  piRankingsSnap,
159
- piPortSnap, // UPDATED: Now a DocumentSnapshot
160
- piDeepSnap, // UPDATED: Now a DocumentSnapshot
161
- piHistSnap, // UPDATED: Now a DocumentSnapshot
162
- signedInPortSnap, signedInHistSnap,
214
+ piPortExists, // UPDATED: Check for any part_* document
215
+ piDeepExists, // UPDATED: Check for any part_* document
216
+ piHistExists, // UPDATED: Check for any part_* document
217
+ signedInPortExists, signedInHistExists,
163
218
  verificationsQuery,
164
219
  universalSocialSnap
165
220
  ] = await Promise.all([
166
- normPortRef.get(), specPortRef.get(),
167
- normHistRef.get(), specHistRef.get(),
168
- insightsRef.get(), socialPostsRef.limit(1).get(),
221
+ checkAnyPartExists(normPortPartsRef),
222
+ checkAnyPartExists(specPortPartsRef),
223
+ checkAnyPartExists(normHistPartsRef),
224
+ checkAnyPartExists(specHistPartsRef),
225
+ insightsRef.get(),
226
+ socialPostsRef.limit(1).get(),
169
227
  piRankingsRef.get(),
170
- piPortfoliosRef.get(), // UPDATED: Direct Doc Get (no limit)
171
- piDeepRef.get(), // UPDATED: Direct Doc Get (no limit)
172
- piHistoryRef.get(), // UPDATED: Direct Doc Get (no limit)
173
- signedInPortRef.get(),
174
- signedInHistRef.get(),
228
+ checkAnyPartExists(piPortfoliosPartsRef),
229
+ checkAnyPartExists(piDeepPartsRef),
230
+ checkAnyPartExists(piHistoryPartsRef),
231
+ checkAnyPartExists(signedInPortPartsRef),
232
+ checkAnyPartExists(signedInHistPartsRef),
175
233
  verificationsRef.limit(1).get(),
176
234
  universalSocialQuery.get()
177
235
  ]);
@@ -189,16 +247,16 @@ exports.runRootDataIndexer = async (config, dependencies) => {
189
247
  }
190
248
 
191
249
  // --- Assign to Availability ---
192
- availability.details.normalPortfolio = normPortSnap.exists;
193
- availability.details.speculatorPortfolio = specPortSnap.exists;
194
- availability.details.normalHistory = normHistSnap.exists;
195
- availability.details.speculatorHistory = specHistSnap.exists;
250
+ availability.details.normalPortfolio = normPortExists;
251
+ availability.details.speculatorPortfolio = specPortExists;
252
+ availability.details.normalHistory = normHistExists;
253
+ availability.details.speculatorHistory = specHistExists;
196
254
  availability.details.piRankings = piRankingsSnap.exists;
197
255
 
198
- // UPDATED: Now checking .exists on DocumentSnapshots
199
- availability.details.piPortfolios = piPortSnap.exists;
200
- availability.details.piDeepPortfolios = piDeepSnap.exists;
201
- availability.details.piHistory = piHistSnap.exists;
256
+ // UPDATED: Now checking for any part_* document existence
257
+ availability.details.piPortfolios = piPortExists;
258
+ availability.details.piDeepPortfolios = piDeepExists;
259
+ availability.details.piHistory = piHistExists;
202
260
 
203
261
  // PI & Signed-In Social Flags (Strict)
204
262
  availability.details.piSocial = foundPISocial;
@@ -207,14 +265,14 @@ exports.runRootDataIndexer = async (config, dependencies) => {
207
265
  availability.details.hasSignedInSocial = foundSignedInSocial;
208
266
 
209
267
  // Signed-In Flags
210
- availability.details.signedInUserPortfolio = signedInPortSnap.exists;
211
- availability.details.signedInUserHistory = signedInHistSnap.exists;
268
+ availability.details.signedInUserPortfolio = signedInPortExists;
269
+ availability.details.signedInUserHistory = signedInHistExists;
212
270
  availability.details.signedInUserVerification = !verificationsQuery.empty;
213
271
 
214
272
  // Aggregates
215
- // UPDATED: piPortSnap.exists used in aggregate logic
216
- availability.hasPortfolio = normPortSnap.exists || specPortSnap.exists || piPortSnap.exists || signedInPortSnap.exists;
217
- availability.hasHistory = normHistSnap.exists || specHistSnap.exists || piHistSnap.exists || signedInHistSnap.exists;
273
+ // UPDATED: Using part existence checks
274
+ availability.hasPortfolio = normPortExists || specPortExists || piPortExists || signedInPortExists;
275
+ availability.hasHistory = normHistExists || specHistExists || piHistExists || signedInHistExists;
218
276
  availability.hasInsights = insightsSnap.exists;
219
277
 
220
278
  // [CRITICAL FIX]
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.403",
3
+ "version": "1.0.404",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [