bulltrackers-module 1.0.103 → 1.0.105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/functions/computation-system/computation_dependencies.json +79 -79
  2. package/functions/price-backfill/helpers/handler_helpers.js +32 -25
  3. package/package.json +1 -1
package/functions/computation-system/computation_dependencies.json CHANGED
@@ -2,25 +2,25 @@
2
2
  "dependencies": {
3
3
  "smart-cohort-flow": ["user-investment-profile"],
4
4
  "dumb-cohort-flow": ["user-investment-profile"],
5
- "crowd_sharpe_ratio_proxy": ["pnl-distribution-per-stock"],
6
- "social_flow_correlation": [
7
- "social_sentiment_aggregation",
8
- "asset_crowd_flow"
5
+ "crowd-sharpe-ratio-proxy": ["pnl-distribution-per-stock"],
6
+ "social-flow-correlation": [
7
+ "social-sentiment-aggregation",
8
+ "asset-crowd-flow"
9
9
  ],
10
10
  "cash-flow-deployment": [
11
11
  "crowd-cash-flow-proxy",
12
- "asset_crowd_flow"
12
+ "asset-crowd-flow"
13
13
  ],
14
14
  "cash-flow-liquidation": [
15
15
  "crowd-cash-flow-proxy",
16
- "asset_crowd_flow"
16
+ "asset-crowd-flow"
17
17
  ],
18
- "capital_deployment_strategy": [
18
+ "capital-deployment-strategy": [
19
19
  "crowd-cash-flow-proxy",
20
20
  "new-allocation-percentage",
21
21
  "reallocation-increase-percentage"
22
22
  ],
23
- "profit_cohort_divergence": [
23
+ "profit-cohort-divergence": [
24
24
  "in-profit-asset-crowd-flow",
25
25
  "in-loss-asset-crowd-flow"
26
26
  ],
@@ -28,93 +28,93 @@
28
28
  "smart-cohort-flow",
29
29
  "dumb-cohort-flow"
30
30
  ],
31
- "capital_vintage_performance": ["cash-flow-deployment"],
32
- "capital_liquidation_performance": ["cash-flow-liquidation"],
31
+ "capital-vintage-performance": ["cash-flow-deployment"],
32
+ "capital-liquidation-performance": ["cash-flow-liquidation"],
33
33
  "strategy-performance": [
34
34
  "smart-dumb-divergence-index",
35
- "profit_cohort_divergence"
35
+ "profit-cohort-divergence"
36
36
  ]
37
37
  },
38
38
  "requirements": {
39
- "activity_by_pnl_status": ["today_portfolio", "yesterday_portfolio"],
40
- "daily_asset_activity": ["today_portfolio", "yesterday_portfolio"],
41
- "daily_user_activity_tracker": ["today_portfolio", "yesterday_portfolio"],
42
- "speculator_adjustment_activity": ["today_portfolio", "yesterday_portfolio"],
43
- "asset_position_size": ["today_portfolio"],
39
+ "activity-by-pnl-status": ["today_portfolio", "yesterday_portfolio"],
40
+ "daily-asset-activity": ["today_portfolio", "yesterday_portfolio"],
41
+ "daily-user-activity-tracker": ["today_portfolio", "yesterday_portfolio"],
42
+ "speculator-adjustment-activity": ["today_portfolio", "yesterday_portfolio"],
43
+ "asset-position-size": ["today_portfolio"],
44
44
  "strategy-performance": ["meta"],
45
- "asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
46
- "drawdown_response": ["today_portfolio", "yesterday_portfolio"],
45
+ "asset-crowd-flow": ["today_portfolio", "yesterday_portfolio"],
46
+ "drawdown-response": ["today_portfolio", "yesterday_portfolio"],
47
47
  "dumb-cohort-flow": ["today_portfolio", "yesterday_portfolio"],
48
- "gain_response": ["today_portfolio", "yesterday_portfolio"],
49
- "in_loss_asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
50
- "in_profit_asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
51
- "paper_vs_diamond_hands": ["today_portfolio", "yesterday_portfolio"],
52
- "position_count_pnl": ["today_portfolio"],
48
+ "gain-response": ["today_portfolio", "yesterday_portfolio"],
49
+ "in-loss-asset-crowd-flow": ["today_portfolio", "yesterday_portfolio"],
50
+ "in-profit-asset-crowd-flow": ["today_portfolio", "yesterday_portfolio"],
51
+ "paper-vs-diamond-hands": ["today_portfolio", "yesterday_portfolio"],
52
+ "position-count-pnl": ["today_portfolio"],
53
53
  "smart-cohort-flow": ["today_portfolio", "yesterday_portfolio"],
54
- "smart_money_flow": ["today_portfolio", "yesterday_portfolio"],
54
+ "smart-money-flow": ["today_portfolio", "yesterday_portfolio"],
55
55
  "user-investment-profile": ["today_portfolio", "yesterday_portfolio"],
56
56
  "crowd-cash-flow-proxy": ["today_portfolio", "yesterday_portfolio"],
57
- "deposit_withdrawal_percentage": ["today_portfolio", "yesterday_portfolio"],
58
- "new_allocation_percentage": ["today_portfolio", "yesterday_portfolio"],
59
- "reallocation_increase_percentage": ["today_portfolio", "yesterday_portfolio"],
60
- "daily_bought_vs_sold_count": ["today_insights", "yesterday_insights"],
61
- "daily_buy_sell_sentiment_count": ["today_insights"],
62
- "daily_ownership_delta": ["today_insights", "yesterday_insights"],
63
- "daily_total_positions_held": ["today_insights"],
64
- "capital_deployment_strategy": ["meta"],
57
+ "deposit-withdrawal-percentage": ["today_portfolio", "yesterday_portfolio"],
58
+ "new-allocation-percentage": ["today_portfolio", "yesterday_portfolio"],
59
+ "reallocation-increase-percentage": ["today_portfolio", "yesterday_portfolio"],
60
+ "daily-bought-vs-sold-count": ["today_insights", "yesterday_insights"],
61
+ "daily-buy-sell-sentiment-count": ["today_insights"],
62
+ "daily-ownership-delta": ["today_insights", "yesterday_insights"],
63
+ "daily-total-positions-held": ["today_insights"],
64
+ "capital-deployment-strategy": ["meta"],
65
65
  "cash-flow-deployment": ["meta"],
66
66
  "cash-flow-liquidation": ["meta"],
67
- "crowd_sharpe_ratio_proxy": ["meta"],
68
- "profit_cohort_divergence": ["meta"],
67
+ "crowd-sharpe-ratio-proxy": ["meta"],
68
+ "profit-cohort-divergence": ["meta"],
69
69
  "smart-dumb-divergence-index": ["meta"],
70
- "social_flow_correlation": ["meta"],
71
- "capital_liquidation_performance": ["meta"],
72
- "capital_vintage_performance": ["meta"],
73
- "profitability_migration": ["today_portfolio", "yesterday_portfolio"],
74
- "user_profitability_tracker": ["today_portfolio"],
75
- "asset_pnl_status": ["today_portfolio"],
76
- "average_daily_pnl_all_users": ["today_portfolio"],
77
- "average_daily_pnl_per_sector": ["today_portfolio"],
78
- "average_daily_pnl_per_stock": ["today_portfolio"],
79
- "average_daily_position_pnl": ["today_portfolio"],
80
- "pnl_distribution_per_stock": ["today_portfolio"],
81
- "profitability_ratio_per_stock": ["today_portfolio"],
82
- "profitability_skew_per_stock": ["today_portfolio"],
83
- "profitable_and_unprofitable_status": ["today_portfolio"],
84
- "users_processed": ["today_portfolio"],
85
- "diversification_pnl": ["today_portfolio", "yesterday_portfolio"],
86
- "sector_rotation": ["today_portfolio", "yesterday_portfolio"],
87
- "total_long_per_sector": ["today_portfolio"],
88
- "total_short_per_sector": ["today_portfolio"],
89
- "crowd_conviction_score": ["today_portfolio", "yesterday_portfolio"],
90
- "long_position_per_stock": ["today_portfolio"],
91
- "sentiment_per_stock": ["today_portfolio"],
92
- "short_position_per_stock": ["today_portfolio"],
93
- "total_long_figures": ["today_portfolio"],
94
- "total_short_figures": ["today_portfolio"],
70
+ "social-flow-correlation": ["meta"],
71
+ "capital-liquidation-performance": ["meta"],
72
+ "capital-vintage-performance": ["meta"],
73
+ "profitability-migration": ["today_portfolio", "yesterday_portfolio"],
74
+ "user-profitability-tracker": ["today_portfolio"],
75
+ "asset-pnl-status": ["today_portfolio"],
76
+ "average-daily-pnl-all-users": ["today_portfolio"],
77
+ "average-daily-pnl-per-sector": ["today_portfolio"],
78
+ "average-daily-pnl-per-stock": ["today_portfolio"],
79
+ "average-daily-position-pnl": ["today_portfolio"],
80
+ "pnl-distribution-per-stock": ["today_portfolio"],
81
+ "profitability-ratio-per-stock": ["today_portfolio"],
82
+ "profitability-skew-per-stock": ["today_portfolio"],
83
+ "profitable-and-unprofitable-status": ["today_portfolio"],
84
+ "users-processed": ["today_portfolio"],
85
+ "diversification-pnl": ["today_portfolio", "yesterday_portfolio"],
86
+ "sector-rotation": ["today_portfolio", "yesterday_portfolio"],
87
+ "total-long-per-sector": ["today_portfolio"],
88
+ "total-short-per-sector": ["today_portfolio"],
89
+ "crowd-conviction-score": ["today_portfolio", "yesterday_portfolio"],
90
+ "long-position-per-stock": ["today_portfolio"],
91
+ "sentiment-per-stock": ["today_portfolio"],
92
+ "short-position-per-stock": ["today_portfolio"],
93
+ "total-long-figures": ["today_portfolio"],
94
+ "total-short-figures": ["today_portfolio"],
95
95
  "social-asset-posts-trend": ["today_social"],
96
96
  "social-top-mentioned-words": ["today_social", "yesterday_social"],
97
97
  "social-topic-interest-evolution": ["today_social"],
98
98
  "social-word-mentions-trend": ["today_social"],
99
- "social_activity_aggregation": ["today_social"],
100
- "social_event_correlation": ["today_social", "yesterday_social"],
101
- "social_sentiment_aggregation": ["today_social"],
102
- "risk_appetite_change": ["today_portfolio", "yesterday_portfolio"],
103
- "tsl_effectiveness": ["today_portfolio"],
104
- "distance_to_stop_loss_per_leverage": ["today_portfolio"],
105
- "distance_to_tp_per_leverage": ["today_portfolio"],
106
- "entry_distance_to_sl_per_leverage": ["today_portfolio"],
107
- "entry_distance_to_tp_per_leverage": ["today_portfolio"],
108
- "holding_duration_per_asset": ["today_portfolio"],
109
- "leverage_per_asset": ["today_portfolio"],
110
- "leverage_per_sector": ["today_portfolio"],
111
- "risk_reward_ratio_per_asset": ["today_portfolio"],
112
- "speculator_asset_sentiment": ["today_portfolio"],
113
- "speculator_danger_zone": ["today_portfolio"],
114
- "stop_loss_distance_by_sector_short_long_breakdown": ["today_portfolio"],
115
- "stop_loss_distance_by_ticker_short_long_breakdown": ["today_portfolio"],
116
- "stop_loss_per_asset": ["today_portfolio"],
117
- "take_profit_per_asset": ["today_portfolio"],
118
- "tsl_per_asset": ["today_portfolio"]
99
+ "social-activity-aggregation": ["today_social"],
100
+ "social-event-correlation": ["today_social", "yesterday_social"],
101
+ "social-sentiment-aggregation": ["today_social"],
102
+ "risk-appetite-change": ["today_portfolio", "yesterday_portfolio"],
103
+ "tsl-effectiveness": ["today_portfolio"],
104
+ "distance-to-stop-loss-per-leverage": ["today_portfolio"],
105
+ "distance-to-tp-per-leverage": ["today_portfolio"],
106
+ "entry-distance-to-sl-per-leverage": ["today_portfolio"],
107
+ "entry-distance-to-tp-per-leverage": ["today_portfolio"],
108
+ "holding-duration-per-asset": ["today_portfolio"],
109
+ "leverage-per-asset": ["today_portfolio"],
110
+ "leverage-per-sector": ["today_portfolio"],
111
+ "risk-reward-ratio-per-asset": ["today_portfolio"],
112
+ "speculator-asset-sentiment": ["today_portfolio"],
113
+ "speculator-danger-zone": ["today_portfolio"],
114
+ "stop-loss-distance-by-sector-short-long-breakdown": ["today_portfolio"],
115
+ "stop-loss-distance-by-ticker-short-long-breakdown": ["today_portfolio"],
116
+ "stop-loss-per-asset": ["today_portfolio"],
117
+ "take-profit-per-asset": ["today_portfolio"],
118
+ "tsl-per-asset": ["today_portfolio"]
119
119
  }
120
120
  }
package/functions/price-backfill/helpers/handler_helpers.js CHANGED
@@ -5,7 +5,6 @@
5
5
  */
6
6
 
7
7
  const { FieldValue } = require('@google-cloud/firestore'); // Todo inject this
8
-
9
8
  const pLimit = require('p-limit'); // TODO inject this
10
9
 
11
10
  // How many tickers to fetch in parallel
@@ -15,24 +14,36 @@ const DAYS_TO_FETCH = 365;
15
14
  // How many tickers to group into one Firestore document
16
15
  const SHARD_SIZE = 40;
17
16
 
17
+ // EXP BACKOFF
18
+ async function fetchWithExponentialBackoff(fn, maxAttempts = 8) {
19
+ let attempt = 0;
20
+ while (attempt < maxAttempts) {
21
+ try {
22
+ return await fn();
23
+ } catch (err) {
24
+ attempt++;
25
+ if (attempt >= maxAttempts) throw err;
26
+ const delay = Math.min(1000 * Math.pow(2, attempt), 60000); // capped 60s
27
+ await new Promise(res => setTimeout(res, delay));
28
+ }
29
+ }
30
+ }
31
+
18
32
  /**
19
33
  * Main pipe: pipe.maintenance.runBackfillAssetPrices
20
34
  * @param {object} config - Configuration object.
21
35
  * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, calculationUtils.
22
36
  */
23
- // --- MODIFIED: Removed calculationUtils, as it's in dependencies ---
37
+
24
38
  exports.runBackfillAssetPrices = async (config, dependencies) => {
25
39
  const { db, logger, headerManager, proxyManager, calculationUtils } = dependencies;
26
- const { loadInstrumentMappings } = calculationUtils; // <-- Get function from dependencies
27
- // --- END MODIFIED ---
28
-
40
+ const { loadInstrumentMappings } = calculationUtils;
41
+
29
42
  logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
30
-
43
+
31
44
  let mappings;
32
45
  try {
33
- // --- MODIFIED: Use the injected utils ---
34
46
  mappings = await loadInstrumentMappings();
35
- // --- END MODIFIED ---
36
47
  if (!mappings || !mappings.instrumentToTicker) {
37
48
  throw new Error("Failed to load instrument mappings.");
38
49
  }
@@ -52,21 +63,22 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
52
63
  return limit(async () => {
53
64
  try {
54
65
  const ticker = mappings.instrumentToTicker[instrumentId] || `unknown_${instrumentId}`;
55
- const url = `https://candle.etoro.com/candles/asc.json/OneDay/${DAYS_TO_FETCH}/${instrumentId}`; //TODO implement config value
56
-
66
+ const url = `https://candle.etoro.com/candles/asc.json/OneDay/${DAYS_TO_FETCH}/${instrumentId}`;
67
+
57
68
  const selectedHeader = await headerManager.selectHeader();
58
- let wasSuccess = false;
59
69
 
60
- const response = await proxyManager.fetch(url, {
61
- headers: selectedHeader.header,
62
- timeout: 20000
63
- });
70
+ const response = await fetchWithExponentialBackoff(() =>
71
+ proxyManager.fetch(url, {
72
+ headers: selectedHeader.header,
73
+ timeout: 20000
74
+ })
75
+ );
64
76
 
65
77
  if (!response.ok) {
66
78
  throw new Error(`API error ${response.status} for instrument ${instrumentId}`);
67
79
  }
68
- wasSuccess = true;
69
- headerManager.updatePerformance(selectedHeader.id, wasSuccess);
80
+
81
+ headerManager.updatePerformance(selectedHeader.id, true);
70
82
 
71
83
  const data = await response.json();
72
84
  const candles = data?.Candles?.[0]?.Candles;
@@ -76,16 +88,14 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
76
88
  return;
77
89
  }
78
90
 
79
- // Format data as a map
80
91
  const prices = {};
81
92
  for (const candle of candles) {
82
93
  const dateKey = candle.FromDate.substring(0, 10);
83
94
  prices[dateKey] = candle.Close;
84
95
  }
85
96
 
86
- // Determine shard ID
87
97
  const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
88
- const docRef = db.collection('asset_prices').doc(shardId); // TODO implement config value
98
+ const docRef = db.collection('asset_prices').doc(shardId);
89
99
 
90
100
  const payload = {
91
101
  [instrumentId]: {
@@ -94,8 +104,7 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
94
104
  lastUpdated: FieldValue.serverTimestamp()
95
105
  }
96
106
  };
97
-
98
- // Write to Firestore
107
+
99
108
  await docRef.set(payload, { merge: true });
100
109
  logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
101
110
  successCount++;
@@ -108,9 +117,7 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
108
117
  });
109
118
 
110
119
  await Promise.all(promises);
111
-
112
- // Flush any remaining header updates
113
120
  await headerManager.flushPerformanceUpdates();
114
121
 
115
122
  logger.log('SUCCESS', `[PriceBackfill] Backfill complete. Success: ${successCount}, Failed: ${errorCount}`);
116
- };
123
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.103",
3
+ "version": "1.0.105",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [