bulltrackers-module 1.0.473 → 1.0.474

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/functions/computation-system/workflows/data_feeder_pipeline.yaml +147 -114
  2. package/package.json +1 -1
package/functions/computation-system/workflows/data_feeder_pipeline.yaml CHANGED
@@ -1,5 +1,6 @@
1
- # Data Feeder Pipeline (V2.3 - Try/Retry Syntax Fixed)
2
- # Orchestrates data fetching with UTC-alignment, Test Mode, and Reliability.
1
+ # Data Feeder Pipeline (V3.1 - Syntax Fixed)
2
+ # Starts at 22:00 UTC via Cloud Scheduler.
3
+ # Fixes: Split assign/call steps and corrected assign syntax.
3
4
 
4
5
  main:
5
6
  params: [input]
@@ -8,17 +9,8 @@ main:
8
9
  assign:
9
10
  - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
10
11
  - location: "europe-west1"
11
- - market_date: '${text.split(time.format(sys.now()), "T")[0]}'
12
- # Define a central retry policy to reuse across all HTTP calls
13
- - default_retry:
14
- predicate: ${http.default_retry_predicate}
15
- max_retries: 5
16
- backoff:
17
- initial_delay: 2
18
- max_delay: 60
19
- multiplier: 2
20
-
21
- # --- TEST MODE / SELECTIVE EXECUTION ---
12
+
13
+ # --- TEST MODE ---
22
14
  - check_test_mode:
23
15
  switch:
24
16
  - condition: '${input != null and "target_step" in input}'
@@ -26,130 +18,164 @@ main:
26
18
  - route_test:
27
19
  switch:
28
20
  - condition: '${input.target_step == "market"}'
29
- next: run_market_close_tasks
30
- - condition: '${input.target_step == "rankings"}'
31
- next: run_rankings_fetch
21
+ next: phase_2200_price
22
+ - condition: '${input.target_step == "midnight"}'
23
+ next: phase_0000_rankings
32
24
  - condition: '${input.target_step == "social"}'
33
- next: run_social_midnight
34
- - condition: '${input.target_step == "global"}'
35
- next: run_global_indexer
36
-
37
- # --- PHASE 1: MARKET CLOSE (22:00 UTC) ---
38
- - run_market_close_tasks:
39
- parallel:
40
- branches:
41
- - price_fetch:
42
- steps:
43
- - call_price_fetcher:
44
- try:
45
- call: http.post
46
- args:
47
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
48
- auth: { type: OIDC }
49
- timeout: 300
50
- retry: ${default_retry} # Fixed: Moved retry to a Try Step
51
- - insights_fetch:
52
- steps:
53
- - call_insights_fetcher:
54
- try:
55
- call: http.post
56
- args:
57
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
58
- auth: { type: OIDC }
59
- timeout: 300
60
- retry: ${default_retry} # Fixed: Moved retry to a Try Step
61
-
62
- - index_market_data:
25
+ next: social_loop_start
26
+
27
+ # ==========================================
28
+ # PHASE 1: MARKET CLOSE (Starts 22:00 UTC)
29
+ # ==========================================
30
+
31
+ - phase_2200_price:
63
32
  try:
64
33
  call: http.post
65
34
  args:
66
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
67
- body:
68
- targetDate: '${market_date}'
35
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
69
36
  auth: { type: OIDC }
70
37
  timeout: 300
71
- retry: ${default_retry}
38
+ except:
39
+ as: e
40
+ steps:
41
+ - log_price_error:
42
+ call: sys.log
43
+ args: { severity: "WARNING", text: "Price fetch timed out/failed. Proceeding." }
44
+
45
+ - wait_10_after_price:
46
+ call: sys.sleep
47
+ args: { seconds: 600 } # 10 Minutes
72
48
 
73
- # --- PHASE 2: ALIGN TO MIDNIGHT ---
74
- - wait_for_midnight:
49
+ # FIX 1: Split assign and call
50
+ - prepare_index_price:
75
51
  assign:
76
- - now_sec: '${int(sys.now())}'
77
- - day_sec: 86400
78
- - sleep_midnight: '${day_sec - (now_sec % day_sec)}'
79
- - do_midnight_sleep:
80
- call: sys.sleep
52
+ - today: '${text.split(time.format(sys.now()), "T")[0]}'
53
+ - index_today_after_price:
54
+ call: http.post
81
55
  args:
82
- seconds: '${sleep_midnight}'
56
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
57
+ body: { targetDate: '${today}' }
58
+ auth: { type: OIDC }
83
59
 
84
- # --- PHASE 3: RANKINGS & INITIAL SOCIAL (00:00 UTC) ---
85
- - run_rankings_fetch:
60
+ - wait_10_before_insights:
61
+ call: sys.sleep
62
+ args: { seconds: 600 }
63
+
64
+ - phase_2200_insights:
86
65
  try:
87
66
  call: http.post
88
67
  args:
89
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
68
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
90
69
  auth: { type: OIDC }
91
70
  timeout: 300
92
- retry: ${default_retry}
93
71
  except:
94
72
  as: e
95
73
  steps:
96
- - log_rankings_error:
74
+ - log_insights_error:
97
75
  call: sys.log
98
- args:
99
- severity: "ERROR"
100
- text: '${"Rankings Fetch Failed: " + json.encode(e)}'
76
+ args: { severity: "WARNING", text: "Insights fetch timed out/failed. Proceeding." }
101
77
 
102
- - run_social_midnight:
103
- try:
104
- call: http.post
105
- args:
106
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
107
- auth: { type: OIDC }
108
- timeout: 300
109
- retry: ${default_retry}
78
+ - wait_10_after_insights:
79
+ call: sys.sleep
80
+ args: { seconds: 600 }
110
81
 
111
- - prepare_midnight_index:
82
+ # FIX 2: Split assign and call
83
+ - prepare_index_insights:
112
84
  assign:
113
- - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
114
- - index_midnight_data:
85
+ - today: '${text.split(time.format(sys.now()), "T")[0]}'
86
+ - index_today_after_insights:
87
+ call: http.post
88
+ args:
89
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
90
+ body: { targetDate: '${today}' }
91
+ auth: { type: OIDC }
92
+
93
+ # ==========================================
94
+ # PHASE 2: WAIT FOR MIDNIGHT
95
+ # ==========================================
96
+
97
+ - align_to_midnight:
98
+ assign:
99
+ - now_sec: '${int(sys.now())}'
100
+ - day_sec: 86400
101
+ - sleep_midnight: '${day_sec - (now_sec % day_sec)}'
102
+ - wait_for_midnight:
103
+ call: sys.sleep
104
+ args: { seconds: '${sleep_midnight}' }
105
+
106
+ # ==========================================
107
+ # PHASE 3: MIDNIGHT TASKS (00:00 UTC)
108
+ # ==========================================
109
+
110
+ - phase_0000_rankings:
115
111
  try:
116
112
  call: http.post
117
113
  args:
118
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
119
- body:
120
- targetDate: '${current_date}'
114
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
121
115
  auth: { type: OIDC }
122
116
  timeout: 300
123
- retry: ${default_retry}
117
+ except:
118
+ as: e
119
+ steps:
120
+ - log_ranking_error:
121
+ call: sys.log
122
+ args: { severity: "WARNING", text: "Rankings failed. Proceeding to Social (risky)." }
124
123
 
125
- - run_global_indexer:
124
+ - wait_10_after_rankings:
125
+ call: sys.sleep
126
+ args: { seconds: 600 }
127
+
128
+ # FIX 3: Split assign and call
129
+ - prepare_index_rankings:
130
+ assign:
131
+ - today: '${text.split(time.format(sys.now()), "T")[0]}'
132
+ - index_today_after_rankings:
133
+ call: http.post
134
+ args:
135
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
136
+ body: { targetDate: '${today}' }
137
+ auth: { type: OIDC }
138
+
139
+ - phase_0000_social:
126
140
  try:
127
141
  call: http.post
128
142
  args:
129
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
143
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
130
144
  auth: { type: OIDC }
131
145
  timeout: 300
132
- retry: ${default_retry}
146
+ except:
147
+ as: e
148
+ steps:
149
+ - log_social_error:
150
+ call: sys.log
151
+ args: { severity: "WARNING", text: "Social failed. Proceeding." }
152
+
153
+ - wait_10_after_social:
154
+ call: sys.sleep
155
+ args: { seconds: 600 }
156
+
157
+ - global_index_midnight:
158
+ call: http.post
159
+ args:
160
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
161
+ # No targetDate = Global Run
162
+ auth: { type: OIDC }
163
+
164
+ # ==========================================
165
+ # PHASE 4: SOCIAL LOOP (Every 3 Hours)
166
+ # ==========================================
133
167
 
134
- # --- PHASE 4: RECURRING SOCIAL FETCH (UTC Aligned 3hr) ---
135
168
  - init_social_loop:
136
169
  assign:
137
170
  - i: 0
138
171
 
139
- - social_loop:
172
+ - social_loop_start:
140
173
  switch:
141
- - condition: ${i < 7}
174
+ - condition: ${i < 7} # Covers the remainder of the 24h cycle
142
175
  steps:
143
- - calculate_next_window:
144
- assign:
145
- - now_sec_loop: '${int(sys.now())}'
146
- - window_size: 10800
147
- - sleep_loop: '${window_size - (now_sec_loop % window_size)}'
148
-
149
- - wait_for_3hr_window:
176
+ - wait_3_hours:
150
177
  call: sys.sleep
151
- args:
152
- seconds: '${sleep_loop}'
178
+ args: { seconds: 10800 }
153
179
 
154
180
  - run_social_recurring:
155
181
  try:
@@ -158,27 +184,34 @@ main:
158
184
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
159
185
  auth: { type: OIDC }
160
186
  timeout: 300
161
- retry: ${default_retry}
187
+ except:
188
+ as: e
189
+ steps:
190
+ - log_loop_social_warn:
191
+ call: sys.log
192
+ args: { severity: "WARNING", text: "Loop Social timed out. Proceeding." }
162
193
 
163
- - prepare_recurring_index:
194
+ - wait_10_in_loop:
195
+ call: sys.sleep
196
+ args: { seconds: 600 }
197
+
198
+ # FIX 4: Split assign and call
199
+ - prepare_index_loop:
164
200
  assign:
165
- - cur_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
166
- - index_recurring:
167
- try:
168
- call: http.post
169
- args:
170
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
171
- body:
172
- targetDate: '${cur_date_rec}'
173
- auth: { type: OIDC }
174
- timeout: 300
175
- retry: ${default_retry}
201
+ - today: '${text.split(time.format(sys.now()), "T")[0]}'
202
+ - index_today_in_loop:
203
+ call: http.post
204
+ args:
205
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
206
+ body: { targetDate: '${today}' }
207
+ auth: { type: OIDC }
176
208
 
177
- - increment:
178
- assign:
179
- - i: ${i + 1}
209
+ # FIX 5: Correct assign syntax (must be a list)
210
+ - increment_loop:
211
+ assign:
212
+ - i: '${i + 1}'
180
213
  - next_iteration:
181
- next: social_loop
214
+ next: social_loop_start
182
215
 
183
216
  - finish:
184
- return: "Daily Data Pipeline Completed"
217
+ return: "Complete 24h Cycle Finished"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.473",
3
+ "version": "1.0.474",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [