bulltrackers-module 1.0.464 → 1.0.465

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/functions/computation-system/workflows/data_feeder_pipeline.yaml +74 -55
  2. package/package.json +1 -1
package/functions/computation-system/workflows/data_feeder_pipeline.yaml CHANGED
@@ -1,5 +1,5 @@
1
- # Data Feeder Pipeline (Final Verified Version)
2
- # Orchestrates data fetching with UTC-alignment, Test Mode, and extended Timeouts.
1
+ # Data Feeder Pipeline (V2.3 - Try/Retry Syntax Fixed)
2
+ # Orchestrates data fetching with UTC-alignment, Test Mode, and Reliability.
3
3
 
4
4
  main:
5
5
  params: [input]
@@ -9,9 +9,16 @@ main:
9
9
  - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
10
10
  - location: "europe-west1"
11
11
  - market_date: '${text.split(time.format(sys.now()), "T")[0]}'
12
+ # Define a central retry policy to reuse across all HTTP calls
13
+ - default_retry:
14
+ predicate: ${http.default_retry_predicate}
15
+ max_retries: 5
16
+ backoff:
17
+ initial_delay: 2
18
+ max_delay: 60
19
+ multiplier: 2
12
20
 
13
21
  # --- TEST MODE / SELECTIVE EXECUTION ---
14
- # Static routing to allow manual testing of specific steps via Input JSON.
15
22
  - check_test_mode:
16
23
  switch:
17
24
  - condition: '${input != null and "target_step" in input}'
@@ -34,33 +41,36 @@ main:
34
41
  - price_fetch:
35
42
  steps:
36
43
  - call_price_fetcher:
37
- call: http.post
38
- args:
39
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
40
- auth: { type: OIDC }
41
- timeout: 300 # Fixed: Prevents timeout errors for long-running functions
42
- retry: ${http.default_retry}
44
+ try:
45
+ call: http.post
46
+ args:
47
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
48
+ auth: { type: OIDC }
49
+ timeout: 300
50
+ retry: ${default_retry} # Fixed: Moved retry to a Try Step
43
51
  - insights_fetch:
44
52
  steps:
45
53
  - call_insights_fetcher:
46
- call: http.post
47
- args:
48
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
49
- auth: { type: OIDC }
50
- timeout: 300 # Fixed: Prevents timeout errors for long-running functions
51
- retry: ${http.default_retry}
54
+ try:
55
+ call: http.post
56
+ args:
57
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
58
+ auth: { type: OIDC }
59
+ timeout: 300
60
+ retry: ${default_retry} # Fixed: Moved retry to a Try Step
52
61
 
53
62
  - index_market_data:
54
- call: http.post
55
- args:
56
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
57
- body:
58
- targetDate: '${market_date}'
59
- auth: { type: OIDC }
60
- timeout: 300
63
+ try:
64
+ call: http.post
65
+ args:
66
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
67
+ body:
68
+ targetDate: '${market_date}'
69
+ auth: { type: OIDC }
70
+ timeout: 300
71
+ retry: ${default_retry}
61
72
 
62
73
  # --- PHASE 2: ALIGN TO MIDNIGHT ---
63
- # Dynamically calculates seconds remaining until exactly 00:00 UTC.
64
74
  - wait_for_midnight:
65
75
  assign:
66
76
  - now_sec: '${int(sys.now())}'
@@ -79,6 +89,7 @@ main:
79
89
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
80
90
  auth: { type: OIDC }
81
91
  timeout: 300
92
+ retry: ${default_retry}
82
93
  except:
83
94
  as: e
84
95
  steps:
@@ -89,31 +100,36 @@ main:
89
100
  text: '${"Rankings Fetch Failed: " + json.encode(e)}'
90
101
 
91
102
  - run_social_midnight:
92
- call: http.post
93
- args:
94
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
95
- auth: { type: OIDC }
96
- timeout: 300
103
+ try:
104
+ call: http.post
105
+ args:
106
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
107
+ auth: { type: OIDC }
108
+ timeout: 300
109
+ retry: ${default_retry}
97
110
 
98
- # Fixed: Split assign and call into two separate steps
99
111
  - prepare_midnight_index:
100
112
  assign:
101
113
  - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
102
114
  - index_midnight_data:
103
- call: http.post
104
- args:
105
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
106
- body:
107
- targetDate: '${current_date}'
108
- auth: { type: OIDC }
109
- timeout: 300
115
+ try:
116
+ call: http.post
117
+ args:
118
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
119
+ body:
120
+ targetDate: '${current_date}'
121
+ auth: { type: OIDC }
122
+ timeout: 300
123
+ retry: ${default_retry}
110
124
 
111
125
  - run_global_indexer:
112
- call: http.post
113
- args:
114
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
115
- auth: { type: OIDC }
116
- timeout: 300
126
+ try:
127
+ call: http.post
128
+ args:
129
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
130
+ auth: { type: OIDC }
131
+ timeout: 300
132
+ retry: ${default_retry}
117
133
 
118
134
  # --- PHASE 4: RECURRING SOCIAL FETCH (UTC Aligned 3hr) ---
119
135
  - init_social_loop:
@@ -127,7 +143,7 @@ main:
127
143
  - calculate_next_window:
128
144
  assign:
129
145
  - now_sec_loop: '${int(sys.now())}'
130
- - window_size: 10800 # Fixed: Aligns to 03:00, 06:00, 09:00 UTC...
146
+ - window_size: 10800
131
147
  - sleep_loop: '${window_size - (now_sec_loop % window_size)}'
132
148
 
133
149
  - wait_for_3hr_window:
@@ -136,24 +152,27 @@ main:
136
152
  seconds: '${sleep_loop}'
137
153
 
138
154
  - run_social_recurring:
139
- call: http.post
140
- args:
141
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
142
- auth: { type: OIDC }
143
- timeout: 300
155
+ try:
156
+ call: http.post
157
+ args:
158
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
159
+ auth: { type: OIDC }
160
+ timeout: 300
161
+ retry: ${default_retry}
144
162
 
145
- # Fixed: Split assign and call into two separate steps
146
163
  - prepare_recurring_index:
147
164
  assign:
148
165
  - cur_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
149
166
  - index_recurring:
150
- call: http.post
151
- args:
152
- url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
153
- body:
154
- targetDate: '${cur_date_rec}'
155
- auth: { type: OIDC }
156
- timeout: 300
167
+ try:
168
+ call: http.post
169
+ args:
170
+ url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
171
+ body:
172
+ targetDate: '${cur_date_rec}'
173
+ auth: { type: OIDC }
174
+ timeout: 300
175
+ retry: ${default_retry}
157
176
 
158
177
  - increment:
159
178
  assign:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.464",
3
+ "version": "1.0.465",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [