bulltrackers-module 1.0.463 → 1.0.465
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
# Data Feeder Pipeline (V2.
|
|
2
|
-
# Orchestrates data fetching with UTC-alignment
|
|
1
|
+
# Data Feeder Pipeline (V2.3 - Try/Retry Syntax Fixed)
|
|
2
|
+
# Orchestrates data fetching with UTC-alignment, Test Mode, and Reliability.
|
|
3
3
|
|
|
4
4
|
main:
|
|
5
5
|
params: [input]
|
|
@@ -9,9 +9,16 @@ main:
|
|
|
9
9
|
- project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
|
|
10
10
|
- location: "europe-west1"
|
|
11
11
|
- market_date: '${text.split(time.format(sys.now()), "T")[0]}'
|
|
12
|
+
# Define a central retry policy to reuse across all HTTP calls
|
|
13
|
+
- default_retry:
|
|
14
|
+
predicate: ${http.default_retry_predicate}
|
|
15
|
+
max_retries: 5
|
|
16
|
+
backoff:
|
|
17
|
+
initial_delay: 2
|
|
18
|
+
max_delay: 60
|
|
19
|
+
multiplier: 2
|
|
12
20
|
|
|
13
21
|
# --- TEST MODE / SELECTIVE EXECUTION ---
|
|
14
|
-
# Since 'next' cannot be dynamic, we map specific strings to steps.
|
|
15
22
|
- check_test_mode:
|
|
16
23
|
switch:
|
|
17
24
|
- condition: '${input != null and "target_step" in input}'
|
|
@@ -34,25 +41,34 @@ main:
|
|
|
34
41
|
- price_fetch:
|
|
35
42
|
steps:
|
|
36
43
|
- call_price_fetcher:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
44
|
+
try:
|
|
45
|
+
call: http.post
|
|
46
|
+
args:
|
|
47
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
|
|
48
|
+
auth: { type: OIDC }
|
|
49
|
+
timeout: 300
|
|
50
|
+
retry: ${default_retry} # Fixed: Moved retry to a Try Step
|
|
41
51
|
- insights_fetch:
|
|
42
52
|
steps:
|
|
43
53
|
- call_insights_fetcher:
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
54
|
+
try:
|
|
55
|
+
call: http.post
|
|
56
|
+
args:
|
|
57
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
|
|
58
|
+
auth: { type: OIDC }
|
|
59
|
+
timeout: 300
|
|
60
|
+
retry: ${default_retry} # Fixed: Moved retry to a Try Step
|
|
48
61
|
|
|
49
62
|
- index_market_data:
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
63
|
+
try:
|
|
64
|
+
call: http.post
|
|
65
|
+
args:
|
|
66
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
|
|
67
|
+
body:
|
|
68
|
+
targetDate: '${market_date}'
|
|
69
|
+
auth: { type: OIDC }
|
|
70
|
+
timeout: 300
|
|
71
|
+
retry: ${default_retry}
|
|
56
72
|
|
|
57
73
|
# --- PHASE 2: ALIGN TO MIDNIGHT ---
|
|
58
74
|
- wait_for_midnight:
|
|
@@ -72,6 +88,8 @@ main:
|
|
|
72
88
|
args:
|
|
73
89
|
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
|
|
74
90
|
auth: { type: OIDC }
|
|
91
|
+
timeout: 300
|
|
92
|
+
retry: ${default_retry}
|
|
75
93
|
except:
|
|
76
94
|
as: e
|
|
77
95
|
steps:
|
|
@@ -82,27 +100,36 @@ main:
|
|
|
82
100
|
text: '${"Rankings Fetch Failed: " + json.encode(e)}'
|
|
83
101
|
|
|
84
102
|
- run_social_midnight:
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
103
|
+
try:
|
|
104
|
+
call: http.post
|
|
105
|
+
args:
|
|
106
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
|
|
107
|
+
auth: { type: OIDC }
|
|
108
|
+
timeout: 300
|
|
109
|
+
retry: ${default_retry}
|
|
89
110
|
|
|
90
111
|
- prepare_midnight_index:
|
|
91
112
|
assign:
|
|
92
113
|
- current_date: '${text.split(time.format(sys.now()), "T")[0]}'
|
|
93
114
|
- index_midnight_data:
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
115
|
+
try:
|
|
116
|
+
call: http.post
|
|
117
|
+
args:
|
|
118
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
|
|
119
|
+
body:
|
|
120
|
+
targetDate: '${current_date}'
|
|
121
|
+
auth: { type: OIDC }
|
|
122
|
+
timeout: 300
|
|
123
|
+
retry: ${default_retry}
|
|
100
124
|
|
|
101
125
|
- run_global_indexer:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
126
|
+
try:
|
|
127
|
+
call: http.post
|
|
128
|
+
args:
|
|
129
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
|
|
130
|
+
auth: { type: OIDC }
|
|
131
|
+
timeout: 300
|
|
132
|
+
retry: ${default_retry}
|
|
106
133
|
|
|
107
134
|
# --- PHASE 4: RECURRING SOCIAL FETCH (UTC Aligned 3hr) ---
|
|
108
135
|
- init_social_loop:
|
|
@@ -125,21 +152,27 @@ main:
|
|
|
125
152
|
seconds: '${sleep_loop}'
|
|
126
153
|
|
|
127
154
|
- run_social_recurring:
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
155
|
+
try:
|
|
156
|
+
call: http.post
|
|
157
|
+
args:
|
|
158
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
|
|
159
|
+
auth: { type: OIDC }
|
|
160
|
+
timeout: 300
|
|
161
|
+
retry: ${default_retry}
|
|
132
162
|
|
|
133
163
|
- prepare_recurring_index:
|
|
134
164
|
assign:
|
|
135
165
|
- cur_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
|
|
136
166
|
- index_recurring:
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
167
|
+
try:
|
|
168
|
+
call: http.post
|
|
169
|
+
args:
|
|
170
|
+
url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
|
|
171
|
+
body:
|
|
172
|
+
targetDate: '${cur_date_rec}'
|
|
173
|
+
auth: { type: OIDC }
|
|
174
|
+
timeout: 300
|
|
175
|
+
retry: ${default_retry}
|
|
143
176
|
|
|
144
177
|
- increment:
|
|
145
178
|
assign:
|