bulltrackers-module 1.0.444 → 1.0.445

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/functions/computation-system/workflows/data_feeder_pipeline.yaml +32 -30
  2. package/package.json +1 -1
package/functions/computation-system/workflows/data_feeder_pipeline.yaml CHANGED
@@ -1,6 +1,5 @@
1
- # Data Feeder Pipeline
2
- # Orchestrates data fetching relative to Market Close (22:00 UTC) and Midnight (00:00 UTC).
3
- # Triggers at 22:00 UTC.
1
+ # Data Feeder Pipeline (V2)
2
+ # Orchestrates data fetching with UTC-alignment and Targeted Step execution.
4
3
 
5
4
  main:
6
5
  params: [input]
@@ -11,6 +10,13 @@ main:
11
10
  - location: "europe-west1"
12
11
  - market_date: '${text.split(time.format(sys.now()), "T")[0]}'
13
12
 
13
+ # --- TEST MODE / SELECTIVE EXECUTION ---
14
+ # Allows triggering specific steps by passing {"target_step": "step_name"} in the input.
15
+ - check_test_mode:
16
+ switch:
17
+ - condition: '${input != null and "target_step" in input}'
18
+ next: '${input.target_step}'
19
+
14
20
  # --- PHASE 1: MARKET CLOSE (22:00 UTC) ---
15
21
  - run_market_close_tasks:
16
22
  parallel:
@@ -30,8 +36,6 @@ main:
30
36
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
31
37
  auth: { type: OIDC }
32
38
 
33
- # [IMMEDIATE INDEX: MARKET DATA]
34
- # Update index for the market date we just fetched
35
39
  - index_market_data:
36
40
  call: http.post
37
41
  args:
@@ -40,15 +44,20 @@ main:
40
44
  targetDate: '${market_date}'
41
45
  auth: { type: OIDC }
42
46
 
43
- # --- PHASE 2: WAIT FOR MIDNIGHT ---
44
- # Trigger is at 22:00. We wait ~2 hours to align with 00:00 UTC.
47
+ # --- PHASE 2: ALIGN TO MIDNIGHT ---
48
+ # Dynamically calculates seconds remaining until 00:00 UTC.
45
49
  - wait_for_midnight:
50
+ assign:
51
+ - now_sec: '${int(sys.now())}'
52
+ - day_sec: 86400
53
+ # Calculate seconds until next 00:00 UTC window
54
+ - sleep_midnight: '${day_sec - (now_sec % day_sec)}'
55
+ - do_midnight_sleep:
46
56
  call: sys.sleep
47
57
  args:
48
- seconds: 7200 # 2 Hours
58
+ seconds: '${sleep_midnight}'
49
59
 
50
60
  # --- PHASE 3: RANKINGS & INITIAL SOCIAL (00:00 UTC) ---
51
- # Rankings must run first.
52
61
  - run_rankings_fetch:
53
62
  try:
54
63
  call: http.post
@@ -62,7 +71,7 @@ main:
62
71
  call: sys.log
63
72
  args:
64
73
  severity: "ERROR"
65
- text: '${"Rankings Fetch Failed. Proceeding to Social Fetch. Error: " + json.encode(e)}'
74
+ text: '${"Rankings Fetch Failed: " + json.encode(e)}'
66
75
 
67
76
  - run_social_midnight:
68
77
  call: http.post
@@ -70,14 +79,9 @@ main:
70
79
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
71
80
  auth: { type: OIDC }
72
81
 
73
- # [IMMEDIATE INDEX: MIDNIGHT DATA]
74
- # We recalculate the date because we are now past midnight
75
82
  - index_midnight_data:
76
83
  assign:
77
84
  - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
78
- next: call_index_midnight
79
-
80
- - call_index_midnight:
81
85
  call: http.post
82
86
  args:
83
87
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
@@ -85,19 +89,13 @@ main:
85
89
  targetDate: '${current_date}'
86
90
  auth: { type: OIDC }
87
91
 
88
- # --- PHASE 4: GLOBAL SYNC (Before 01:00 UTC) ---
89
- # Critical: Indexes EVERYTHING (no targetDate) to ensure the Computation System
90
- # (which starts at 01:00 UTC) has a fully consistent view of history.
91
92
  - run_global_indexer:
92
93
  call: http.post
93
94
  args:
94
95
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
95
- # No body = Full Scan
96
96
  auth: { type: OIDC }
97
- result: global_index_res
98
97
 
99
- # --- PHASE 5: RECURRING SOCIAL FETCH (Every 3 Hours) ---
100
- # We loop to cover the rest of the 24h cycle.
98
+ # --- PHASE 4: RECURRING SOCIAL FETCH (UTC Aligned 3hr) ---
101
99
  - init_social_loop:
102
100
  assign:
103
101
  - i: 0
@@ -106,10 +104,17 @@ main:
106
104
  switch:
107
105
  - condition: ${i < 7}
108
106
  steps:
109
- - wait_3_hours:
107
+ - calculate_next_window:
108
+ assign:
109
+ - now_sec_loop: '${int(sys.now())}'
110
+ - window_size: 10800 # 3 Hours
111
+ # Calculates remaining seconds to hit the next 3-hour mark (03, 06, 09...)
112
+ - sleep_loop: '${window_size - (now_sec_loop % window_size)}'
113
+
114
+ - wait_for_3hr_window:
110
115
  call: sys.sleep
111
116
  args:
112
- seconds: 10800 # 3 hours
117
+ seconds: '${sleep_loop}'
113
118
 
114
119
  - run_social_recurring:
115
120
  call: http.post
@@ -117,17 +122,14 @@ main:
117
122
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
118
123
  auth: { type: OIDC }
119
124
 
120
- # [IMMEDIATE INDEX: RECURRING]
121
- - calculate_recurring_date:
122
- assign:
123
- - current_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
124
-
125
125
  - index_recurring:
126
+ assign:
127
+ - cur_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
126
128
  call: http.post
127
129
  args:
128
130
  url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
129
131
  body:
130
- targetDate: '${current_date_rec}'
132
+ targetDate: '${cur_date_rec}'
131
133
  auth: { type: OIDC }
132
134
 
133
135
  - increment:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.444",
3
+ "version": "1.0.445",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [