bulltrackers-module 1.0.303 → 1.0.304

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/functions/computation-system/helpers/computation_dispatcher.js +29 -29
  2. package/functions/computation-system/workflows/bulltrackers_pipeline.yaml +28 -17
  3. package/package.json +1 -1
package/functions/computation-system/helpers/computation_dispatcher.js CHANGED
@@ -20,9 +20,9 @@ const SECONDS_PER_CALC_MARGIN = 25; // 20s base + safety margin
20
20
  async function getHighMemReroutes(db, date, pass, tasks) {
21
21
  const reroutes = [];
22
22
  for (const task of tasks) {
23
- const name = normalizeName(task.name);
23
+ const name = normalizeName(task.name);
24
24
  const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
25
- const doc = await db.doc(ledgerPath).get();
25
+ const doc = await db.doc(ledgerPath).get();
26
26
 
27
27
  if (doc.exists) {
28
28
  const data = doc.data();
@@ -41,15 +41,15 @@ async function getHighMemReroutes(db, date, pass, tasks) {
41
41
 
42
42
  async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
43
43
  const { logger, db } = dependencies;
44
- const pubsubUtils = new PubSubUtils(dependencies);
44
+ const pubsubUtils = new PubSubUtils(dependencies);
45
45
 
46
46
  // Inputs from Workflow Cursor
47
- const passToRun = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN);
47
+ const passToRun = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN);
48
48
  const targetCursorN = parseInt(reqBody.cursorIndex || 1);
49
- const dateLimitStr = reqBody.date || config.date;
49
+ const dateLimitStr = reqBody.date || config.date;
50
50
 
51
- const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
52
- const passes = groupByPass(computationManifest);
51
+ const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
52
+ const passes = groupByPass(computationManifest);
53
53
  const calcsInThisPass = passes[passToRun] || [];
54
54
 
55
55
  if (!calcsInThisPass.length) {
@@ -58,35 +58,35 @@ async function dispatchComputationPass(config, dependencies, computationManifest
58
58
 
59
59
  // 1. Discover all "Dirty" Dates (Dates needing work)
60
60
  const earliestDates = await getEarliestDataDates(config, dependencies);
61
- const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
61
+ const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
62
62
 
63
63
  const dirtyDates = [];
64
64
  for (const d of allDates) {
65
- const dailyStatus = await fetchComputationStatus(d, config, dependencies);
65
+ const dailyStatus = await fetchComputationStatus(d, config, dependencies);
66
66
  const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
67
67
 
68
68
  const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
69
- const tasks = [...report.runnable, ...report.reRuns];
69
+ const tasks = [...report.runnable, ...report.reRuns];
70
70
 
71
71
  if (tasks.length > 0) {
72
72
  dirtyDates.push({ date: d, tasks });
73
73
  }
74
74
  }
75
75
 
76
- let selectedDate = null;
76
+ let selectedDate = null;
77
77
  let selectedTasks = [];
78
- let isReroute = false;
79
- let isSweep = false;
78
+ let isReroute = false;
79
+ let isSweep = false;
80
80
 
81
81
  // 2. Logic: Prioritize Reroute for N-1
82
82
  if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
83
83
  const prevEntry = dirtyDates[targetCursorN - 2];
84
- const reroutes = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
84
+ const reroutes = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
85
85
 
86
86
  if (reroutes.length > 0) {
87
- selectedDate = prevEntry.date;
87
+ selectedDate = prevEntry.date;
88
88
  selectedTasks = reroutes;
89
- isReroute = true;
89
+ isReroute = true;
90
90
  logger.log('INFO', `[Dispatcher] Reroute detected for ${selectedDate}. Pausing N increment.`);
91
91
  }
92
92
  }
@@ -94,14 +94,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
94
94
  // 3. Logic: N-th Dirty Date or Final Sweep
95
95
  if (!selectedDate) {
96
96
  if (targetCursorN <= dirtyDates.length) {
97
- const entry = dirtyDates[targetCursorN - 1];
98
- selectedDate = entry.date;
97
+ const entry = dirtyDates[targetCursorN - 1];
98
+ selectedDate = entry.date;
99
99
  selectedTasks = entry.tasks;
100
100
  } else {
101
101
  // Final Satiation Sweep: Check if anything was missed (recovery)
102
102
  if (dirtyDates.length > 0) {
103
- isSweep = true;
104
- selectedDate = dirtyDates[0].date;
103
+ isSweep = true;
104
+ selectedDate = dirtyDates[0].date;
105
105
  selectedTasks = dirtyDates[0].tasks;
106
106
  }
107
107
  }
@@ -114,21 +114,21 @@ async function dispatchComputationPass(config, dependencies, computationManifest
114
114
 
115
115
  // 5. Dispatch to PubSub (Standard vs. High-Mem)
116
116
  const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
117
- const highMemTasks = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
117
+ const highMemTasks = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
118
118
 
119
119
  const pubPromises = [];
120
120
  if (standardTasks.length > 0) {
121
121
  pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
122
122
  topicName: config.computationTopicStandard || 'computation-tasks',
123
- tasks: standardTasks,
124
- taskType: `pass-${passToRun}-std`
123
+ tasks : standardTasks,
124
+ taskType : `pass-${passToRun}-std`
125
125
  }));
126
126
  }
127
127
  if (highMemTasks.length > 0) {
128
128
  pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
129
129
  topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
130
- tasks: highMemTasks,
131
- taskType: `pass-${passToRun}-high`
130
+ tasks : highMemTasks,
131
+ taskType : `pass-${passToRun}-high`
132
132
  }));
133
133
  }
134
134
  await Promise.all(pubPromises);
@@ -138,11 +138,11 @@ async function dispatchComputationPass(config, dependencies, computationManifest
138
138
  logger.log('INFO', `[Dispatcher] ${isReroute ? 'Reroute' : (isSweep ? 'Sweep' : 'Standard')} Run: ${selectedDate}. Tasks: ${selectedTasks.length}. ETA: ${etaSeconds}s`);
139
139
 
140
140
  return {
141
- status: isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
142
- dateProcessed: selectedDate,
143
- dispatched: selectedTasks.length,
141
+ status : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
142
+ dateProcessed : selectedDate,
143
+ dispatched : selectedTasks.length,
144
144
  n_cursor_ignored: isReroute, // Tell workflow to stay on same N
145
- etaSeconds: etaSeconds
145
+ etaSeconds : etaSeconds
146
146
  };
147
147
  }
148
148
 
package/functions/computation-system/workflows/bulltrackers_pipeline.yaml CHANGED
@@ -1,12 +1,14 @@
1
1
  # Cloud Workflows: Precision Cursor-Based Orchestrator
2
+ # PURPOSE: Orchestrates 5 passes with deterministic pacing and syntax fixes.
3
+
2
4
  main:
3
5
  params: [input]
4
6
  steps:
5
7
  - init:
6
8
  assign:
7
- - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
9
+ - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
8
10
  - passes: ["1", "2", "3", "4", "5"]
9
- - date_to_run: ${default(map.get(input, "date"), "2025-01-01")}
11
+ - date_to_run: '${default(map.get(input, "date"), "2025-01-01")}'
10
12
 
11
13
  - run_sequential_passes:
12
14
  for:
@@ -25,29 +27,38 @@ main:
25
27
  - call_dispatcher:
26
28
  call: http.post
27
29
  args:
28
- url: ${"https://europe-west1-" + project + ".cloudfunctions.net/dispatch-pass-" + pass_id}
30
+ url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}'
29
31
  body:
30
- pass: ${pass_id}
31
- cursorIndex: ${n_cursor}
32
- date: ${date_to_run}
32
+ pass: '${pass_id}'
33
+ cursorIndex: '${n_cursor}'
34
+ date: '${date_to_run}'
33
35
  auth: { type: OIDC }
34
36
  result: dispatch_res
35
37
 
36
38
  - evaluate_dispatch:
37
39
  switch:
38
- # State 1: Dispatcher signals pass is fully completed
39
- - condition: ${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}
40
- assign: [pass_complete: true]
40
+ # State 1: Pass exhausted and Satiation Sweep complete
41
+ - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
42
+ assign:
43
+ - pass_complete: true
41
44
 
42
- # State 2: Tasks dispatched (Standard, Reroute, or Recovery)
43
- - condition: ${dispatch_res.body.dispatched > 0}
45
+ # State 2: Tasks dispatched (Standard, Reroute, or Recovery Sweep)
46
+ - condition: '${dispatch_res.body.dispatched > 0}'
44
47
  steps:
45
- - wait_for_eta:
48
+ - log_dispatch:
49
+ call: sys.log
50
+ args:
51
+ text: '${"Pass " + pass_id + " - Dispatching " + string(dispatch_res.body.dispatched) + " tasks for " + dispatch_res.body.dateProcessed + ". ETA: " + string(dispatch_res.body.etaSeconds) + "s."}'
52
+ - wait_for_completion:
46
53
  call: sys.sleep
47
- args: { seconds: ${dispatch_res.body.etaSeconds} }
54
+ args:
55
+ seconds: '${int(dispatch_res.body.etaSeconds)}'
48
56
  - update_cursor:
49
57
  assign:
50
- # If it was a reroute, stay on the same N to retry the sequence
51
- - n_cursor: ${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}
52
- - next_step:
53
- next: sequential_date_loop
58
+ # If n_cursor_ignored is true (Reroute or Sweep Recovery), we stay on same N
59
+ - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
60
+ - next_loop:
61
+ next: sequential_date_loop
62
+
63
+ - finish:
64
+ return: "Pipeline Execution Satiated and Complete"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.303",
3
+ "version": "1.0.304",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [