bulltrackers-module 1.0.759 → 1.0.761
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,33 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Computation Dispatcher
|
|
3
|
-
*
|
|
4
|
-
* 1. Receive HTTP request (from Cloud Tasks or User)
|
|
5
|
-
* 2. Delegate execution to the Orchestrator
|
|
6
|
-
* 3. Translate Orchestrator STATUS (blocked, completed) into HTTP CODES (503, 200)
|
|
7
|
-
* * Why is this file so short?
|
|
8
|
-
* All validation logic (Schedule, Dependencies, Data) has moved to
|
|
9
|
-
* framework/core/RunAnalyzer.js to ensure consistency between
|
|
10
|
-
* the Scheduler and the Worker.
|
|
3
|
+
* ...
|
|
11
4
|
*/
|
|
12
5
|
|
|
13
|
-
|
|
14
|
-
// We will lazy-load this inside the handler to prevent circular dependency cycles.
|
|
6
|
+
const crypto = require('crypto');
|
|
15
7
|
|
|
16
8
|
exports.dispatcherHandler = async (req, res) => {
|
|
17
9
|
const startTime = Date.now();
|
|
18
10
|
|
|
19
11
|
try {
|
|
20
|
-
// LAZY LOAD: Require index.js here to ensure it is fully initialized
|
|
21
|
-
// This fixes the "Accessing non-existent property inside circular dependency" error
|
|
22
12
|
const system = require('../index');
|
|
23
13
|
|
|
24
14
|
const {
|
|
25
15
|
computationName,
|
|
26
16
|
targetDate,
|
|
27
|
-
source = 'scheduled',
|
|
28
|
-
entityIds,
|
|
17
|
+
source = 'scheduled',
|
|
18
|
+
entityIds,
|
|
29
19
|
dryRun = false,
|
|
30
|
-
force = false
|
|
20
|
+
force = false,
|
|
21
|
+
configHash
|
|
31
22
|
} = req.body || {};
|
|
32
23
|
|
|
33
24
|
// 1. Basic Validation
|
|
@@ -38,7 +29,37 @@ exports.dispatcherHandler = async (req, res) => {
|
|
|
38
29
|
message: 'computationName is required'
|
|
39
30
|
});
|
|
40
31
|
}
|
|
41
|
-
|
|
32
|
+
|
|
33
|
+
// [FIXED LOGIC HERE] --------------------------------------------------
|
|
34
|
+
// Stale Task Protection
|
|
35
|
+
if (configHash && !force) {
|
|
36
|
+
// FIX: Use getManifest() as system.manifest is not exposed directly.
|
|
37
|
+
const manifest = await system.getManifest();
|
|
38
|
+
|
|
39
|
+
// Normalize name to match manifest keys (matches logic in core-api.js)
|
|
40
|
+
const normalizedName = computationName.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
41
|
+
const entry = manifest.find(c => c.name === normalizedName);
|
|
42
|
+
|
|
43
|
+
if (entry) {
|
|
44
|
+
// 1. Re-calculate the hash of the CURRENTLY DEPLOYED code
|
|
45
|
+
const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
|
|
46
|
+
const currentHash = crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
|
|
47
|
+
|
|
48
|
+
// 2. Compare
|
|
49
|
+
if (configHash !== currentHash) {
|
|
50
|
+
console.warn(`[Dispatcher] ♻️ Skipped STALE task for ${computationName}. (Task Hash: ${configHash} != Current: ${currentHash})`);
|
|
51
|
+
|
|
52
|
+
return res.status(200).json({
|
|
53
|
+
status: 'skipped',
|
|
54
|
+
reason: 'STALE_CONFIG',
|
|
55
|
+
message: 'Task configuration (schedule/pass) is obsolete relative to current deployment.',
|
|
56
|
+
hash: currentHash
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// ---------------------------------------------------------------------
|
|
62
|
+
|
|
42
63
|
const date = targetDate || new Date().toISOString().split('T')[0];
|
|
43
64
|
console.log(`[Dispatcher] Received ${source} request: ${computationName} for ${date}`);
|
|
44
65
|
|
|
@@ -48,16 +69,12 @@ exports.dispatcherHandler = async (req, res) => {
|
|
|
48
69
|
}
|
|
49
70
|
|
|
50
71
|
// 2. DELEGATE TO ORCHESTRATOR
|
|
51
|
-
// The Orchestrator calls RunAnalyzer internally to check:
|
|
52
|
-
// - Is it scheduled?
|
|
53
|
-
// - Are dependencies ready?
|
|
54
|
-
// - Is data available?
|
|
55
|
-
// - Has the code changed?
|
|
56
72
|
const result = await system.runComputation({
|
|
57
73
|
date,
|
|
58
74
|
computation: computationName,
|
|
59
75
|
entityIds,
|
|
60
|
-
dryRun
|
|
76
|
+
dryRun,
|
|
77
|
+
force // Pass force down to orchestrator if needed
|
|
61
78
|
});
|
|
62
79
|
|
|
63
80
|
const duration = Date.now() - startTime;
|
|
@@ -78,11 +95,6 @@ exports.dispatcherHandler = async (req, res) => {
|
|
|
78
95
|
}
|
|
79
96
|
|
|
80
97
|
// 4. HANDLE NON-RUNNABLE STATES (Blocked / Impossible)
|
|
81
|
-
// NEW BEHAVIOUR:
|
|
82
|
-
// - We NEVER return 503 for logical states like "blocked" or "impossible".
|
|
83
|
-
// - Cloud Tasks retries are reserved for genuine execution failures (5xx from errors).
|
|
84
|
-
// - Scheduler + dependency cascade should avoid dispatching truly blocked tasks;
|
|
85
|
-
// if we still see them here, we surface the status once and BIN the task.
|
|
86
98
|
if (result.status === 'blocked' || result.status === 'impossible') {
|
|
87
99
|
console.log(`[Dispatcher] ${computationName} ${result.status}: ${result.reason}`);
|
|
88
100
|
|
|
@@ -93,14 +105,13 @@ exports.dispatcherHandler = async (req, res) => {
|
|
|
93
105
|
});
|
|
94
106
|
}
|
|
95
107
|
|
|
96
|
-
// 5. Fallback for other statuses
|
|
108
|
+
// 5. Fallback for other statuses
|
|
97
109
|
return res.status(200).json(result);
|
|
98
110
|
|
|
99
111
|
} catch (error) {
|
|
100
112
|
const duration = Date.now() - startTime;
|
|
101
113
|
console.error(`[Dispatcher] Error after ${duration}ms:`, error);
|
|
102
114
|
|
|
103
|
-
// Handle "Not Found" specifically
|
|
104
115
|
if (error.message && error.message.includes('Computation not found')) {
|
|
105
116
|
return res.status(400).json({
|
|
106
117
|
status: 'error',
|
|
@@ -109,7 +120,6 @@ exports.dispatcherHandler = async (req, res) => {
|
|
|
109
120
|
});
|
|
110
121
|
}
|
|
111
122
|
|
|
112
|
-
// Return 500 to trigger Cloud Tasks retry for crash/network errors
|
|
113
123
|
return res.status(500).json({
|
|
114
124
|
status: 'error',
|
|
115
125
|
reason: 'EXECUTION_FAILED',
|
|
@@ -2,22 +2,26 @@
|
|
|
2
2
|
* @fileoverview Cloud Function Handlers
|
|
3
3
|
*
|
|
4
4
|
* Export handlers for deployment as Cloud Functions:
|
|
5
|
-
* -
|
|
5
|
+
* - computePlanner: Plans root computations into Cloud Tasks
|
|
6
|
+
* - computeWatchdog: Recovers zombies / stuck computations
|
|
6
7
|
* - computeDispatcher: Receives tasks from Cloud Tasks queue
|
|
7
8
|
* - computeOnDemand: Receives requests from frontend
|
|
8
9
|
* - computationWorker: Serverless worker for entity-level computation
|
|
9
10
|
*/
|
|
10
11
|
|
|
11
|
-
const {
|
|
12
|
+
const { planComputations, runWatchdog } = require('./scheduler');
|
|
12
13
|
const { dispatcherHandler } = require('./dispatcher');
|
|
13
14
|
const { onDemandHandler } = require('./onDemand');
|
|
14
15
|
const { workerHandler, executeLocal } = require('./worker');
|
|
15
16
|
const { adminTestHandler } = require('./adminTest');
|
|
16
17
|
|
|
17
18
|
module.exports = {
|
|
18
|
-
//
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
// Planner: projects upcoming root computations into Cloud Tasks
|
|
20
|
+
computePlanner: planComputations,
|
|
21
|
+
|
|
22
|
+
// Watchdog: detects and recovers zombies
|
|
23
|
+
computeWatchdog: runWatchdog,
|
|
24
|
+
|
|
21
25
|
// Main dispatcher - handles scheduled tasks from Cloud Tasks
|
|
22
26
|
computeDispatcher: dispatcherHandler,
|
|
23
27
|
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview
|
|
3
|
-
* *
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* @fileoverview Scheduler V2: Planner & Watchdog
|
|
3
|
+
* * 1. planComputations: Runs infrequently (e.g. Hourly/Daily).
|
|
4
|
+
* - Loads Manifest.
|
|
5
|
+
* - Forecasts all Root executions for the next 24-48h.
|
|
6
|
+
* - Enqueues Cloud Tasks with `scheduleTime` and `configHash`.
|
|
7
|
+
* * 2. runWatchdog: Runs frequently (e.g. every 15 mins).
|
|
8
|
+
* - Detects Zombies (stuck running tasks).
|
|
9
|
+
* - Re-queues them immediately.
|
|
7
10
|
*/
|
|
8
11
|
|
|
9
12
|
const { CloudTasksClient } = require('@google-cloud/tasks');
|
|
13
|
+
const crypto = require('crypto');
|
|
10
14
|
const pLimit = require('p-limit');
|
|
11
15
|
const { ManifestBuilder } = require('../framework');
|
|
12
16
|
const { StorageManager } = require('../framework/storage/StorageManager');
|
|
@@ -14,188 +18,226 @@ const config = require('../config/bulltrackers.config');
|
|
|
14
18
|
|
|
15
19
|
const CLOUD_TASKS_CONCURRENCY = 10;
|
|
16
20
|
const ZOMBIE_THRESHOLD_MINUTES = 15;
|
|
21
|
+
const PLANNING_WINDOW_HOURS = 24; // Look ahead window
|
|
17
22
|
|
|
23
|
+
// Cache singleton instances
|
|
18
24
|
let manifest = null;
|
|
19
25
|
let tasksClient = null;
|
|
20
26
|
let storageManager = null;
|
|
21
27
|
|
|
22
28
|
async function initialize() {
|
|
23
29
|
if (manifest) return;
|
|
30
|
+
console.log('[Scheduler] Initializing services...');
|
|
24
31
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// Core Services
|
|
28
|
-
// We pass a no-op logger to ManifestBuilder to keep logs clean during frequent scheduling
|
|
32
|
+
// We pass a no-op logger to prevent noise during frequent checks
|
|
29
33
|
const builder = new ManifestBuilder(config, { log: () => {} });
|
|
30
34
|
manifest = builder.build(config.computations || []);
|
|
31
35
|
|
|
32
|
-
// Infrastructure
|
|
33
36
|
tasksClient = new CloudTasksClient();
|
|
34
37
|
storageManager = new StorageManager(config, console);
|
|
35
38
|
|
|
36
|
-
console.log(`[Scheduler]
|
|
39
|
+
console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
|
|
37
40
|
}
|
|
38
41
|
|
|
39
|
-
|
|
42
|
+
/**
|
|
43
|
+
* ENTRY POINT 1: The Planner
|
|
44
|
+
* Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
|
|
45
|
+
* Goals: Ensure all future tasks for the next 24h are in the queue.
|
|
46
|
+
*/
|
|
47
|
+
async function planComputations(req, res) {
|
|
40
48
|
const startTime = Date.now();
|
|
41
|
-
|
|
42
49
|
try {
|
|
43
50
|
await initialize();
|
|
44
|
-
|
|
45
|
-
const now = new Date(); // Exact current time
|
|
46
|
-
const targetDate = now.toISOString().split('T')[0];
|
|
47
|
-
|
|
48
|
-
// 1. ROLLING WINDOW SCHEDULE
|
|
49
|
-
// Strategy: Look ahead 60 minutes.
|
|
50
|
-
// If a task is due in this window, we dispatch it to Cloud Tasks with a 'scheduleTime'.
|
|
51
|
-
// Cloud Tasks deduplication (via task name) ensures we don't schedule it twice.
|
|
52
|
-
const windowEnd = new Date(now.getTime() + 60 * 60 * 1000);
|
|
53
|
-
|
|
54
|
-
const dueComputations = findDueComputations(now, windowEnd);
|
|
55
|
-
|
|
56
|
-
if (dueComputations.length > 0) {
|
|
57
|
-
console.log(`[Scheduler] Found ${dueComputations.length} Pass 0 tasks due between ${formatTime(now)} and ${formatTime(windowEnd)}`);
|
|
58
|
-
}
|
|
59
51
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
52
|
+
const now = new Date();
|
|
53
|
+
const windowEnd = new Date(now.getTime() + PLANNING_WINDOW_HOURS * 60 * 60 * 1000);
|
|
54
|
+
|
|
55
|
+
console.log(`[Planner] Planning window: ${now.toISOString()} to ${windowEnd.toISOString()}`);
|
|
56
|
+
|
|
57
|
+
const tasksToSchedule = [];
|
|
58
|
+
|
|
59
|
+
// 1. Walk the Manifest
|
|
60
|
+
for (const entry of manifest) {
|
|
61
|
+
// FILTER: Only Roots (Pass 0)
|
|
62
|
+
// Resilience: If code changes and a comp becomes Pass 1, it won't be scheduled here.
|
|
63
|
+
if (entry.pass !== 0) continue;
|
|
64
|
+
|
|
65
|
+
// Calculate Occurrences
|
|
66
|
+
const occurrences = getOccurrencesInWindow(entry.schedule, now, windowEnd);
|
|
65
67
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
68
|
+
// Generate Tasks for each occurrence
|
|
69
|
+
for (const dateObj of occurrences) {
|
|
70
|
+
// Resilience: Generate a hash of the critical scheduling config.
|
|
71
|
+
// If schedule OR pass changes, this hash changes, creating a new Task ID.
|
|
72
|
+
const configHash = generateConfigHash(entry);
|
|
73
|
+
const targetDateStr = dateObj.toISOString().split('T')[0];
|
|
74
|
+
|
|
75
|
+
tasksToSchedule.push({
|
|
76
|
+
computation: entry.originalName,
|
|
77
|
+
targetDate: targetDateStr,
|
|
78
|
+
runAtSeconds: dateObj.getTime() / 1000,
|
|
79
|
+
configHash: configHash,
|
|
80
|
+
queuePath: getQueuePath(entry)
|
|
81
|
+
});
|
|
74
82
|
}
|
|
75
|
-
} catch (e) {
|
|
76
|
-
console.error(`[Scheduler] Zombie check failed: ${e.message}`);
|
|
77
83
|
}
|
|
78
84
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
return res.status(200).json({ status: 'ok', message: 'Nothing due' });
|
|
85
|
+
if (tasksToSchedule.length === 0) {
|
|
86
|
+
return res.status(200).send('No root computations due in planning window.');
|
|
82
87
|
}
|
|
83
88
|
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
// 2. Dispatch to Cloud Tasks (Idempotent)
|
|
90
|
+
const results = await dispatchPlannedTasks(tasksToSchedule);
|
|
91
|
+
|
|
92
|
+
const created = results.filter(r => r.status === 'scheduled').length;
|
|
93
|
+
const exists = results.filter(r => r.status === 'exists').length;
|
|
94
|
+
|
|
95
|
+
console.log(`[Planner] Window Processed. Created: ${created}, Already Existed: ${exists}, Errors: ${results.length - created - exists}`);
|
|
96
|
+
|
|
97
|
+
return res.status(200).json({
|
|
98
|
+
status: 'ok',
|
|
99
|
+
window: `${PLANNING_WINDOW_HOURS}h`,
|
|
100
|
+
created,
|
|
101
|
+
exists,
|
|
102
|
+
details: results
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
} catch (error) {
|
|
106
|
+
console.error('[Planner] Fatal Error:', error);
|
|
107
|
+
return res.status(500).json({ error: error.message });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* ENTRY POINT 2: The Watchdog
|
|
113
|
+
* Trigger: Cloud Scheduler -> "*\/15 * * * *" (Every 15 mins)
|
|
114
|
+
* Goals: Find stuck tasks and re-queue them.
|
|
115
|
+
*/
|
|
116
|
+
async function runWatchdog(req, res) {
|
|
117
|
+
try {
|
|
118
|
+
await initialize();
|
|
119
|
+
|
|
120
|
+
// 1. Find Zombies
|
|
121
|
+
const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
|
|
122
|
+
|
|
123
|
+
if (zombies.length === 0) {
|
|
124
|
+
return res.status(200).send('No zombies detected.');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
console.log(`[Watchdog] 🧟 Found ${zombies.length} zombies. Initiating recovery...`);
|
|
128
|
+
|
|
129
|
+
// 2. Claim & Recover
|
|
130
|
+
// We claim them first so the next watchdog doesn't grab them while we are dispatching
|
|
131
|
+
await Promise.all(zombies.map(z => storageManager.claimZombie(z.checkpointId)));
|
|
132
|
+
|
|
133
|
+
const recoveryTasks = zombies.map(z => {
|
|
134
|
+
const entry = manifest.find(m => m.name === z.name);
|
|
135
|
+
if (!entry) {
|
|
136
|
+
console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
90
139
|
return {
|
|
91
|
-
|
|
140
|
+
computation: entry.originalName,
|
|
141
|
+
targetDate: z.date,
|
|
92
142
|
isRecovery: true,
|
|
93
|
-
originalDate: z.date,
|
|
94
143
|
recoveryId: z.checkpointId,
|
|
95
|
-
|
|
144
|
+
queuePath: getQueuePath(entry)
|
|
96
145
|
};
|
|
97
146
|
}).filter(Boolean);
|
|
98
147
|
|
|
99
|
-
const
|
|
148
|
+
const results = await dispatchRecoveryTasks(recoveryTasks);
|
|
100
149
|
|
|
101
|
-
const results = await dispatchComputations(allTasks, targetDate);
|
|
102
|
-
|
|
103
|
-
const duration = Date.now() - startTime;
|
|
104
|
-
|
|
105
150
|
return res.status(200).json({
|
|
106
|
-
status: '
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
errors: results.filter(r => r.status === 'error').length,
|
|
110
|
-
duration,
|
|
111
|
-
results
|
|
151
|
+
status: 'recovered',
|
|
152
|
+
count: results.length,
|
|
153
|
+
details: results
|
|
112
154
|
});
|
|
113
|
-
|
|
155
|
+
|
|
114
156
|
} catch (error) {
|
|
115
|
-
console.error('[
|
|
116
|
-
return res.status(500).json({
|
|
157
|
+
console.error('[Watchdog] Error:', error);
|
|
158
|
+
return res.status(500).json({ error: error.message });
|
|
117
159
|
}
|
|
118
160
|
}
|
|
119
161
|
|
|
162
|
+
// =============================================================================
|
|
163
|
+
// HELPER FUNCTIONS
|
|
164
|
+
// =============================================================================
|
|
165
|
+
|
|
120
166
|
/**
|
|
121
|
-
*
|
|
167
|
+
* Calculates all execution times for a schedule within a start/end window.
|
|
168
|
+
* Returns Array<Date>
|
|
122
169
|
*/
|
|
123
|
-
function
|
|
124
|
-
const
|
|
170
|
+
function getOccurrencesInWindow(schedule, start, end) {
|
|
171
|
+
const times = [];
|
|
172
|
+
const [h, m] = (schedule.time || '02:00').split(':').map(Number);
|
|
125
173
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if (Array.isArray(entry.dependencies) && entry.dependencies.length > 0) {
|
|
130
|
-
continue;
|
|
131
|
-
}
|
|
174
|
+
// Clone start date to iterate
|
|
175
|
+
let current = new Date(start);
|
|
176
|
+
current.setUTCHours(h, m, 0, 0);
|
|
132
177
|
|
|
133
|
-
|
|
178
|
+
// If current is before start (e.g. window starts at 10:00, schedule is 02:00), move to tomorrow
|
|
179
|
+
if (current < start) {
|
|
180
|
+
current.setDate(current.getDate() + 1);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
while (current <= end) {
|
|
184
|
+
let match = true;
|
|
185
|
+
|
|
186
|
+
// Weekly Check
|
|
187
|
+
if (schedule.frequency === 'weekly' && current.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
|
|
188
|
+
match = false;
|
|
189
|
+
}
|
|
134
190
|
|
|
135
|
-
//
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
191
|
+
// Monthly Check
|
|
192
|
+
if (schedule.frequency === 'monthly' && current.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
|
|
193
|
+
match = false;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (match) {
|
|
197
|
+
times.push(new Date(current));
|
|
142
198
|
}
|
|
199
|
+
|
|
200
|
+
// Advance 1 day
|
|
201
|
+
current.setDate(current.getDate() + 1);
|
|
143
202
|
}
|
|
144
|
-
|
|
203
|
+
|
|
204
|
+
return times;
|
|
145
205
|
}
|
|
146
206
|
|
|
147
207
|
/**
|
|
148
|
-
*
|
|
208
|
+
* Generates a short hash of the Scheduling Config + Pass.
|
|
209
|
+
* If this changes, we want a new Task ID to enforce the new schedule.
|
|
149
210
|
*/
|
|
150
|
-
function
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const target = new Date(now);
|
|
155
|
-
target.setUTCHours(h, m, 0, 0);
|
|
156
|
-
|
|
157
|
-
// Day of Week check (for Weekly frequency)
|
|
158
|
-
// 0 = Sunday, 1 = Monday, etc.
|
|
159
|
-
if (schedule.frequency === 'weekly' && target.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
|
|
160
|
-
return null;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// Day of Month check (for Monthly frequency)
|
|
164
|
-
if (schedule.frequency === 'monthly' && target.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
|
|
165
|
-
return null;
|
|
166
|
-
}
|
|
211
|
+
function generateConfigHash(entry) {
|
|
212
|
+
const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
|
|
213
|
+
return crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
|
|
214
|
+
}
|
|
167
215
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
// eventually "tomorrow" becomes "today".
|
|
172
|
-
|
|
173
|
-
return target;
|
|
216
|
+
function getQueuePath(entry) {
|
|
217
|
+
const { projectId, location, queueName } = config.cloudTasks;
|
|
218
|
+
return tasksClient.queuePath(projectId, location, queueName);
|
|
174
219
|
}
|
|
175
220
|
|
|
176
|
-
|
|
221
|
+
/**
|
|
222
|
+
* Dispatches Planned Root Tasks
|
|
223
|
+
* Uses deterministic naming for deduplication.
|
|
224
|
+
*/
|
|
225
|
+
async function dispatchPlannedTasks(tasks) {
|
|
177
226
|
const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
|
|
178
|
-
const {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
const tasks = computations.map(entry => limit(async () => {
|
|
227
|
+
const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
|
|
228
|
+
|
|
229
|
+
return Promise.all(tasks.map(t => limit(async () => {
|
|
182
230
|
try {
|
|
183
|
-
|
|
184
|
-
|
|
231
|
+
// Task Name: root-{name}-{date}-{configHash}
|
|
232
|
+
// If developer changes schedule -> hash changes -> new task created.
|
|
233
|
+
// If developer changes code but not schedule -> hash same -> existing task preserved.
|
|
234
|
+
const taskName = `${t.queuePath}/tasks/root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
|
|
185
235
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const taskNameSuffix = entry.isRecovery
|
|
192
|
-
? `recovery-${entry.recoveryId}-${Date.now()}`
|
|
193
|
-
: `${taskDate}`;
|
|
194
|
-
|
|
195
|
-
const taskPayload = {
|
|
196
|
-
computationName: entry.originalName,
|
|
197
|
-
targetDate: taskDate,
|
|
198
|
-
source: taskSource
|
|
236
|
+
const payload = {
|
|
237
|
+
computationName: t.computation,
|
|
238
|
+
targetDate: t.targetDate,
|
|
239
|
+
source: 'scheduled',
|
|
240
|
+
configHash: t.configHash // Sent to dispatcher for potential validation
|
|
199
241
|
};
|
|
200
242
|
|
|
201
243
|
const task = {
|
|
@@ -203,45 +245,68 @@ async function dispatchComputations(computations, defaultDate) {
|
|
|
203
245
|
httpMethod: 'POST',
|
|
204
246
|
url: dispatcherUrl,
|
|
205
247
|
headers: { 'Content-Type': 'application/json' },
|
|
206
|
-
body: Buffer.from(JSON.stringify(
|
|
248
|
+
body: Buffer.from(JSON.stringify(payload)).toString('base64'),
|
|
207
249
|
oidcToken: { serviceAccountEmail }
|
|
208
250
|
},
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
name: `${queuePath}/tasks/${entry.name}-${taskNameSuffix}`
|
|
251
|
+
scheduleTime: { seconds: t.runAtSeconds },
|
|
252
|
+
name: taskName
|
|
212
253
|
};
|
|
254
|
+
|
|
255
|
+
await tasksClient.createTask({ parent: t.queuePath, task });
|
|
256
|
+
return { computation: t.computation, date: t.targetDate, status: 'scheduled' };
|
|
257
|
+
|
|
258
|
+
} catch (e) {
|
|
259
|
+
if (e.code === 6 || e.code === 409) {
|
|
260
|
+
return { computation: t.computation, date: t.targetDate, status: 'exists' };
|
|
261
|
+
}
|
|
262
|
+
console.error(`[Planner] Failed to schedule ${t.computation}:`, e.message);
|
|
263
|
+
return { computation: t.computation, status: 'error', error: e.message };
|
|
264
|
+
}
|
|
265
|
+
})));
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Dispatches Recovery Tasks (Zombies)
|
|
270
|
+
* Always creates unique task names to ensure retry.
|
|
271
|
+
*/
|
|
272
|
+
async function dispatchRecoveryTasks(tasks) {
|
|
273
|
+
const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
|
|
274
|
+
const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
|
|
275
|
+
|
|
276
|
+
return Promise.all(tasks.map(t => limit(async () => {
|
|
277
|
+
try {
|
|
278
|
+
// Unique ID for every recovery attempt
|
|
279
|
+
const taskName = `${t.queuePath}/tasks/recovery-${toKebab(t.computation)}-${t.recoveryId}-${Date.now()}`;
|
|
213
280
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
status: 'dispatched',
|
|
219
|
-
scheduledFor: entry.runAt > 0 ? new Date(entry.runAt * 1000).toISOString() : 'now'
|
|
281
|
+
const payload = {
|
|
282
|
+
computationName: t.computation,
|
|
283
|
+
targetDate: t.targetDate,
|
|
284
|
+
source: 'zombie-recovery'
|
|
220
285
|
};
|
|
221
|
-
|
|
222
|
-
} catch (error) {
|
|
223
|
-
// ALREADY_EXISTS (Code 6) or ABORTED/CONFLICT (Code 409)
|
|
224
|
-
// This is expected and desired behavior for the rolling window.
|
|
225
|
-
if (error.code === 6 || error.code === 409) {
|
|
226
|
-
return { computation: entry.originalName, status: 'skipped', reason: 'duplicate' };
|
|
227
|
-
}
|
|
228
286
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
287
|
+
const task = {
|
|
288
|
+
httpRequest: {
|
|
289
|
+
httpMethod: 'POST',
|
|
290
|
+
url: dispatcherUrl,
|
|
291
|
+
headers: { 'Content-Type': 'application/json' },
|
|
292
|
+
body: Buffer.from(JSON.stringify(payload)).toString('base64'),
|
|
293
|
+
oidcToken: { serviceAccountEmail }
|
|
294
|
+
},
|
|
295
|
+
// Run Immediately (no scheduleTime)
|
|
296
|
+
name: taskName
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
await tasksClient.createTask({ parent: t.queuePath, task });
|
|
300
|
+
return { computation: t.computation, status: 'recovered' };
|
|
234
301
|
|
|
235
|
-
|
|
236
|
-
return { computation:
|
|
302
|
+
} catch (e) {
|
|
303
|
+
return { computation: t.computation, status: 'error', error: e.message };
|
|
237
304
|
}
|
|
238
|
-
}));
|
|
239
|
-
|
|
240
|
-
return Promise.all(tasks);
|
|
305
|
+
})));
|
|
241
306
|
}
|
|
242
307
|
|
|
243
|
-
function
|
|
244
|
-
return
|
|
308
|
+
function toKebab(str) {
|
|
309
|
+
return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
|
|
245
310
|
}
|
|
246
311
|
|
|
247
|
-
module.exports = {
|
|
312
|
+
module.exports = { planComputations, runWatchdog };
|