oqronkit 0.0.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -0
- package/dist/chunk-I6QFT3MR.mjs +1732 -0
- package/dist/chunk-PLN5A6LU.mjs +1447 -0
- package/dist/core/config/config-loader.d.ts +3 -0
- package/dist/core/config/config-loader.d.ts.map +1 -0
- package/dist/core/config/default-config.d.ts +4 -0
- package/dist/core/config/default-config.d.ts.map +1 -0
- package/dist/core/config/define-config.d.ts +3 -0
- package/dist/core/config/define-config.d.ts.map +1 -0
- package/dist/core/config/find-up.d.ts +2 -0
- package/dist/core/config/find-up.d.ts.map +1 -0
- package/dist/core/config/schema.d.ts +306 -0
- package/dist/core/config/schema.d.ts.map +1 -0
- package/dist/core/context/cron-context.d.ts +14 -0
- package/dist/core/context/cron-context.d.ts.map +1 -0
- package/dist/core/context/cron-context.interface.d.ts +14 -0
- package/dist/core/context/cron-context.interface.d.ts.map +1 -0
- package/dist/core/context/job-context.d.ts +22 -0
- package/dist/core/context/job-context.d.ts.map +1 -0
- package/dist/core/context/schedule-context.d.ts +31 -0
- package/dist/core/context/schedule-context.d.ts.map +1 -0
- package/dist/core/errors/base.error.d.ts +13 -0
- package/dist/core/errors/base.error.d.ts.map +1 -0
- package/dist/core/events/event-bus.d.ts +16 -0
- package/dist/core/events/event-bus.d.ts.map +1 -0
- package/dist/core/index.d.ts +24 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/logger/index.d.ts +30 -0
- package/dist/core/logger/index.d.ts.map +1 -0
- package/dist/core/registry.d.ts +13 -0
- package/dist/core/registry.d.ts.map +1 -0
- package/dist/core/types/config.types.d.ts +119 -0
- package/dist/core/types/config.types.d.ts.map +1 -0
- package/dist/core/types/cron.types.d.ts +53 -0
- package/dist/core/types/cron.types.d.ts.map +1 -0
- package/dist/core/types/db.types.d.ts +32 -0
- package/dist/core/types/db.types.d.ts.map +1 -0
- package/dist/core/types/index.d.ts +7 -0
- package/dist/core/types/index.d.ts.map +1 -0
- package/dist/core/types/lock.types.d.ts +7 -0
- package/dist/core/types/lock.types.d.ts.map +1 -0
- package/dist/core/types/module.types.d.ts +8 -0
- package/dist/core/types/module.types.d.ts.map +1 -0
- package/dist/core/types/scheduler.types.d.ts +62 -0
- package/dist/core/types/scheduler.types.d.ts.map +1 -0
- package/dist/cron-V0k1GcxJ.d.mts +102 -0
- package/dist/cron-V0k1GcxJ.d.ts +102 -0
- package/dist/cron.d.mts +2 -0
- package/dist/cron.d.ts +2 -0
- package/dist/cron.d.ts.map +1 -0
- package/dist/cron.js +215 -0
- package/dist/cron.mjs +1 -0
- package/dist/db/adapters/memory.adapter.d.ts +25 -0
- package/dist/db/adapters/memory.adapter.d.ts.map +1 -0
- package/dist/db/adapters/namespaced.adapter.d.ts +26 -0
- package/dist/db/adapters/namespaced.adapter.d.ts.map +1 -0
- package/dist/db/adapters/sqlite.adapter.d.ts +30 -0
- package/dist/db/adapters/sqlite.adapter.d.ts.map +1 -0
- package/dist/db/index.d.ts +4 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/index.d.mts +797 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2738 -0
- package/dist/index.mjs +747 -0
- package/dist/lock/adapters/db-lock.adapter.d.ts +17 -0
- package/dist/lock/adapters/db-lock.adapter.d.ts.map +1 -0
- package/dist/lock/adapters/memory-lock.adapter.d.ts +13 -0
- package/dist/lock/adapters/memory-lock.adapter.d.ts.map +1 -0
- package/dist/lock/adapters/namespaced-lock.adapter.d.ts +12 -0
- package/dist/lock/adapters/namespaced-lock.adapter.d.ts.map +1 -0
- package/dist/lock/heartbeat-worker.d.ts +16 -0
- package/dist/lock/heartbeat-worker.d.ts.map +1 -0
- package/dist/lock/index.d.ts +7 -0
- package/dist/lock/index.d.ts.map +1 -0
- package/dist/lock/leader-election.d.ts +16 -0
- package/dist/lock/leader-election.d.ts.map +1 -0
- package/dist/lock/stall-detector.d.ts +23 -0
- package/dist/lock/stall-detector.d.ts.map +1 -0
- package/dist/scheduler/cron-engine.d.ts +42 -0
- package/dist/scheduler/cron-engine.d.ts.map +1 -0
- package/dist/scheduler/define-cron.d.ts +36 -0
- package/dist/scheduler/define-cron.d.ts.map +1 -0
- package/dist/scheduler/define-schedule.d.ts +52 -0
- package/dist/scheduler/define-schedule.d.ts.map +1 -0
- package/dist/scheduler/expression-parser.d.ts +2 -0
- package/dist/scheduler/expression-parser.d.ts.map +1 -0
- package/dist/scheduler/index.d.ts +10 -0
- package/dist/scheduler/index.d.ts.map +1 -0
- package/dist/scheduler/missed-fire.handler.d.ts +8 -0
- package/dist/scheduler/missed-fire.handler.d.ts.map +1 -0
- package/dist/scheduler/registry-schedule.d.ts +6 -0
- package/dist/scheduler/registry-schedule.d.ts.map +1 -0
- package/dist/scheduler/registry.d.ts +6 -0
- package/dist/scheduler/registry.d.ts.map +1 -0
- package/dist/scheduler/schedule-engine.d.ts +36 -0
- package/dist/scheduler/schedule-engine.d.ts.map +1 -0
- package/dist/scheduler-HRR3UXGE.mjs +1 -0
- package/dist/scheduler.d.mts +248 -0
- package/dist/scheduler.d.ts +248 -0
- package/dist/scheduler.js +1461 -0
- package/dist/scheduler.mjs +1 -0
- package/dist/server/express.d.ts +9 -0
- package/dist/server/express.d.ts.map +1 -0
- package/dist/server/fastify.d.ts +9 -0
- package/dist/server/fastify.d.ts.map +1 -0
- package/dist/server/handlers.d.ts +15 -0
- package/dist/server/handlers.d.ts.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,1447 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
import 'url';
|
|
3
|
+
import 'find-up';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { EventEmitter } from 'eventemitter3';
|
|
6
|
+
import { redactionMiddleware, prettyTransport, consoleTransport, createLogger as createLogger$1 } from 'voltlog-io';
|
|
7
|
+
import 'better-sqlite3';
|
|
8
|
+
import _cronParser from 'cron-parser';
|
|
9
|
+
import { rrulestr, RRule } from 'rrule';
|
|
10
|
+
|
|
11
|
+
// src/scheduler/cron-engine.ts
|
|
12
|
+
var isIOqronAdapter = (val) => {
|
|
13
|
+
if (!val || typeof val !== "object") return false;
|
|
14
|
+
return typeof val.upsertSchedule === "function" && typeof val.getDueSchedules === "function";
|
|
15
|
+
};
|
|
16
|
+
var isILockAdapter = (val) => {
|
|
17
|
+
if (!val || typeof val !== "object") return false;
|
|
18
|
+
return typeof val.acquire === "function" && typeof val.renew === "function";
|
|
19
|
+
};
|
|
20
|
+
z.object({
|
|
21
|
+
project: z.string().optional(),
|
|
22
|
+
environment: z.string().default("development"),
|
|
23
|
+
// Infrastructure — Union of explicit DI or declarative config
|
|
24
|
+
db: z.union([
|
|
25
|
+
z.custom(
|
|
26
|
+
isIOqronAdapter,
|
|
27
|
+
"db must be an instance of IOqronAdapter"
|
|
28
|
+
),
|
|
29
|
+
z.object({
|
|
30
|
+
adapter: z.enum([
|
|
31
|
+
"sqlite",
|
|
32
|
+
"memory",
|
|
33
|
+
"postgres",
|
|
34
|
+
"mysql",
|
|
35
|
+
"mongodb",
|
|
36
|
+
"redis"
|
|
37
|
+
]),
|
|
38
|
+
url: z.string().optional(),
|
|
39
|
+
poolMin: z.number().default(2),
|
|
40
|
+
poolMax: z.number().default(10),
|
|
41
|
+
tablePrefix: z.string().default("chrono_"),
|
|
42
|
+
migrations: z.union([z.enum(["auto", "manual"]), z.literal(false)]).default("auto"),
|
|
43
|
+
ssl: z.boolean().default(false)
|
|
44
|
+
})
|
|
45
|
+
]).optional(),
|
|
46
|
+
lock: z.union([
|
|
47
|
+
z.custom(
|
|
48
|
+
isILockAdapter,
|
|
49
|
+
"lock must be an instance of ILockAdapter"
|
|
50
|
+
),
|
|
51
|
+
z.object({
|
|
52
|
+
adapter: z.enum(["db", "memory", "redis"]),
|
|
53
|
+
url: z.string().optional(),
|
|
54
|
+
ttl: z.number().default(3e4),
|
|
55
|
+
retryDelay: z.number().default(200),
|
|
56
|
+
retryCount: z.number().default(5)
|
|
57
|
+
})
|
|
58
|
+
]).optional(),
|
|
59
|
+
broker: z.any().optional(),
|
|
60
|
+
// Modules
|
|
61
|
+
modules: z.array(
|
|
62
|
+
z.enum([
|
|
63
|
+
"cron",
|
|
64
|
+
"scheduler",
|
|
65
|
+
"queue",
|
|
66
|
+
"workflow",
|
|
67
|
+
"batch",
|
|
68
|
+
"webhook",
|
|
69
|
+
"pipeline"
|
|
70
|
+
])
|
|
71
|
+
).default([]),
|
|
72
|
+
// Module-specific configs
|
|
73
|
+
cron: z.object({
|
|
74
|
+
enable: z.boolean().default(true),
|
|
75
|
+
timezone: z.string().optional(),
|
|
76
|
+
tickInterval: z.number().default(1e3),
|
|
77
|
+
missedFirePolicy: z.enum(["skip", "run-once", "run-all"]).default("run-once"),
|
|
78
|
+
maxConcurrentJobs: z.number().default(5),
|
|
79
|
+
leaderElection: z.boolean().default(true),
|
|
80
|
+
keepJobHistory: z.union([z.boolean(), z.number()]).default(true),
|
|
81
|
+
keepFailedJobHistory: z.union([z.boolean(), z.number()]).default(true)
|
|
82
|
+
}).default({}),
|
|
83
|
+
scheduler: z.object({
|
|
84
|
+
enable: z.boolean().default(true),
|
|
85
|
+
tickInterval: z.number().default(1e3),
|
|
86
|
+
keepJobHistory: z.union([z.boolean(), z.number()]).default(true),
|
|
87
|
+
keepFailedJobHistory: z.union([z.boolean(), z.number()]).default(true)
|
|
88
|
+
}).default({}),
|
|
89
|
+
// Auto-discovery directory
|
|
90
|
+
jobsDir: z.string().default("./src/jobs"),
|
|
91
|
+
// Global tags
|
|
92
|
+
tags: z.array(z.string()).default([]),
|
|
93
|
+
// Worker
|
|
94
|
+
worker: z.object({
|
|
95
|
+
concurrency: z.number().default(50),
|
|
96
|
+
gracefulShutdownMs: z.number().default(3e4)
|
|
97
|
+
}).default({ concurrency: 50, gracefulShutdownMs: 3e4 }),
|
|
98
|
+
// Logger (voltlog-io config or false to disable)
|
|
99
|
+
logger: z.union([
|
|
100
|
+
z.literal(false),
|
|
101
|
+
z.object({
|
|
102
|
+
enabled: z.boolean().default(true),
|
|
103
|
+
level: z.string().default("info"),
|
|
104
|
+
prettify: z.boolean().default(false),
|
|
105
|
+
showMetadata: z.boolean().default(true),
|
|
106
|
+
redact: z.array(z.string()).default([])
|
|
107
|
+
})
|
|
108
|
+
]).default({
|
|
109
|
+
enabled: true,
|
|
110
|
+
level: "info",
|
|
111
|
+
prettify: false,
|
|
112
|
+
showMetadata: true,
|
|
113
|
+
redact: []
|
|
114
|
+
}),
|
|
115
|
+
// Telemetry
|
|
116
|
+
telemetry: z.object({
|
|
117
|
+
prometheus: z.object({
|
|
118
|
+
enabled: z.boolean().default(false),
|
|
119
|
+
path: z.string().default("/metrics")
|
|
120
|
+
}).default({}),
|
|
121
|
+
opentelemetry: z.object({
|
|
122
|
+
enabled: z.boolean().default(false)
|
|
123
|
+
}).default({})
|
|
124
|
+
}).default({}),
|
|
125
|
+
// Shutdown
|
|
126
|
+
shutdown: z.object({
|
|
127
|
+
enabled: z.boolean().default(true),
|
|
128
|
+
timeout: z.number().default(3e4),
|
|
129
|
+
signals: z.array(z.string()).default(["SIGINT", "SIGTERM"])
|
|
130
|
+
}).default({})
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// src/core/context/job-context.ts
|
|
134
|
+
var JobContext = class {
|
|
135
|
+
id;
|
|
136
|
+
log;
|
|
137
|
+
signal;
|
|
138
|
+
environment;
|
|
139
|
+
project;
|
|
140
|
+
_progress = 0;
|
|
141
|
+
_onProgress;
|
|
142
|
+
constructor(opts) {
|
|
143
|
+
this.id = opts.id;
|
|
144
|
+
this.log = opts.logger;
|
|
145
|
+
this.signal = opts.signal;
|
|
146
|
+
this.environment = opts.environment;
|
|
147
|
+
this.project = opts.project;
|
|
148
|
+
this._onProgress = opts.onProgress;
|
|
149
|
+
this.progress = this.progress.bind(this);
|
|
150
|
+
}
|
|
151
|
+
progress(value, label) {
|
|
152
|
+
this._progress = Math.max(0, Math.min(100, value));
|
|
153
|
+
if (this._onProgress) {
|
|
154
|
+
this._onProgress(this._progress, label);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
getProgress() {
|
|
158
|
+
return this._progress;
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
// src/core/context/cron-context.ts
|
|
163
|
+
var CronContext = class extends JobContext {
|
|
164
|
+
firedAt;
|
|
165
|
+
scheduleName;
|
|
166
|
+
startedLocalAt;
|
|
167
|
+
constructor(opts) {
|
|
168
|
+
super(opts);
|
|
169
|
+
this.firedAt = opts.firedAt;
|
|
170
|
+
this.scheduleName = opts.scheduleName;
|
|
171
|
+
this.startedLocalAt = Date.now();
|
|
172
|
+
}
|
|
173
|
+
get duration() {
|
|
174
|
+
return Date.now() - this.startedLocalAt;
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
// src/core/context/schedule-context.ts
|
|
179
|
+
var ScheduleContext = class {
|
|
180
|
+
id;
|
|
181
|
+
name;
|
|
182
|
+
firedAt;
|
|
183
|
+
payload;
|
|
184
|
+
environment;
|
|
185
|
+
project;
|
|
186
|
+
logger;
|
|
187
|
+
signal;
|
|
188
|
+
startedLocalAt;
|
|
189
|
+
_onProgress;
|
|
190
|
+
constructor(opts) {
|
|
191
|
+
this.id = opts.id;
|
|
192
|
+
this.name = opts.scheduleName;
|
|
193
|
+
this.firedAt = opts.firedAt;
|
|
194
|
+
this.payload = opts.payload;
|
|
195
|
+
this.logger = opts.logger;
|
|
196
|
+
this.signal = opts.signal;
|
|
197
|
+
this.environment = opts.environment;
|
|
198
|
+
this.project = opts.project;
|
|
199
|
+
this.startedLocalAt = Date.now();
|
|
200
|
+
this._onProgress = opts.onProgress;
|
|
201
|
+
this.log = this.log.bind(this);
|
|
202
|
+
this.progress = this.progress.bind(this);
|
|
203
|
+
}
|
|
204
|
+
get aborted() {
|
|
205
|
+
return this.signal.aborted;
|
|
206
|
+
}
|
|
207
|
+
get duration() {
|
|
208
|
+
return Date.now() - this.startedLocalAt;
|
|
209
|
+
}
|
|
210
|
+
log(level, message, meta) {
|
|
211
|
+
if (this.logger && typeof this.logger[level] === "function") {
|
|
212
|
+
this.logger[level](message, meta);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
progress(percent, label) {
|
|
216
|
+
if (this._onProgress) {
|
|
217
|
+
this._onProgress(percent, label);
|
|
218
|
+
} else {
|
|
219
|
+
this.logger.debug("Progress updated", { percent, label, runId: this.id });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
var OqronEventBusClass = class _OqronEventBusClass extends EventEmitter {
|
|
224
|
+
static _instance;
|
|
225
|
+
static getInstance() {
|
|
226
|
+
if (!_OqronEventBusClass._instance) {
|
|
227
|
+
_OqronEventBusClass._instance = new _OqronEventBusClass();
|
|
228
|
+
}
|
|
229
|
+
return _OqronEventBusClass._instance;
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
OqronEventBusClass.getInstance();
|
|
233
|
+
var NOOP_LOGGER = {
|
|
234
|
+
trace() {
|
|
235
|
+
},
|
|
236
|
+
debug() {
|
|
237
|
+
},
|
|
238
|
+
info() {
|
|
239
|
+
},
|
|
240
|
+
warn() {
|
|
241
|
+
},
|
|
242
|
+
error() {
|
|
243
|
+
},
|
|
244
|
+
fatal() {
|
|
245
|
+
},
|
|
246
|
+
child() {
|
|
247
|
+
return NOOP_LOGGER;
|
|
248
|
+
},
|
|
249
|
+
addTransport() {
|
|
250
|
+
},
|
|
251
|
+
removeTransport() {
|
|
252
|
+
},
|
|
253
|
+
addMiddleware() {
|
|
254
|
+
},
|
|
255
|
+
removeMiddleware() {
|
|
256
|
+
},
|
|
257
|
+
setLevel() {
|
|
258
|
+
},
|
|
259
|
+
getLevel() {
|
|
260
|
+
return "SILENT";
|
|
261
|
+
},
|
|
262
|
+
isLevelEnabled() {
|
|
263
|
+
return false;
|
|
264
|
+
},
|
|
265
|
+
startTimer() {
|
|
266
|
+
return { done() {
|
|
267
|
+
}, elapsed: () => 0 };
|
|
268
|
+
},
|
|
269
|
+
async flush() {
|
|
270
|
+
},
|
|
271
|
+
async close() {
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
function createLogger(config, context) {
|
|
275
|
+
if (typeof config?.enabled !== "undefined" && config?.enabled === false)
|
|
276
|
+
return NOOP_LOGGER;
|
|
277
|
+
if (config?.logger) return config.logger;
|
|
278
|
+
const level = (config?.level ?? "INFO").toUpperCase();
|
|
279
|
+
const opts = {
|
|
280
|
+
level,
|
|
281
|
+
transports: [],
|
|
282
|
+
context
|
|
283
|
+
};
|
|
284
|
+
if (config?.redact?.length) {
|
|
285
|
+
const keys = config.redact;
|
|
286
|
+
opts.middleware = [
|
|
287
|
+
redactionMiddleware({
|
|
288
|
+
paths: keys,
|
|
289
|
+
deep: true
|
|
290
|
+
})
|
|
291
|
+
];
|
|
292
|
+
}
|
|
293
|
+
if (config?.prettify) {
|
|
294
|
+
opts.transports.push(prettyTransport());
|
|
295
|
+
} else {
|
|
296
|
+
opts.transports.push(consoleTransport());
|
|
297
|
+
}
|
|
298
|
+
return createLogger$1(opts);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// src/lock/heartbeat-worker.ts
|
|
302
|
+
var HeartbeatWorker = class {
|
|
303
|
+
constructor(lock, logger, key, ownerId, ttlMs = 3e4, heartbeatMs) {
|
|
304
|
+
this.lock = lock;
|
|
305
|
+
this.logger = logger;
|
|
306
|
+
this.key = key;
|
|
307
|
+
this.ownerId = ownerId;
|
|
308
|
+
this.ttlMs = ttlMs;
|
|
309
|
+
this.heartbeatMs = heartbeatMs;
|
|
310
|
+
}
|
|
311
|
+
heartbeatTimer;
|
|
312
|
+
_active = false;
|
|
313
|
+
async start() {
|
|
314
|
+
const acquired = await this.lock.acquire(
|
|
315
|
+
this.key,
|
|
316
|
+
this.ownerId,
|
|
317
|
+
this.ttlMs
|
|
318
|
+
);
|
|
319
|
+
if (!acquired) return false;
|
|
320
|
+
this._active = true;
|
|
321
|
+
const pingInterval = this.heartbeatMs ?? Math.floor(this.ttlMs / 3);
|
|
322
|
+
this.heartbeatTimer = setInterval(async () => {
|
|
323
|
+
if (!this._active) return;
|
|
324
|
+
try {
|
|
325
|
+
const renewed = await this.lock.renew(
|
|
326
|
+
this.key,
|
|
327
|
+
this.ownerId,
|
|
328
|
+
this.ttlMs
|
|
329
|
+
);
|
|
330
|
+
if (!renewed) {
|
|
331
|
+
this.logger.warn("Heartbeat renewal failed \u2014 lock lost", {
|
|
332
|
+
key: this.key,
|
|
333
|
+
ownerId: this.ownerId
|
|
334
|
+
});
|
|
335
|
+
this._active = false;
|
|
336
|
+
clearInterval(this.heartbeatTimer);
|
|
337
|
+
}
|
|
338
|
+
} catch (err) {
|
|
339
|
+
this.logger.error("Heartbeat renewal threw", {
|
|
340
|
+
key: this.key,
|
|
341
|
+
err: String(err)
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
}, pingInterval);
|
|
345
|
+
return true;
|
|
346
|
+
}
|
|
347
|
+
async stop() {
|
|
348
|
+
this._active = false;
|
|
349
|
+
if (this.heartbeatTimer) {
|
|
350
|
+
clearInterval(this.heartbeatTimer);
|
|
351
|
+
this.heartbeatTimer = void 0;
|
|
352
|
+
}
|
|
353
|
+
try {
|
|
354
|
+
await this.lock.release(this.key, this.ownerId);
|
|
355
|
+
} catch (err) {
|
|
356
|
+
this.logger.error("Failed to release lock cleanly", {
|
|
357
|
+
key: this.key,
|
|
358
|
+
err: String(err)
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
get isActive() {
|
|
363
|
+
return this._active;
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
// src/lock/leader-election.ts
|
|
368
|
+
var LeaderElection = class {
|
|
369
|
+
constructor(lock, logger, leaderKey, nodeId, ttlMs = 3e4) {
|
|
370
|
+
this.lock = lock;
|
|
371
|
+
this.logger = logger;
|
|
372
|
+
this.leaderKey = leaderKey;
|
|
373
|
+
this.nodeId = nodeId;
|
|
374
|
+
this.ttlMs = ttlMs;
|
|
375
|
+
}
|
|
376
|
+
electionTimer;
|
|
377
|
+
_isLeader = false;
|
|
378
|
+
async start() {
|
|
379
|
+
await this.campaign();
|
|
380
|
+
const interval = Math.floor(this.ttlMs / 3);
|
|
381
|
+
this.electionTimer = setInterval(() => void this.campaign(), interval);
|
|
382
|
+
}
|
|
383
|
+
async campaign() {
|
|
384
|
+
try {
|
|
385
|
+
if (this._isLeader) {
|
|
386
|
+
const ok = await this.lock.renew(
|
|
387
|
+
this.leaderKey,
|
|
388
|
+
this.nodeId,
|
|
389
|
+
this.ttlMs
|
|
390
|
+
);
|
|
391
|
+
if (!ok) {
|
|
392
|
+
this._isLeader = false;
|
|
393
|
+
this.logger.warn("Lost leadership", {
|
|
394
|
+
leaderKey: this.leaderKey,
|
|
395
|
+
nodeId: this.nodeId
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
} else {
|
|
399
|
+
const ok = await this.lock.acquire(
|
|
400
|
+
this.leaderKey,
|
|
401
|
+
this.nodeId,
|
|
402
|
+
this.ttlMs
|
|
403
|
+
);
|
|
404
|
+
if (ok) {
|
|
405
|
+
this._isLeader = true;
|
|
406
|
+
this.logger.info("Became leader", {
|
|
407
|
+
leaderKey: this.leaderKey,
|
|
408
|
+
nodeId: this.nodeId
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
} catch (err) {
|
|
413
|
+
this.logger.error("Leader election error", {
|
|
414
|
+
leaderKey: this.leaderKey,
|
|
415
|
+
err: String(err)
|
|
416
|
+
});
|
|
417
|
+
this._isLeader = false;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
get isLeader() {
|
|
421
|
+
return this._isLeader;
|
|
422
|
+
}
|
|
423
|
+
async stop() {
|
|
424
|
+
if (this.electionTimer) {
|
|
425
|
+
clearInterval(this.electionTimer);
|
|
426
|
+
this.electionTimer = void 0;
|
|
427
|
+
}
|
|
428
|
+
if (this._isLeader) {
|
|
429
|
+
try {
|
|
430
|
+
await this.lock.release(this.leaderKey, this.nodeId);
|
|
431
|
+
} catch {
|
|
432
|
+
}
|
|
433
|
+
this._isLeader = false;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
// src/lock/stall-detector.ts
|
|
439
|
+
var StallDetector = class {
|
|
440
|
+
constructor(lock, logger, checkIntervalMs = 15e3) {
|
|
441
|
+
this.lock = lock;
|
|
442
|
+
this.logger = logger;
|
|
443
|
+
this.checkIntervalMs = checkIntervalMs;
|
|
444
|
+
}
|
|
445
|
+
timer;
|
|
446
|
+
/**
|
|
447
|
+
* Start the stall detection loop.
|
|
448
|
+
* @param getActiveJobs - Returns a list of { key, ownerId } for all currently tracked jobs.
|
|
449
|
+
* @param onStalled - Called when a job is detected as stalled (lock lost).
|
|
450
|
+
*/
|
|
451
|
+
start(getActiveJobs, onStalled) {
|
|
452
|
+
this.timer = setInterval(async () => {
|
|
453
|
+
try {
|
|
454
|
+
const jobs = getActiveJobs();
|
|
455
|
+
for (const job of jobs) {
|
|
456
|
+
const owned = await this.lock.isOwner(job.key, job.ownerId);
|
|
457
|
+
if (!owned) {
|
|
458
|
+
this.logger.warn("Stalled job detected \u2014 lock lost", {
|
|
459
|
+
key: job.key,
|
|
460
|
+
ownerId: job.ownerId
|
|
461
|
+
});
|
|
462
|
+
onStalled(job.key);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
} catch (err) {
|
|
466
|
+
this.logger.error("StallDetector tick error", { err: String(err) });
|
|
467
|
+
}
|
|
468
|
+
}, this.checkIntervalMs);
|
|
469
|
+
}
|
|
470
|
+
stop() {
|
|
471
|
+
if (this.timer) {
|
|
472
|
+
clearInterval(this.timer);
|
|
473
|
+
this.timer = void 0;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
var cronParser = _cronParser.default ?? _cronParser;
|
|
478
|
+
function getNextRunDate(expression, timezone, from = /* @__PURE__ */ new Date()) {
|
|
479
|
+
const opts = { currentDate: from };
|
|
480
|
+
if (timezone) opts.tz = timezone;
|
|
481
|
+
return cronParser.parseExpression(expression, opts).next().toDate();
|
|
482
|
+
}
|
|
483
|
+
var cronParser2 = _cronParser.default ?? _cronParser;
|
|
484
|
+
var MissedFireHandler = class {
|
|
485
|
+
constructor(logger, _db) {
|
|
486
|
+
this.logger = logger;
|
|
487
|
+
this._db = _db;
|
|
488
|
+
}
|
|
489
|
+
async checkMissed(def, lastRunAt, now) {
|
|
490
|
+
if (!lastRunAt) return false;
|
|
491
|
+
try {
|
|
492
|
+
let missed = false;
|
|
493
|
+
if (def.expression) {
|
|
494
|
+
const opts = { currentDate: now, tz: def.timezone };
|
|
495
|
+
const prevRun = cronParser2.parseExpression(def.expression, opts).prev().toDate();
|
|
496
|
+
missed = prevRun > lastRunAt;
|
|
497
|
+
} else if (def.intervalMs) {
|
|
498
|
+
const elapsed = now.getTime() - lastRunAt.getTime();
|
|
499
|
+
missed = elapsed > def.intervalMs;
|
|
500
|
+
}
|
|
501
|
+
if (missed) {
|
|
502
|
+
this.logger.warn("Missed execution detected", {
|
|
503
|
+
name: def.name,
|
|
504
|
+
policy: def.missedFire
|
|
505
|
+
});
|
|
506
|
+
if (def.hooks?.onMissedFire) {
|
|
507
|
+
try {
|
|
508
|
+
const ctx = {
|
|
509
|
+
id: randomUUID(),
|
|
510
|
+
log: this.logger.child({
|
|
511
|
+
schedule: def.name,
|
|
512
|
+
scope: "missed-fire"
|
|
513
|
+
}),
|
|
514
|
+
logger: this.logger.child({
|
|
515
|
+
schedule: def.name,
|
|
516
|
+
scope: "missed-fire"
|
|
517
|
+
}),
|
|
518
|
+
signal: new AbortController().signal,
|
|
519
|
+
firedAt: now,
|
|
520
|
+
scheduleName: def.name,
|
|
521
|
+
progress: () => {
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
await def.hooks.onMissedFire(ctx, lastRunAt);
|
|
525
|
+
} catch (err) {
|
|
526
|
+
this.logger.error("onMissedFire hook threw", {
|
|
527
|
+
name: def.name,
|
|
528
|
+
err: String(err)
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
if (def.missedFire === "run-once" || def.missedFire === "run-all") {
|
|
533
|
+
return true;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
} catch {
|
|
537
|
+
}
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
|
|
542
|
+
// src/scheduler/cron-engine.ts
|
|
543
|
+
var SchedulerModule = class {
|
|
544
|
+
constructor(schedules, db, lock, logger, environment, project, config) {
|
|
545
|
+
this.schedules = schedules;
|
|
546
|
+
this.db = db;
|
|
547
|
+
this.lock = lock;
|
|
548
|
+
this.environment = environment;
|
|
549
|
+
this.project = project;
|
|
550
|
+
this.config = config;
|
|
551
|
+
this.nodeId = randomUUID();
|
|
552
|
+
this.logger = logger ?? createLogger({ level: "info" }, { module: "scheduler" });
|
|
553
|
+
this.stallDetector = new StallDetector(this.lock, this.logger, 15e3);
|
|
554
|
+
this.missedFireHandler = new MissedFireHandler(this.logger, this.db);
|
|
555
|
+
}
|
|
556
|
+
name = "cron";
|
|
557
|
+
enabled = true;
|
|
558
|
+
nodeId;
|
|
559
|
+
logger;
|
|
560
|
+
leader;
|
|
561
|
+
stallDetector;
|
|
562
|
+
missedFireHandler;
|
|
563
|
+
tickTimer;
|
|
564
|
+
activeJobs = /* @__PURE__ */ new Map();
|
|
565
|
+
_hasRunLeaderInit = false;
|
|
566
|
+
// ── Lifecycle ───────────────────────────────────────────────────────────────
|
|
567
|
+
async init() {
|
|
568
|
+
this.logger.info("Initializing scheduler", {
|
|
569
|
+
nodeId: this.nodeId,
|
|
570
|
+
count: this.schedules.length
|
|
571
|
+
});
|
|
572
|
+
for (const def of this.schedules) {
|
|
573
|
+
await this.db.upsertSchedule(def);
|
|
574
|
+
this.logger.debug("Registered schedule", {
|
|
575
|
+
name: def.name,
|
|
576
|
+
expression: def.expression,
|
|
577
|
+
intervalMs: def.intervalMs
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
const existing = await this.db.getSchedules();
|
|
581
|
+
const now = /* @__PURE__ */ new Date();
|
|
582
|
+
for (const record of existing) {
|
|
583
|
+
if (record.nextRunAt !== null) continue;
|
|
584
|
+
const def = this.schedules.find((s) => s.name === record.name);
|
|
585
|
+
if (!def) continue;
|
|
586
|
+
const nextRun = this.computeNextRun(def, now);
|
|
587
|
+
if (nextRun) {
|
|
588
|
+
await this.db.updateNextRun(def.name, nextRun);
|
|
589
|
+
this.logger.debug("Seeded nextRunAt", {
|
|
590
|
+
name: def.name,
|
|
591
|
+
nextRunAt: nextRun.toISOString()
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
async start() {
|
|
597
|
+
if (this.config?.leaderElection !== false) {
|
|
598
|
+
this.leader = new LeaderElection(
|
|
599
|
+
this.lock,
|
|
600
|
+
this.logger,
|
|
601
|
+
"oqron:scheduler:leader",
|
|
602
|
+
this.nodeId,
|
|
603
|
+
3e4
|
|
604
|
+
);
|
|
605
|
+
await this.leader.start();
|
|
606
|
+
}
|
|
607
|
+
const interval = this.config?.tickInterval ?? 1e3;
|
|
608
|
+
this.tickTimer = setInterval(() => {
|
|
609
|
+
void this.tick();
|
|
610
|
+
}, interval);
|
|
611
|
+
this.logger.info("Scheduler started", { nodeId: this.nodeId, interval });
|
|
612
|
+
}
|
|
613
|
+
async stop() {
|
|
614
|
+
if (this.tickTimer) clearInterval(this.tickTimer);
|
|
615
|
+
if (this.leader) await this.leader.stop();
|
|
616
|
+
this.stallDetector.stop();
|
|
617
|
+
for (const job of this.activeJobs.values()) {
|
|
618
|
+
if (job.abort) job.abort.abort();
|
|
619
|
+
if (job.worker) {
|
|
620
|
+
await job.worker.stop();
|
|
621
|
+
} else {
|
|
622
|
+
await this.lock.release(job.lockKey, this.nodeId).catch(() => {
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
this.activeJobs.clear();
|
|
627
|
+
this.logger.info("Scheduler stopped");
|
|
628
|
+
}
|
|
629
|
+
// ── Core scheduling helpers ─────────────────────────────────────────────────
|
|
630
|
+
/**
|
|
631
|
+
* Compute the next fire time for a definition.
|
|
632
|
+
* Returns null if computation fails (invalid expression, etc).
|
|
633
|
+
*/
|
|
634
|
+
computeNextRun(def, from) {
|
|
635
|
+
if (def.expression) {
|
|
636
|
+
try {
|
|
637
|
+
const timezone = def.timezone ?? this.config?.timezone;
|
|
638
|
+
return getNextRunDate(def.expression, timezone, from);
|
|
639
|
+
} catch (err) {
|
|
640
|
+
this.logger.error("Failed to compute next run from expression", {
|
|
641
|
+
name: def.name,
|
|
642
|
+
expression: def.expression,
|
|
643
|
+
err: String(err)
|
|
644
|
+
});
|
|
645
|
+
return null;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
if (def.intervalMs) {
|
|
649
|
+
return new Date(from.getTime() + def.intervalMs);
|
|
650
|
+
}
|
|
651
|
+
this.logger.error("Schedule has neither expression nor intervalMs", {
|
|
652
|
+
name: def.name
|
|
653
|
+
});
|
|
654
|
+
return null;
|
|
655
|
+
}
|
|
656
|
+
// ── Leader init (missed-fire recovery + stall detector) ─────────────────────
|
|
657
|
+
async handleLeaderInit() {
|
|
658
|
+
this.logger.info("Performing leader initialization...");
|
|
659
|
+
const knownSchedules = await this.db.getSchedules();
|
|
660
|
+
const now = /* @__PURE__ */ new Date();
|
|
661
|
+
for (const record of knownSchedules) {
|
|
662
|
+
const def = this.schedules.find((s) => s.name === record.name);
|
|
663
|
+
if (!def) continue;
|
|
664
|
+
const missed = await this.missedFireHandler.checkMissed(
|
|
665
|
+
def,
|
|
666
|
+
record.lastRunAt,
|
|
667
|
+
now
|
|
668
|
+
);
|
|
669
|
+
if (missed) {
|
|
670
|
+
this.logger.info("Triggering recovery run for missed schedule", {
|
|
671
|
+
name: def.name
|
|
672
|
+
});
|
|
673
|
+
void this.fire(def);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
this.stallDetector.start(
|
|
677
|
+
() => {
|
|
678
|
+
return Array.from(this.activeJobs.values()).filter((job) => job.worker !== void 0).map((job) => ({
|
|
679
|
+
key: job.lockKey,
|
|
680
|
+
ownerId: this.nodeId
|
|
681
|
+
}));
|
|
682
|
+
},
|
|
683
|
+
(key) => {
|
|
684
|
+
this.logger.warn("Local stall detected", { key });
|
|
685
|
+
}
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
async detectClusterStalls() {
|
|
689
|
+
try {
|
|
690
|
+
const activeDbJobs = await this.db.getActiveJobs();
|
|
691
|
+
for (const job of activeDbJobs) {
|
|
692
|
+
if (!job.scheduleId) continue;
|
|
693
|
+
const def = this.schedules.find((s) => s.name === job.scheduleId);
|
|
694
|
+
if (!def?.guaranteedWorker) continue;
|
|
695
|
+
const ageMs = Date.now() - job.startedAt.getTime();
|
|
696
|
+
const ttl = def.lockTtlMs ?? 5e4;
|
|
697
|
+
if (ageMs > ttl + 1e4) {
|
|
698
|
+
this.logger.warn("Cluster stall detected", { runId: job.id });
|
|
699
|
+
await this.db.recordExecution({
|
|
700
|
+
id: job.id,
|
|
701
|
+
scheduleId: job.scheduleId,
|
|
702
|
+
status: "failed",
|
|
703
|
+
error: "Stall detected (lock assumed expired)",
|
|
704
|
+
startedAt: job.startedAt,
|
|
705
|
+
completedAt: /* @__PURE__ */ new Date()
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
} catch (err) {
|
|
710
|
+
this.logger.error("Failed to detect cluster stalls", {
|
|
711
|
+
err: String(err)
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
// ── Tick loop ───────────────────────────────────────────────────────────────
|
|
716
|
+
async tick() {
|
|
717
|
+
if (this.leader && !this.leader.isLeader) return;
|
|
718
|
+
if (!this._hasRunLeaderInit) {
|
|
719
|
+
this._hasRunLeaderInit = true;
|
|
720
|
+
await this.handleLeaderInit();
|
|
721
|
+
}
|
|
722
|
+
try {
|
|
723
|
+
if (Math.random() < 0.1) {
|
|
724
|
+
void this.detectClusterStalls();
|
|
725
|
+
}
|
|
726
|
+
const now = /* @__PURE__ */ new Date();
|
|
727
|
+
const due = await this.db.getDueSchedules(now, 50);
|
|
728
|
+
for (const { name } of due) {
|
|
729
|
+
const def = this.schedules.find((s) => s.name === name);
|
|
730
|
+
if (!def) continue;
|
|
731
|
+
const nextRun = this.computeNextRun(def, now);
|
|
732
|
+
if (!nextRun) {
|
|
733
|
+
this.logger.error(
|
|
734
|
+
"Cannot compute next run \u2014 suspending cron to prevent runaway loop",
|
|
735
|
+
{ name: def.name }
|
|
736
|
+
);
|
|
737
|
+
await this.db.updateNextRun(def.name, null).catch(() => {
|
|
738
|
+
});
|
|
739
|
+
continue;
|
|
740
|
+
}
|
|
741
|
+
await this.db.updateNextRun(def.name, nextRun);
|
|
742
|
+
void this.fire(def);
|
|
743
|
+
}
|
|
744
|
+
} catch (err) {
|
|
745
|
+
this.logger.error("Tick error", { err: String(err) });
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
// ── Fire handler ────────────────────────────────────────────────────────────
|
|
749
|
+
async fire(def) {
|
|
750
|
+
const isOverlapSkip = def.overlap === "skip" || def.overlap === false;
|
|
751
|
+
if (isOverlapSkip) {
|
|
752
|
+
for (const job of this.activeJobs.values()) {
|
|
753
|
+
if (job.lockKey === `oqron:run:${def.name}`) {
|
|
754
|
+
this.logger.debug("Skipping overlapping run", { name: def.name });
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
const runId = randomUUID();
|
|
760
|
+
const lockKey = isOverlapSkip ? `oqron:run:${def.name}` : `oqron:run:${def.name}:${runId}`;
|
|
761
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
762
|
+
let worker;
|
|
763
|
+
let acquired = false;
|
|
764
|
+
if (def.guaranteedWorker) {
|
|
765
|
+
worker = new HeartbeatWorker(
|
|
766
|
+
this.lock,
|
|
767
|
+
this.logger,
|
|
768
|
+
lockKey,
|
|
769
|
+
this.nodeId,
|
|
770
|
+
def.lockTtlMs ?? 3e4,
|
|
771
|
+
def.heartbeatMs ?? 1e4
|
|
772
|
+
);
|
|
773
|
+
acquired = await worker.start();
|
|
774
|
+
} else {
|
|
775
|
+
acquired = await this.lock.acquire(
|
|
776
|
+
lockKey,
|
|
777
|
+
this.nodeId,
|
|
778
|
+
def.lockTtlMs ?? 3e4
|
|
779
|
+
);
|
|
780
|
+
}
|
|
781
|
+
if (!acquired) return;
|
|
782
|
+
const abort = new AbortController();
|
|
783
|
+
this.activeJobs.set(runId, { runId, lockKey, worker, abort });
|
|
784
|
+
await this.db.recordExecution({
|
|
785
|
+
id: runId,
|
|
786
|
+
scheduleId: def.name,
|
|
787
|
+
status: "running",
|
|
788
|
+
startedAt
|
|
789
|
+
});
|
|
790
|
+
void Promise.resolve().then(async () => {
|
|
791
|
+
const ctx = new CronContext({
|
|
792
|
+
id: runId,
|
|
793
|
+
logger: this.logger.child({ schedule: def.name }),
|
|
794
|
+
signal: abort.signal,
|
|
795
|
+
firedAt: startedAt,
|
|
796
|
+
scheduleName: def.name,
|
|
797
|
+
environment: this.environment,
|
|
798
|
+
project: this.project,
|
|
799
|
+
onProgress: (percent, label) => {
|
|
800
|
+
this.db.updateJobProgress(runId, percent, label).catch(
|
|
801
|
+
(err) => this.logger.error("Failed to update progress", { runId, err })
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
});
|
|
805
|
+
let status = "completed";
|
|
806
|
+
let error;
|
|
807
|
+
let timeoutHandle;
|
|
808
|
+
let finalResult;
|
|
809
|
+
let attempts = 1;
|
|
810
|
+
const maxAttempts = (def.retries?.max ?? 0) + 1;
|
|
811
|
+
while (attempts <= maxAttempts) {
|
|
812
|
+
try {
|
|
813
|
+
if (def.hooks?.beforeRun) await def.hooks.beforeRun(ctx);
|
|
814
|
+
if (def.timeout) {
|
|
815
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
816
|
+
timeoutHandle = setTimeout(() => {
|
|
817
|
+
abort.abort();
|
|
818
|
+
reject(new Error(`Handler timed out after ${def.timeout}ms`));
|
|
819
|
+
}, def.timeout);
|
|
820
|
+
});
|
|
821
|
+
finalResult = await Promise.race([
|
|
822
|
+
def.handler(ctx),
|
|
823
|
+
timeoutPromise
|
|
824
|
+
]);
|
|
825
|
+
} else {
|
|
826
|
+
finalResult = await def.handler(ctx);
|
|
827
|
+
}
|
|
828
|
+
if (def.hooks?.afterRun) {
|
|
829
|
+
await def.hooks.afterRun(ctx, finalResult);
|
|
830
|
+
}
|
|
831
|
+
status = "completed";
|
|
832
|
+
error = void 0;
|
|
833
|
+
break;
|
|
834
|
+
} catch (err) {
|
|
835
|
+
error = err instanceof Error ? err.message : String(err);
|
|
836
|
+
status = "failed";
|
|
837
|
+
if (def.hooks?.onError && err instanceof Error) {
|
|
838
|
+
try {
|
|
839
|
+
await Promise.resolve(def.hooks.onError(ctx, err));
|
|
840
|
+
} catch (e) {
|
|
841
|
+
this.logger.error("onError hook threw", { err: String(e) });
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
if (attempts < maxAttempts) {
|
|
845
|
+
this.logger.warn("Job handler threw, retrying...", {
|
|
846
|
+
name: def.name,
|
|
847
|
+
runId,
|
|
848
|
+
attempt: attempts,
|
|
849
|
+
error
|
|
850
|
+
});
|
|
851
|
+
const baseDelay = def.retries?.baseDelay ?? 2e3;
|
|
852
|
+
const delay = def.retries?.strategy === "exponential" ? baseDelay * 2 ** (attempts - 1) : baseDelay;
|
|
853
|
+
attempts++;
|
|
854
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
855
|
+
} else {
|
|
856
|
+
this.logger.error("Job handler failed completely", {
|
|
857
|
+
name: def.name,
|
|
858
|
+
runId,
|
|
859
|
+
attempts,
|
|
860
|
+
error
|
|
861
|
+
});
|
|
862
|
+
break;
|
|
863
|
+
}
|
|
864
|
+
} finally {
|
|
865
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
const completedAt = /* @__PURE__ */ new Date();
|
|
869
|
+
await this.db.recordExecution({
|
|
870
|
+
id: runId,
|
|
871
|
+
scheduleId: def.name,
|
|
872
|
+
status,
|
|
873
|
+
startedAt,
|
|
874
|
+
completedAt,
|
|
875
|
+
error,
|
|
876
|
+
result: finalResult !== void 0 ? JSON.stringify(finalResult) : void 0,
|
|
877
|
+
attempts,
|
|
878
|
+
durationMs: completedAt.getTime() - startedAt.getTime(),
|
|
879
|
+
...status === "completed" ? { progressPercent: 100, progressLabel: "Completed" } : {}
|
|
880
|
+
});
|
|
881
|
+
if (worker) {
|
|
882
|
+
await worker.stop();
|
|
883
|
+
} else {
|
|
884
|
+
await this.lock.release(lockKey, this.nodeId).catch(() => {
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
this.activeJobs.delete(runId);
|
|
888
|
+
if (def.intervalMs) {
|
|
889
|
+
const nextRun = this.computeNextRun(def, /* @__PURE__ */ new Date());
|
|
890
|
+
if (nextRun) {
|
|
891
|
+
await this.db.updateNextRun(def.name, nextRun).catch(() => {
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
const keepJobHistory = def.keepHistory ?? this.config?.keepJobHistory ?? true;
|
|
896
|
+
const keepFailedHistory = def.keepFailedHistory ?? this.config?.keepFailedJobHistory ?? true;
|
|
897
|
+
if (keepJobHistory !== true || keepFailedHistory !== true) {
|
|
898
|
+
this.db.pruneHistoryForSchedule(def.name, keepJobHistory, keepFailedHistory).catch(
|
|
899
|
+
(err) => this.logger.debug("Failed to prune history", {
|
|
900
|
+
err,
|
|
901
|
+
name: def.name
|
|
902
|
+
})
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
this.logger.info("Job finished", {
|
|
906
|
+
name: def.name,
|
|
907
|
+
runId,
|
|
908
|
+
status,
|
|
909
|
+
attempts
|
|
910
|
+
});
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
};
|
|
914
|
+
|
|
915
|
+
// src/scheduler/registry.ts
|
|
916
|
+
var GLOBAL_KEY = "__oqronkit_pending_crons__";
|
|
917
|
+
function _getPending() {
|
|
918
|
+
if (!globalThis[GLOBAL_KEY]) {
|
|
919
|
+
globalThis[GLOBAL_KEY] = [];
|
|
920
|
+
}
|
|
921
|
+
return globalThis[GLOBAL_KEY];
|
|
922
|
+
}
|
|
923
|
+
function _registerCron(def) {
|
|
924
|
+
_getPending().push(def);
|
|
925
|
+
}
|
|
926
|
+
function _drainPending() {
|
|
927
|
+
const pending = _getPending();
|
|
928
|
+
return pending.splice(0);
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
// src/scheduler/define-cron.ts
|
|
932
|
+
var cronParser3 = _cronParser.default ?? _cronParser;
|
|
933
|
+
function everyToIntervalMs(every) {
|
|
934
|
+
let ms = 0;
|
|
935
|
+
if (every.seconds) ms += every.seconds * 1e3;
|
|
936
|
+
if (every.minutes) ms += every.minutes * 6e4;
|
|
937
|
+
if (every.hours) ms += every.hours * 36e5;
|
|
938
|
+
if (ms <= 0)
|
|
939
|
+
throw new Error(
|
|
940
|
+
"[OqronKit] `every` config must resolve to a positive interval"
|
|
941
|
+
);
|
|
942
|
+
return ms;
|
|
943
|
+
}
|
|
944
|
+
var cron = (options) => {
|
|
945
|
+
let expression;
|
|
946
|
+
let intervalMs;
|
|
947
|
+
if ("expression" in options && options.expression) {
|
|
948
|
+
try {
|
|
949
|
+
cronParser3.parseExpression(options.expression, { tz: options.timezone });
|
|
950
|
+
} catch {
|
|
951
|
+
throw new Error(
|
|
952
|
+
`[OqronKit] Invalid cron expression for "${options.name}": "${options.expression}"`
|
|
953
|
+
);
|
|
954
|
+
}
|
|
955
|
+
expression = options.expression;
|
|
956
|
+
} else if ("every" in options && options.every) {
|
|
957
|
+
intervalMs = everyToIntervalMs(options.every);
|
|
958
|
+
} else {
|
|
959
|
+
throw new Error(
|
|
960
|
+
`[OqronKit] Cron "${options.name}" must specify either "expression" or "every"`
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
const def = {
|
|
964
|
+
name: options.name,
|
|
965
|
+
expression,
|
|
966
|
+
intervalMs,
|
|
967
|
+
timezone: options.timezone,
|
|
968
|
+
missedFire: options.missedFire ?? "skip",
|
|
969
|
+
overlap: options.overlap ?? "skip",
|
|
970
|
+
guaranteedWorker: options.guaranteedWorker ?? false,
|
|
971
|
+
heartbeatMs: options.heartbeatMs,
|
|
972
|
+
lockTtlMs: options.lockTtlMs,
|
|
973
|
+
timeout: options.timeout,
|
|
974
|
+
tags: options.tags ?? [],
|
|
975
|
+
handler: options.handler,
|
|
976
|
+
hooks: options.hooks,
|
|
977
|
+
retries: options.retries
|
|
978
|
+
};
|
|
979
|
+
_registerCron(def);
|
|
980
|
+
return def;
|
|
981
|
+
};
|
|
982
|
+
|
|
983
|
+
// src/scheduler/registry-schedule.ts
|
|
984
|
+
var GLOBAL_KEY2 = "__oqronkit_pending_schedules__";
|
|
985
|
+
function _getPending2() {
|
|
986
|
+
if (!globalThis[GLOBAL_KEY2]) {
|
|
987
|
+
globalThis[GLOBAL_KEY2] = [];
|
|
988
|
+
}
|
|
989
|
+
return globalThis[GLOBAL_KEY2];
|
|
990
|
+
}
|
|
991
|
+
function _registerSchedule(def) {
|
|
992
|
+
_getPending2().push(def);
|
|
993
|
+
}
|
|
994
|
+
function _drainPendingSchedules() {
|
|
995
|
+
const pending = _getPending2();
|
|
996
|
+
return pending.splice(0);
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
// src/scheduler/define-schedule.ts
|
|
1000
|
+
var engineRef = { current: null };
|
|
1001
|
+
function _attachScheduleEngine(engine) {
|
|
1002
|
+
engineRef.current = engine;
|
|
1003
|
+
}
|
|
1004
|
+
var schedule = (options) => {
|
|
1005
|
+
const def = {
|
|
1006
|
+
name: options.name,
|
|
1007
|
+
runAt: options.runAt,
|
|
1008
|
+
runAfter: options.runAfter,
|
|
1009
|
+
recurring: options.recurring,
|
|
1010
|
+
rrule: options.rrule,
|
|
1011
|
+
every: options.every,
|
|
1012
|
+
timezone: options.timezone,
|
|
1013
|
+
missedFire: options.missedFire ?? "skip",
|
|
1014
|
+
overlap: options.overlap ?? "skip",
|
|
1015
|
+
guaranteedWorker: options.guaranteedWorker ?? false,
|
|
1016
|
+
heartbeatMs: options.heartbeatMs,
|
|
1017
|
+
lockTtlMs: options.lockTtlMs,
|
|
1018
|
+
timeout: options.timeout,
|
|
1019
|
+
tags: options.tags ?? [],
|
|
1020
|
+
condition: options.condition,
|
|
1021
|
+
handler: options.handler,
|
|
1022
|
+
hooks: options.hooks,
|
|
1023
|
+
payload: options.payload,
|
|
1024
|
+
retries: options.retries
|
|
1025
|
+
};
|
|
1026
|
+
_registerSchedule(def);
|
|
1027
|
+
return {
|
|
1028
|
+
...def,
|
|
1029
|
+
trigger: async (opts) => {
|
|
1030
|
+
if (!engineRef.current) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
`[OqronKit] Cannot trigger "${options.name}" \u2014 ScheduleEngine is not running.`
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1035
|
+
const dynamicDef = { ...def };
|
|
1036
|
+
if (opts) {
|
|
1037
|
+
if (opts.nameSuffix) dynamicDef.name = `${def.name}:${opts.nameSuffix}`;
|
|
1038
|
+
if (opts.runAt) dynamicDef.runAt = opts.runAt;
|
|
1039
|
+
if (opts.runAfter) dynamicDef.runAfter = opts.runAfter;
|
|
1040
|
+
if (opts.recurring) dynamicDef.recurring = opts.recurring;
|
|
1041
|
+
if (opts.rrule) dynamicDef.rrule = opts.rrule;
|
|
1042
|
+
if (opts.every) dynamicDef.every = opts.every;
|
|
1043
|
+
if (opts.payload) dynamicDef.payload = opts.payload;
|
|
1044
|
+
}
|
|
1045
|
+
if (!opts?.runAt && !opts?.runAfter && !opts?.recurring && !opts?.rrule && !opts?.every) {
|
|
1046
|
+
dynamicDef.runAt = /* @__PURE__ */ new Date();
|
|
1047
|
+
}
|
|
1048
|
+
await engineRef.current.registerDynamic(dynamicDef);
|
|
1049
|
+
},
|
|
1050
|
+
schedule: async (opts) => {
|
|
1051
|
+
if (!engineRef.current) {
|
|
1052
|
+
throw new Error(
|
|
1053
|
+
`[OqronKit] Cannot schedule "${options.name}" \u2014 ScheduleEngine is not running.`
|
|
1054
|
+
);
|
|
1055
|
+
}
|
|
1056
|
+
const dynamicDef = { ...def };
|
|
1057
|
+
if (opts) {
|
|
1058
|
+
if (opts.nameSuffix) dynamicDef.name = `${def.name}:${opts.nameSuffix}`;
|
|
1059
|
+
if (opts.runAt) dynamicDef.runAt = opts.runAt;
|
|
1060
|
+
if (opts.runAfter) dynamicDef.runAfter = opts.runAfter;
|
|
1061
|
+
if (opts.recurring) dynamicDef.recurring = opts.recurring;
|
|
1062
|
+
if (opts.rrule) dynamicDef.rrule = opts.rrule;
|
|
1063
|
+
if (opts.every) dynamicDef.every = opts.every;
|
|
1064
|
+
if (opts.payload) dynamicDef.payload = opts.payload;
|
|
1065
|
+
}
|
|
1066
|
+
await engineRef.current.registerDynamic(dynamicDef);
|
|
1067
|
+
},
|
|
1068
|
+
cancel: async () => {
|
|
1069
|
+
if (!engineRef.current) return;
|
|
1070
|
+
await engineRef.current.cancel(def.name);
|
|
1071
|
+
}
|
|
1072
|
+
};
|
|
1073
|
+
};
|
|
1074
|
+
var ScheduleEngine = class {
|
|
1075
|
+
constructor(staticSchedules, db, lock, logger, environment, project, config) {
|
|
1076
|
+
this.db = db;
|
|
1077
|
+
this.lock = lock;
|
|
1078
|
+
this.environment = environment;
|
|
1079
|
+
this.project = project;
|
|
1080
|
+
this.config = config;
|
|
1081
|
+
this.nodeId = randomUUID();
|
|
1082
|
+
this.logger = logger ?? createLogger({ level: "info" }, { module: "scheduler" });
|
|
1083
|
+
this.stallDetector = new StallDetector(this.lock, this.logger, 15e3);
|
|
1084
|
+
for (const def of staticSchedules) {
|
|
1085
|
+
this.schedules.set(def.name, def);
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
name = "scheduler";
|
|
1089
|
+
enabled = true;
|
|
1090
|
+
nodeId;
|
|
1091
|
+
logger;
|
|
1092
|
+
leader;
|
|
1093
|
+
stallDetector;
|
|
1094
|
+
tickTimer;
|
|
1095
|
+
activeJobs = /* @__PURE__ */ new Map();
|
|
1096
|
+
// Includes static instances and dynamically triggered instances
|
|
1097
|
+
schedules = /* @__PURE__ */ new Map();
|
|
1098
|
+
_hasRunLeaderInit = false;
|
|
1099
|
+
// ── Lifecycle ───────────────────────────────────────────────────────────────
|
|
1100
|
+
async init() {
|
|
1101
|
+
this.logger.info("Initializing schedule engine", {
|
|
1102
|
+
nodeId: this.nodeId,
|
|
1103
|
+
staticCount: this.schedules.size
|
|
1104
|
+
});
|
|
1105
|
+
_attachScheduleEngine(this);
|
|
1106
|
+
const now = /* @__PURE__ */ new Date();
|
|
1107
|
+
for (const def of this.schedules.values()) {
|
|
1108
|
+
await this.upsertAndSeed(def, now);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async upsertAndSeed(def, now) {
|
|
1112
|
+
await this.db.upsertSchedule(def);
|
|
1113
|
+
const nextRun = this.computeNextRun(def, now);
|
|
1114
|
+
if (nextRun) {
|
|
1115
|
+
await this.db.updateNextRun(def.name, nextRun);
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
/** Dynamically register and schedule a definition from trigger/schedule call */
|
|
1119
|
+
async registerDynamic(def) {
|
|
1120
|
+
this.schedules.set(def.name, def);
|
|
1121
|
+
await this.upsertAndSeed(def, /* @__PURE__ */ new Date());
|
|
1122
|
+
this.logger.debug("Registered dynamic schedule", { name: def.name });
|
|
1123
|
+
}
|
|
1124
|
+
async cancel(name) {
|
|
1125
|
+
this.schedules.delete(name);
|
|
1126
|
+
}
|
|
1127
|
+
async start() {
|
|
1128
|
+
this.leader = new LeaderElection(
|
|
1129
|
+
this.lock,
|
|
1130
|
+
this.logger,
|
|
1131
|
+
"oqron:scheduleengine:leader",
|
|
1132
|
+
this.nodeId,
|
|
1133
|
+
3e4
|
|
1134
|
+
);
|
|
1135
|
+
await this.leader.start();
|
|
1136
|
+
const interval = this.config?.tickInterval ?? 1e3;
|
|
1137
|
+
this.tickTimer = setInterval(() => {
|
|
1138
|
+
void this.tick();
|
|
1139
|
+
}, interval);
|
|
1140
|
+
this.logger.info("Schedule engine started", {
|
|
1141
|
+
nodeId: this.nodeId,
|
|
1142
|
+
interval
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
async stop() {
|
|
1146
|
+
if (this.tickTimer) clearInterval(this.tickTimer);
|
|
1147
|
+
if (this.leader) await this.leader.stop();
|
|
1148
|
+
this.stallDetector.stop();
|
|
1149
|
+
for (const job of this.activeJobs.values()) {
|
|
1150
|
+
if (job.abort) job.abort.abort();
|
|
1151
|
+
if (job.worker) {
|
|
1152
|
+
await job.worker.stop();
|
|
1153
|
+
} else {
|
|
1154
|
+
await this.lock.release(job.lockKey, this.nodeId).catch(() => {
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
this.activeJobs.clear();
|
|
1159
|
+
this.logger.info("Schedule engine stopped");
|
|
1160
|
+
}
|
|
1161
|
+
// ── Core scheduling helpers ─────────────────────────────────────────────────
|
|
1162
|
+
computeNextRun(def, from) {
|
|
1163
|
+
try {
|
|
1164
|
+
if (def.runAt) {
|
|
1165
|
+
return new Date(def.runAt);
|
|
1166
|
+
}
|
|
1167
|
+
if (def.runAfter) {
|
|
1168
|
+
const add = (def.runAfter.days ?? 0) * 864e5 + (def.runAfter.hours ?? 0) * 36e5 + (def.runAfter.minutes ?? 0) * 6e4 + (def.runAfter.seconds ?? 0) * 1e3;
|
|
1169
|
+
return new Date(from.getTime() + add);
|
|
1170
|
+
}
|
|
1171
|
+
if (def.rrule) {
|
|
1172
|
+
const rule = rrulestr(def.rrule);
|
|
1173
|
+
return rule.after(from);
|
|
1174
|
+
}
|
|
1175
|
+
if (def.recurring) {
|
|
1176
|
+
const freqMapHash = {
|
|
1177
|
+
daily: RRule.DAILY,
|
|
1178
|
+
weekly: RRule.WEEKLY,
|
|
1179
|
+
monthly: RRule.MONTHLY,
|
|
1180
|
+
yearly: RRule.YEARLY
|
|
1181
|
+
};
|
|
1182
|
+
const freq = freqMapHash[def.recurring.frequency] ?? RRule.DAILY;
|
|
1183
|
+
const options = { freq };
|
|
1184
|
+
if (def.recurring.months?.length)
|
|
1185
|
+
options.bymonth = def.recurring.months;
|
|
1186
|
+
if (def.recurring.dayOfMonth)
|
|
1187
|
+
options.bymonthday = [def.recurring.dayOfMonth];
|
|
1188
|
+
if (def.recurring.at) {
|
|
1189
|
+
options.byhour = [def.recurring.at.hour];
|
|
1190
|
+
options.byminute = [def.recurring.at.minute];
|
|
1191
|
+
options.bysecond = [0];
|
|
1192
|
+
}
|
|
1193
|
+
const rule = new RRule(options);
|
|
1194
|
+
return rule.after(from);
|
|
1195
|
+
}
|
|
1196
|
+
if (def.every) {
|
|
1197
|
+
const add = (def.every.hours ?? 0) * 36e5 + (def.every.minutes ?? 0) * 6e4 + (def.every.seconds ?? 0) * 1e3;
|
|
1198
|
+
return new Date(from.getTime() + add);
|
|
1199
|
+
}
|
|
1200
|
+
} catch (err) {
|
|
1201
|
+
this.logger.error("Failed to compute next run", {
|
|
1202
|
+
name: def.name,
|
|
1203
|
+
err: String(err)
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
return null;
|
|
1207
|
+
}
|
|
1208
|
+
// ── Leader init (missed-fire recovery + stall detector) ─────────────────────
|
|
1209
|
+
async handleLeaderInit() {
|
|
1210
|
+
this.logger.info("Schedule Engine: Performing leader initialization...");
|
|
1211
|
+
this.stallDetector.start(
|
|
1212
|
+
() => {
|
|
1213
|
+
return Array.from(this.activeJobs.values()).filter((job) => job.worker !== void 0).map((job) => ({
|
|
1214
|
+
key: job.lockKey,
|
|
1215
|
+
ownerId: this.nodeId
|
|
1216
|
+
}));
|
|
1217
|
+
},
|
|
1218
|
+
(key) => {
|
|
1219
|
+
this.logger.warn("Local stall detected", { key });
|
|
1220
|
+
}
|
|
1221
|
+
);
|
|
1222
|
+
}
|
|
1223
|
+
// ── Tick loop ───────────────────────────────────────────────────────────────
|
|
1224
|
+
async tick() {
|
|
1225
|
+
if (!this.leader?.isLeader) return;
|
|
1226
|
+
if (!this._hasRunLeaderInit) {
|
|
1227
|
+
this._hasRunLeaderInit = true;
|
|
1228
|
+
await this.handleLeaderInit();
|
|
1229
|
+
}
|
|
1230
|
+
try {
|
|
1231
|
+
const now = /* @__PURE__ */ new Date();
|
|
1232
|
+
const due = await this.db.getDueSchedules(now, 50);
|
|
1233
|
+
for (const { name } of due) {
|
|
1234
|
+
const def = this.schedules.get(name);
|
|
1235
|
+
if (!def) continue;
|
|
1236
|
+
if (def.condition) {
|
|
1237
|
+
try {
|
|
1238
|
+
const conditionVal = await def.condition(
|
|
1239
|
+
new ScheduleContext({
|
|
1240
|
+
id: "eval",
|
|
1241
|
+
scheduleName: def.name,
|
|
1242
|
+
firedAt: now,
|
|
1243
|
+
logger: this.logger,
|
|
1244
|
+
signal: new AbortController().signal,
|
|
1245
|
+
payload: def.payload,
|
|
1246
|
+
environment: this.environment,
|
|
1247
|
+
project: this.project
|
|
1248
|
+
})
|
|
1249
|
+
);
|
|
1250
|
+
if (!conditionVal) {
|
|
1251
|
+
const nextRun2 = this.computeNextRun(def, now);
|
|
1252
|
+
if (nextRun2) await this.db.updateNextRun(def.name, nextRun2);
|
|
1253
|
+
continue;
|
|
1254
|
+
}
|
|
1255
|
+
} catch (e) {
|
|
1256
|
+
this.logger.error("Condition crashed", {
|
|
1257
|
+
name: def.name,
|
|
1258
|
+
error: String(e)
|
|
1259
|
+
});
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
let nextRun = null;
|
|
1264
|
+
if (!def.runAt && !def.runAfter) {
|
|
1265
|
+
nextRun = this.computeNextRun(def, now);
|
|
1266
|
+
if (!nextRun) {
|
|
1267
|
+
this.logger.error(
|
|
1268
|
+
"Cannot compute next run \u2014 suspending schedule to prevent runaway loop",
|
|
1269
|
+
{ name: def.name }
|
|
1270
|
+
);
|
|
1271
|
+
await this.db.updateNextRun(def.name, null).catch(() => {
|
|
1272
|
+
});
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
await this.db.updateNextRun(def.name, nextRun);
|
|
1277
|
+
void this.fire(def);
|
|
1278
|
+
}
|
|
1279
|
+
} catch (err) {
|
|
1280
|
+
this.logger.error("Tick error", { err: String(err) });
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
// ── Fire handler ────────────────────────────────────────────────────────────
|
|
1284
|
+
async fire(def) {
|
|
1285
|
+
const isOverlapSkip = def.overlap === "skip" || def.overlap === false;
|
|
1286
|
+
if (isOverlapSkip) {
|
|
1287
|
+
for (const job of this.activeJobs.values()) {
|
|
1288
|
+
if (job.lockKey === `oqron:schedule:run:${def.name}`) {
|
|
1289
|
+
this.logger.debug("Skipping overlapping run", { name: def.name });
|
|
1290
|
+
return;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
const runId = randomUUID();
|
|
1295
|
+
const lockKey = isOverlapSkip ? `oqron:schedule:run:${def.name}` : `oqron:schedule:run:${def.name}:${runId}`;
|
|
1296
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
1297
|
+
let worker;
|
|
1298
|
+
let acquired = false;
|
|
1299
|
+
if (def.guaranteedWorker) {
|
|
1300
|
+
worker = new HeartbeatWorker(
|
|
1301
|
+
this.lock,
|
|
1302
|
+
this.logger,
|
|
1303
|
+
lockKey,
|
|
1304
|
+
this.nodeId,
|
|
1305
|
+
def.lockTtlMs ?? 3e4,
|
|
1306
|
+
def.heartbeatMs ?? 1e4
|
|
1307
|
+
);
|
|
1308
|
+
acquired = await worker.start();
|
|
1309
|
+
} else {
|
|
1310
|
+
acquired = await this.lock.acquire(
|
|
1311
|
+
lockKey,
|
|
1312
|
+
this.nodeId,
|
|
1313
|
+
def.lockTtlMs ?? 3e4
|
|
1314
|
+
);
|
|
1315
|
+
}
|
|
1316
|
+
if (!acquired) return;
|
|
1317
|
+
const abort = new AbortController();
|
|
1318
|
+
this.activeJobs.set(runId, { runId, lockKey, worker, abort });
|
|
1319
|
+
await this.db.recordExecution({
|
|
1320
|
+
id: runId,
|
|
1321
|
+
scheduleId: def.name,
|
|
1322
|
+
status: "running",
|
|
1323
|
+
startedAt
|
|
1324
|
+
});
|
|
1325
|
+
void Promise.resolve().then(async () => {
|
|
1326
|
+
const ctx = new ScheduleContext({
|
|
1327
|
+
id: runId,
|
|
1328
|
+
logger: this.logger.child({ schedule: def.name }),
|
|
1329
|
+
signal: abort.signal,
|
|
1330
|
+
firedAt: startedAt,
|
|
1331
|
+
scheduleName: def.name,
|
|
1332
|
+
payload: def.payload,
|
|
1333
|
+
environment: this.environment,
|
|
1334
|
+
project: this.project,
|
|
1335
|
+
onProgress: (percent, label) => {
|
|
1336
|
+
this.db.updateJobProgress(runId, percent, label).catch(
|
|
1337
|
+
(err) => this.logger.error("Failed to update schedule progress", {
|
|
1338
|
+
runId,
|
|
1339
|
+
err
|
|
1340
|
+
})
|
|
1341
|
+
);
|
|
1342
|
+
}
|
|
1343
|
+
});
|
|
1344
|
+
let status = "completed";
|
|
1345
|
+
let error;
|
|
1346
|
+
let timeoutHandle;
|
|
1347
|
+
let finalResult;
|
|
1348
|
+
let attempts = 1;
|
|
1349
|
+
const maxAttempts = (def.retries?.max ?? 0) + 1;
|
|
1350
|
+
while (attempts <= maxAttempts) {
|
|
1351
|
+
try {
|
|
1352
|
+
if (def.hooks?.beforeRun) await def.hooks.beforeRun(ctx);
|
|
1353
|
+
if (def.timeout) {
|
|
1354
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1355
|
+
timeoutHandle = setTimeout(() => {
|
|
1356
|
+
abort.abort();
|
|
1357
|
+
reject(new Error(`Handler timed out after ${def.timeout}ms`));
|
|
1358
|
+
}, def.timeout);
|
|
1359
|
+
});
|
|
1360
|
+
finalResult = await Promise.race([
|
|
1361
|
+
def.handler(ctx),
|
|
1362
|
+
timeoutPromise
|
|
1363
|
+
]);
|
|
1364
|
+
} else {
|
|
1365
|
+
finalResult = await def.handler(ctx);
|
|
1366
|
+
}
|
|
1367
|
+
if (def.hooks?.afterRun) {
|
|
1368
|
+
await def.hooks.afterRun(ctx, finalResult);
|
|
1369
|
+
}
|
|
1370
|
+
status = "completed";
|
|
1371
|
+
error = void 0;
|
|
1372
|
+
break;
|
|
1373
|
+
} catch (err) {
|
|
1374
|
+
error = err instanceof Error ? err.message : String(err);
|
|
1375
|
+
status = "failed";
|
|
1376
|
+
if (def.hooks?.onError && err instanceof Error) {
|
|
1377
|
+
try {
|
|
1378
|
+
await Promise.resolve(def.hooks.onError(ctx, err));
|
|
1379
|
+
} catch (e) {
|
|
1380
|
+
this.logger.error("onError hook threw", { err: String(e) });
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
if (attempts < maxAttempts) {
|
|
1384
|
+
this.logger.warn("Schedule handler threw, retrying...", {
|
|
1385
|
+
name: def.name,
|
|
1386
|
+
runId,
|
|
1387
|
+
attempt: attempts,
|
|
1388
|
+
error
|
|
1389
|
+
});
|
|
1390
|
+
const baseDelay = def.retries?.baseDelay ?? 2e3;
|
|
1391
|
+
const delay = def.retries?.strategy === "exponential" ? baseDelay * 2 ** (attempts - 1) : baseDelay;
|
|
1392
|
+
attempts++;
|
|
1393
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1394
|
+
} else {
|
|
1395
|
+
this.logger.error("Schedule handler failed completely", {
|
|
1396
|
+
name: def.name,
|
|
1397
|
+
runId,
|
|
1398
|
+
attempts,
|
|
1399
|
+
error
|
|
1400
|
+
});
|
|
1401
|
+
break;
|
|
1402
|
+
}
|
|
1403
|
+
} finally {
|
|
1404
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
const completedAt = /* @__PURE__ */ new Date();
|
|
1408
|
+
await this.db.recordExecution({
|
|
1409
|
+
id: runId,
|
|
1410
|
+
scheduleId: def.name,
|
|
1411
|
+
status,
|
|
1412
|
+
startedAt,
|
|
1413
|
+
completedAt,
|
|
1414
|
+
error,
|
|
1415
|
+
result: finalResult !== void 0 ? JSON.stringify(finalResult) : void 0,
|
|
1416
|
+
attempts,
|
|
1417
|
+
durationMs: completedAt.getTime() - startedAt.getTime(),
|
|
1418
|
+
...status === "completed" ? { progressPercent: 100, progressLabel: "Completed" } : {}
|
|
1419
|
+
});
|
|
1420
|
+
if (worker) {
|
|
1421
|
+
await worker.stop();
|
|
1422
|
+
} else {
|
|
1423
|
+
await this.lock.release(lockKey, this.nodeId).catch(() => {
|
|
1424
|
+
});
|
|
1425
|
+
}
|
|
1426
|
+
this.activeJobs.delete(runId);
|
|
1427
|
+
const keepJobHistory = def.keepHistory ?? this.config?.keepJobHistory ?? true;
|
|
1428
|
+
const keepFailedHistory = def.keepFailedHistory ?? this.config?.keepFailedJobHistory ?? true;
|
|
1429
|
+
if (keepJobHistory !== true || keepFailedHistory !== true) {
|
|
1430
|
+
this.db.pruneHistoryForSchedule(def.name, keepJobHistory, keepFailedHistory).catch(
|
|
1431
|
+
(err) => this.logger.debug("Failed to prune history", {
|
|
1432
|
+
err,
|
|
1433
|
+
name: def.name
|
|
1434
|
+
})
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
this.logger.info("Schedule finished", {
|
|
1438
|
+
name: def.name,
|
|
1439
|
+
runId,
|
|
1440
|
+
status,
|
|
1441
|
+
attempts
|
|
1442
|
+
});
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
};
|
|
1446
|
+
|
|
1447
|
+
export { ScheduleEngine, SchedulerModule, _drainPending, _drainPendingSchedules, _registerCron, _registerSchedule, cron, getNextRunDate, schedule };
|