@dbos-inc/dbos-sdk 5.5.5-preview → 5.5.7-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/dbos-executor.d.ts +3 -1
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +8 -13
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +3 -3
- package/dist/src/dbos.js +3 -3
- package/dist/src/scheduler/scheduler.d.ts +78 -3
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +414 -174
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/system_database.d.ts +20 -4
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +155 -85
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +2 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +9 -1
- package/dist/src/utils.js.map +1 -1
- package/dist/src/workflow_management.d.ts +12 -3
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +63 -27
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.scheduleShardKey = exports.backfillSchedule = exports.triggerSchedule = exports.DynamicSchedulerLoop = exports.scheduleSignature = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
|
|
3
|
+
exports.scheduleShardKey = exports.backfillSchedule = exports.triggerSchedule = exports.enqueueTicks = exports.DynamicSchedulerLoop = exports.schedulerLoop = exports.Scheduler = exports.ScheduleEntry = exports.scheduleDefinition = exports.schedulerConfig = exports.scheduleSignature = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
|
|
4
4
|
const serialization_1 = require("../serialization");
|
|
5
5
|
const crypto_1 = require("crypto");
|
|
6
|
-
const __1 = require("..");
|
|
7
6
|
const utils_1 = require("../utils");
|
|
8
7
|
const crontab_1 = require("./crontab");
|
|
9
|
-
const dbos_executor_1 = require("../dbos-executor");
|
|
10
8
|
const error_1 = require("../error");
|
|
11
9
|
const workflow_1 = require("../workflow");
|
|
12
10
|
const context_1 = require("../context");
|
|
@@ -33,7 +31,7 @@ function createScheduleId() {
|
|
|
33
31
|
return (0, crypto_1.randomUUID)();
|
|
34
32
|
}
|
|
35
33
|
exports.createScheduleId = createScheduleId;
|
|
36
|
-
// Definition fields
|
|
34
|
+
// Definition fields an armed schedule captures; a change re-arms it. Runtime state (schedule_id, status, last_fired_at) is excluded so an unchanged re-apply doesn't re-arm it.
|
|
37
35
|
function scheduleSignature(sched) {
|
|
38
36
|
return JSON.stringify([
|
|
39
37
|
sched.workflowName,
|
|
@@ -46,194 +44,413 @@ function scheduleSignature(sched) {
|
|
|
46
44
|
]);
|
|
47
45
|
}
|
|
48
46
|
exports.scheduleSignature = scheduleSignature;
|
|
47
|
+
exports.schedulerConfig = {
|
|
48
|
+
/** Most scheduled runs enqueued in one transaction. */
|
|
49
|
+
fireBatchSize: 1000,
|
|
50
|
+
/** How long a standby keeps a tick no poll has shown fired, or three polling intervals if longer. */
|
|
51
|
+
bufferMinMs: 3_600_000,
|
|
52
|
+
/** How far back a starting scheduler arms each schedule, so a tick missed while its leader died still fires. */
|
|
53
|
+
startupLookbackMs: 30_000,
|
|
54
|
+
/** How often a scheduler that isn't firing checks whether it has taken over. */
|
|
55
|
+
standbyCheckMs: 1000,
|
|
56
|
+
};
|
|
49
57
|
// During the first minute after startup, poll for schedules every second so schedules
|
|
50
58
|
// registered around launch are picked up promptly rather than after a full polling interval.
|
|
51
59
|
const STARTUP_FAST_POLL_DURATION_MS = 60_000;
|
|
52
60
|
const STARTUP_FAST_POLL_INTERVAL_MS = 1_000;
|
|
53
|
-
// Timers can't sleep past ~24.8 days, so wait for distant
|
|
61
|
+
// Timers can't sleep past ~24.8 days, so wait for distant ticks in bounded chunks.
|
|
54
62
|
const MAX_SCHEDULE_SLEEP_MS = 60 * 60 * 1000;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
63
|
+
async function scheduleDefinition(sched, serializer) {
|
|
64
|
+
return {
|
|
65
|
+
scheduleName: sched.scheduleName,
|
|
66
|
+
workflowName: sched.workflowName,
|
|
67
|
+
workflowClassName: sched.workflowClassName,
|
|
68
|
+
context: await serializer.parse(sched.context),
|
|
69
|
+
queueName: sched.queueName ?? undefined,
|
|
70
|
+
applicationName: sched.applicationName,
|
|
71
|
+
shardKey: sched.shardKey,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
exports.scheduleDefinition = scheduleDefinition;
|
|
75
|
+
/** The first tick strictly after `after`; throws if the schedule can never fire again. */
|
|
76
|
+
function nextTick(matcher, after) {
|
|
77
|
+
const next = matcher.nextWakeupTime(after).getTime();
|
|
78
|
+
// TimeMatcher returns a non-matching instant only past its search horizon.
|
|
79
|
+
if (!matcher.match(next)) {
|
|
80
|
+
throw new error_1.DBOSError(`The schedule never fires after ${new Date(after).toISOString()}`);
|
|
62
81
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
82
|
+
return next;
|
|
83
|
+
}
|
|
84
|
+
/** An active schedule in the scheduler and its next tick. */
|
|
85
|
+
class ScheduleEntry {
|
|
86
|
+
definition;
|
|
87
|
+
matcher;
|
|
88
|
+
scheduleId;
|
|
89
|
+
// Definition snapshot; the scheduler re-arms the entry when it changes.
|
|
90
|
+
signature;
|
|
91
|
+
nextTime;
|
|
92
|
+
constructor(sched, definition, matcher, after) {
|
|
93
|
+
this.definition = definition;
|
|
94
|
+
this.matcher = matcher;
|
|
95
|
+
this.scheduleId = sched.scheduleId;
|
|
96
|
+
this.signature = scheduleSignature(sched);
|
|
97
|
+
this.nextTime = nextTick(matcher, after);
|
|
69
98
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
99
|
+
/** Arm `sched` at its first tick after `after`, by default now. Throws if it can't run. */
|
|
100
|
+
static async create(sched, matcher, serializer, after) {
|
|
101
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
102
|
+
return new ScheduleEntry(sched, definition, matcher, after ?? Date.now());
|
|
103
|
+
}
|
|
104
|
+
advance() {
|
|
105
|
+
this.nextTime = nextTick(this.matcher, this.nextTime);
|
|
106
|
+
}
|
|
107
|
+
advancePast(lastFired) {
|
|
108
|
+
while (this.nextTime <= lastFired) {
|
|
109
|
+
this.advance();
|
|
75
110
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
exports.ScheduleEntry = ScheduleEntry;
|
|
114
|
+
/** A min-heap of entries by next tick, ties in push order. */
|
|
115
|
+
class TickHeap {
|
|
116
|
+
#items = [];
|
|
117
|
+
#seq = 0;
|
|
118
|
+
get size() {
|
|
119
|
+
return this.#items.length;
|
|
120
|
+
}
|
|
121
|
+
/** The earliest tick, or Infinity if empty. */
|
|
122
|
+
peek() {
|
|
123
|
+
return this.#items[0]?.at ?? Infinity;
|
|
124
|
+
}
|
|
125
|
+
push(entry) {
|
|
126
|
+
const items = this.#items;
|
|
127
|
+
items.push({ at: entry.nextTime, seq: this.#seq++, entry });
|
|
128
|
+
let i = items.length - 1;
|
|
129
|
+
while (i > 0) {
|
|
130
|
+
const parent = (i - 1) >> 1;
|
|
131
|
+
if (!this.#before(i, parent))
|
|
132
|
+
break;
|
|
133
|
+
[items[i], items[parent]] = [items[parent], items[i]];
|
|
134
|
+
i = parent;
|
|
79
135
|
}
|
|
80
|
-
|
|
81
|
-
|
|
136
|
+
}
|
|
137
|
+
pop() {
|
|
138
|
+
const items = this.#items;
|
|
139
|
+
const top = items[0];
|
|
140
|
+
const last = items.pop();
|
|
141
|
+
if (items.length > 0) {
|
|
142
|
+
items[0] = last;
|
|
143
|
+
let i = 0;
|
|
144
|
+
while (true) {
|
|
145
|
+
const left = 2 * i + 1;
|
|
146
|
+
const right = left + 1;
|
|
147
|
+
let least = i;
|
|
148
|
+
if (left < items.length && this.#before(left, least))
|
|
149
|
+
least = left;
|
|
150
|
+
if (right < items.length && this.#before(right, least))
|
|
151
|
+
least = right;
|
|
152
|
+
if (least === i)
|
|
153
|
+
break;
|
|
154
|
+
[items[i], items[least]] = [items[least], items[i]];
|
|
155
|
+
i = least;
|
|
156
|
+
}
|
|
82
157
|
}
|
|
83
|
-
|
|
84
|
-
this.#scheduleLoops.clear();
|
|
158
|
+
return top.entry;
|
|
85
159
|
}
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
160
|
+
#before(a, b) {
|
|
161
|
+
const x = this.#items[a];
|
|
162
|
+
const y = this.#items[b];
|
|
163
|
+
return x.at < y.at || (x.at === y.at && x.seq < y.seq);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/** Fires every active schedule from one loop, enqueueing each tick's runs in batches. */
|
|
167
|
+
class Scheduler {
|
|
168
|
+
systemDatabase;
|
|
169
|
+
serializer;
|
|
170
|
+
logger;
|
|
171
|
+
// Active schedules keyed by schedule ID.
|
|
172
|
+
entries = new Map();
|
|
173
|
+
// One item per entry in `entries`; rebuilt by every poll.
|
|
174
|
+
heap = new TickHeap();
|
|
175
|
+
// Whether a poll has armed the schedules this scheduler started with.
|
|
176
|
+
#started = false;
|
|
177
|
+
constructor(systemDatabase, serializer, logger) {
|
|
178
|
+
this.systemDatabase = systemDatabase;
|
|
179
|
+
this.serializer = serializer;
|
|
180
|
+
this.logger = logger;
|
|
181
|
+
}
|
|
182
|
+
/** Sync the armed schedules with the database, then rebuild the heap from them. */
|
|
183
|
+
async poll() {
|
|
184
|
+
try {
|
|
185
|
+
await this.#reconcile();
|
|
186
|
+
}
|
|
187
|
+
finally {
|
|
188
|
+
// Even after a partial reconcile, so the heap never holds dropped entries or misses one.
|
|
189
|
+
this.heap = new TickHeap();
|
|
190
|
+
for (const entry of this.entries.values()) {
|
|
191
|
+
this.heap.push(entry);
|
|
98
192
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
async #reconcile() {
|
|
196
|
+
const schedules = await this.systemDatabase.listSchedules({ applicationName: this.systemDatabase.appName });
|
|
197
|
+
const currentIds = new Set(schedules.map((s) => s.scheduleId));
|
|
198
|
+
for (const scheduleId of this.entries.keys()) {
|
|
199
|
+
if (!currentIds.has(scheduleId)) {
|
|
200
|
+
this.entries.delete(scheduleId);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
for (const sched of schedules) {
|
|
204
|
+
const entry = this.entries.get(sched.scheduleId);
|
|
205
|
+
if (sched.status !== 'ACTIVE') {
|
|
206
|
+
// Paused: drop it; resuming re-arms it, backfilling if enabled.
|
|
207
|
+
this.entries.delete(sched.scheduleId);
|
|
102
208
|
continue;
|
|
103
209
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
entry.controller.abort();
|
|
110
|
-
this.#scheduleLoops.delete(name);
|
|
210
|
+
if (entry !== undefined) {
|
|
211
|
+
// Running: re-arm on a changed definition, without backfill.
|
|
212
|
+
if (entry.signature !== scheduleSignature(sched)) {
|
|
213
|
+
this.entries.delete(sched.scheduleId);
|
|
214
|
+
await this.#arm(sched);
|
|
111
215
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
for (const sched of schedules) {
|
|
115
|
-
const existing = this.#scheduleLoops.get(sched.scheduleName);
|
|
116
|
-
const signature = scheduleSignature(sched);
|
|
117
|
-
if (sched.status === 'PAUSED' && existing) {
|
|
118
|
-
// Paused but has a running loop — stop it
|
|
119
|
-
existing.controller.abort();
|
|
120
|
-
this.#scheduleLoops.delete(sched.scheduleName);
|
|
216
|
+
else {
|
|
217
|
+
this.#skipFired(entry, sched);
|
|
121
218
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const now = new Date();
|
|
135
|
-
if (lastFired < now) {
|
|
136
|
-
const executor = dbos_executor_1.DBOSExecutor.globalInstance;
|
|
137
|
-
await backfillSchedule(executor.systemDatabase, executor.serializer, sched.scheduleName, lastFired, now);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
catch (e) {
|
|
141
|
-
__1.DBOS.logger.warn(`Dynamic scheduler: error backfilling schedule "${sched.scheduleName}": ${e.message}`);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
// An abandoned backfill means shutdown began; a loop started now would outlive destroy().
|
|
145
|
-
if (signal.aborted)
|
|
146
|
-
return;
|
|
147
|
-
// Active and no running loop — start one
|
|
148
|
-
const controller = new AbortController();
|
|
149
|
-
const executor = dbos_executor_1.DBOSExecutor.globalInstance;
|
|
150
|
-
// Only enqueues, never runs the workflow, so its retries may be abandoned when the loop stops.
|
|
151
|
-
const promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => DynamicSchedulerLoop.#scheduleLoop(sched.scheduleName, sched.workflowName, sched.workflowClassName, sched.schedule, sched.context, executor.serializer, controller.signal, sched.cronTimezone ?? undefined, sched.queueName ?? undefined, sched.applicationName, sched.shardKey));
|
|
152
|
-
this.#scheduleLoops.set(sched.scheduleName, { controller, promise, signature });
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
// Not running: arm it, then backfill missed executions if enabled; a broken one is retried next poll.
|
|
222
|
+
const armed = await this.#arm(sched, this.#started ? undefined : startupAfter(sched));
|
|
223
|
+
if (armed === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
if (sched.automaticBackfill && sched.lastFiredAt) {
|
|
226
|
+
try {
|
|
227
|
+
const lastFired = new Date(sched.lastFiredAt);
|
|
228
|
+
const now = new Date();
|
|
229
|
+
if (lastFired < now) {
|
|
230
|
+
await backfillSchedule(this.systemDatabase, this.serializer, sched.scheduleName, lastFired, now);
|
|
153
231
|
}
|
|
154
232
|
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
this.logger.warn(`Exception during automatic backfill for schedule '${sched.scheduleName}': ${errorText(e)}`);
|
|
235
|
+
}
|
|
155
236
|
}
|
|
156
|
-
|
|
237
|
+
}
|
|
238
|
+
this.#started = true;
|
|
239
|
+
}
|
|
240
|
+
/** Pass over the ticks this schedule's last_fired_at shows some process enqueued. */
|
|
241
|
+
#skipFired(entry, sched) {
|
|
242
|
+
if (!sched.lastFiredAt)
|
|
243
|
+
return;
|
|
244
|
+
const lastFired = Date.parse(sched.lastFiredAt);
|
|
245
|
+
// Unreadable: the entry keeps its ticks, refired at worst.
|
|
246
|
+
if (Number.isNaN(lastFired))
|
|
247
|
+
return;
|
|
248
|
+
try {
|
|
249
|
+
entry.advancePast(lastFired);
|
|
250
|
+
}
|
|
251
|
+
catch (e) {
|
|
252
|
+
this.#drop(entry, e);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/** Register a new entry for this schedule, or log why it can't run and return undefined. */
|
|
256
|
+
async #arm(sched, after) {
|
|
257
|
+
let matcher;
|
|
258
|
+
try {
|
|
259
|
+
matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
|
|
260
|
+
}
|
|
261
|
+
catch (e) {
|
|
262
|
+
this.logger.error(`Cannot run schedule '${sched.scheduleName}'. Invalid crontab "${sched.schedule}": ${errorText(e)}`);
|
|
263
|
+
return undefined;
|
|
264
|
+
}
|
|
265
|
+
let entry;
|
|
266
|
+
try {
|
|
267
|
+
entry = await ScheduleEntry.create(sched, matcher, this.serializer, after);
|
|
268
|
+
}
|
|
269
|
+
catch (e) {
|
|
270
|
+
this.logger.error(`Cannot run schedule '${sched.scheduleName}': ${errorText(e)}`);
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
this.entries.set(entry.scheduleId, entry);
|
|
274
|
+
return entry;
|
|
275
|
+
}
|
|
276
|
+
/** Milliseconds until the earliest tick is `lag` milliseconds old, Infinity if none. */
|
|
277
|
+
msUntilDue(lag = 0) {
|
|
278
|
+
return this.heap.peek() + lag - Date.now();
|
|
279
|
+
}
|
|
280
|
+
/** Fire every due tick, batch by batch, until `signal` aborts. */
|
|
281
|
+
async fireDue(signal) {
|
|
282
|
+
const now = Date.now();
|
|
283
|
+
const due = [];
|
|
284
|
+
while (this.heap.size > 0 && this.heap.peek() <= now) {
|
|
285
|
+
due.push(this.heap.pop());
|
|
286
|
+
}
|
|
287
|
+
const batchSize = exports.schedulerConfig.fireBatchSize;
|
|
288
|
+
let fired = 0;
|
|
289
|
+
while (fired < due.length && !signal.aborted) {
|
|
290
|
+
await this.#fire(due.slice(fired, fired + batchSize), signal);
|
|
291
|
+
fired += batchSize;
|
|
292
|
+
}
|
|
293
|
+
for (const entry of due.slice(0, fired)) {
|
|
294
|
+
this.#advance(entry);
|
|
295
|
+
}
|
|
296
|
+
// Left unstamped, so the process that takes over fires them.
|
|
297
|
+
for (const entry of due.slice(fired)) {
|
|
298
|
+
this.heap.push(entry);
|
|
157
299
|
}
|
|
158
300
|
}
|
|
159
|
-
|
|
160
|
-
|
|
301
|
+
/** Pass over ticks older than `before` without firing them. */
|
|
302
|
+
skipDue(before) {
|
|
303
|
+
while (this.heap.size > 0 && this.heap.peek() <= before) {
|
|
304
|
+
this.#advance(this.heap.pop());
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
#advance(entry) {
|
|
308
|
+
// A late tick's successor may already be due; it fires on the next pass.
|
|
161
309
|
try {
|
|
162
|
-
|
|
310
|
+
entry.advance();
|
|
163
311
|
}
|
|
164
312
|
catch (e) {
|
|
165
|
-
|
|
313
|
+
this.#drop(entry, e);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
this.heap.push(entry);
|
|
317
|
+
}
|
|
318
|
+
#drop(entry, e) {
|
|
319
|
+
// Dropped; the next poll tries to arm it again.
|
|
320
|
+
this.entries.delete(entry.scheduleId);
|
|
321
|
+
this.logger.error(`Cannot run schedule '${entry.definition.scheduleName}': ${errorText(e)}`);
|
|
322
|
+
}
|
|
323
|
+
async #fire(batch, signal) {
|
|
324
|
+
try {
|
|
325
|
+
await enqueueTicks(this.systemDatabase, this.serializer, batch.map((entry) => [entry.definition, entry.nextTime]), true);
|
|
166
326
|
return;
|
|
167
327
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
workflowClassName,
|
|
173
|
-
schedule: cronExpression,
|
|
174
|
-
status: 'ACTIVE',
|
|
175
|
-
context: serializedContext,
|
|
176
|
-
lastFiredAt: null,
|
|
177
|
-
automaticBackfill: false,
|
|
178
|
-
cronTimezone: cronTimezone ?? null,
|
|
179
|
-
queueName: queueName ?? null,
|
|
180
|
-
applicationName,
|
|
181
|
-
shardKey,
|
|
182
|
-
};
|
|
183
|
-
let lastExec = new Date().setMilliseconds(0);
|
|
184
|
-
while (!signal.aborted) {
|
|
185
|
-
const nextExec = timeMatcher.nextWakeupTime(lastExec).getTime();
|
|
186
|
-
const untilNext = nextExec - Date.now();
|
|
187
|
-
// Apply jitter to prevent thundering herd
|
|
188
|
-
const wakeTime = untilNext > 0 ? nextExec + Math.random() * Math.min(untilNext / 10, 10000) : nextExec;
|
|
189
|
-
while (!signal.aborted && Date.now() < wakeTime) {
|
|
190
|
-
await (0, utils_1.interruptibleSleep)(Math.min(wakeTime - Date.now(), MAX_SCHEDULE_SLEEP_MS), signal);
|
|
328
|
+
catch (e) {
|
|
329
|
+
if (batch.length === 1) {
|
|
330
|
+
this.logger.warn(`Exception in schedule '${batch[0].definition.scheduleName}': ${errorText(e)}`);
|
|
331
|
+
return;
|
|
191
332
|
}
|
|
333
|
+
// A stopping loop's retries are cut short, so retrying each schedule would only log a warning per schedule.
|
|
192
334
|
if (signal.aborted) {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// If TimeMatcher did not find the next occurrence of the schedule yet,
|
|
196
|
-
// we must have it determine the next wakeup time.
|
|
197
|
-
if (!timeMatcher.match(nextExec)) {
|
|
198
|
-
lastExec = nextExec;
|
|
199
|
-
continue;
|
|
335
|
+
this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows: ${errorText(e)}`);
|
|
336
|
+
return;
|
|
200
337
|
}
|
|
201
|
-
|
|
202
|
-
|
|
338
|
+
// Logged here, as the per-schedule retries may succeed and never show it.
|
|
339
|
+
this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows, retrying each schedule alone: ${errorText(e)}`);
|
|
340
|
+
}
|
|
341
|
+
// One failing schedule must not hold back the rest of its batch.
|
|
342
|
+
for (const entry of batch) {
|
|
343
|
+
if (signal.aborted)
|
|
344
|
+
return;
|
|
345
|
+
await this.#fire([entry], signal);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
exports.Scheduler = Scheduler;
|
|
350
|
+
/** Where a starting scheduler arms a schedule: past its last fired tick, back at most the startup look-back. Later arms (creates, resumes) start from now. */
|
|
351
|
+
function startupAfter(sched) {
|
|
352
|
+
if (!sched.lastFiredAt)
|
|
353
|
+
return undefined;
|
|
354
|
+
const lastFired = Date.parse(sched.lastFiredAt);
|
|
355
|
+
if (Number.isNaN(lastFired))
|
|
356
|
+
return undefined;
|
|
357
|
+
return Math.max(lastFired, Date.now() - exports.schedulerConfig.startupLookbackMs);
|
|
358
|
+
}
|
|
359
|
+
function errorText(e) {
|
|
360
|
+
return e instanceof Error ? (e.stack ?? e.message) : String(e);
|
|
361
|
+
}
|
|
362
|
+
/** Poll schedules until aborted, firing only while `leading()`; a standby keeps each tick until a poll shows it fired, so a taker fires all it missed. */
|
|
363
|
+
async function schedulerLoop(signal, scheduler, pollingIntervalMs, leading) {
|
|
364
|
+
const bufferMs = Math.max(exports.schedulerConfig.bufferMinMs, 3 * pollingIntervalMs);
|
|
365
|
+
const startupDeadline = performance.now() + STARTUP_FAST_POLL_DURATION_MS;
|
|
366
|
+
const pollTimeout = () => performance.now() < startupDeadline
|
|
367
|
+
? Math.min(STARTUP_FAST_POLL_INTERVAL_MS, pollingIntervalMs)
|
|
368
|
+
: pollingIntervalMs;
|
|
369
|
+
let nextPoll = performance.now();
|
|
370
|
+
while (!signal.aborted) {
|
|
371
|
+
if (performance.now() >= nextPoll) {
|
|
203
372
|
try {
|
|
204
|
-
|
|
205
|
-
const existing = await __1.DBOS.getWorkflowStatus(workflowID, { shardKey });
|
|
206
|
-
if (existing) {
|
|
207
|
-
lastExec = nextExec;
|
|
208
|
-
continue;
|
|
209
|
-
}
|
|
210
|
-
const context = await serializer.parse(serializedContext);
|
|
211
|
-
const systemDatabase = dbos_executor_1.DBOSExecutor.globalInstance.systemDatabase;
|
|
212
|
-
await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, date, context);
|
|
213
|
-
await systemDatabase.updateLastFiredAt(scheduleName, date.toISOString());
|
|
373
|
+
await scheduler.poll();
|
|
214
374
|
}
|
|
215
375
|
catch (e) {
|
|
216
|
-
|
|
376
|
+
scheduler.logger.warn(`Exception polling schedules: ${errorText(e)}`);
|
|
217
377
|
}
|
|
218
|
-
|
|
378
|
+
nextPoll = performance.now() + pollTimeout();
|
|
379
|
+
}
|
|
380
|
+
// Checked once per pass: a holder losing the lock mid-pass still fires all it took.
|
|
381
|
+
const firing = leading();
|
|
382
|
+
try {
|
|
383
|
+
if (firing) {
|
|
384
|
+
await scheduler.fireDue(signal);
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
scheduler.skipDue(Date.now() - bufferMs);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
scheduler.logger.warn(`Exception firing schedules: ${errorText(e)}`);
|
|
392
|
+
}
|
|
393
|
+
const untilPoll = nextPoll - performance.now();
|
|
394
|
+
const wait = firing
|
|
395
|
+
? Math.min(untilPoll, scheduler.msUntilDue())
|
|
396
|
+
: Math.min(untilPoll, scheduler.msUntilDue(bufferMs), exports.schedulerConfig.standbyCheckMs);
|
|
397
|
+
// Always yields, so a pass that finds more due work never starves the event loop.
|
|
398
|
+
await (0, utils_1.interruptibleSleep)(Math.min(Math.max(wait, 0), MAX_SCHEDULE_SLEEP_MS), signal);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
exports.schedulerLoop = schedulerLoop;
|
|
402
|
+
/** Runs the scheduler loop from `initialize` until `destroy`, firing while `leading()`. */
|
|
403
|
+
class DynamicSchedulerLoop {
|
|
404
|
+
systemDatabase;
|
|
405
|
+
serializer;
|
|
406
|
+
logger;
|
|
407
|
+
leading;
|
|
408
|
+
#controller;
|
|
409
|
+
#promise;
|
|
410
|
+
#pollingIntervalMs;
|
|
411
|
+
constructor(systemDatabase, serializer, logger, pollingIntervalMs, leading) {
|
|
412
|
+
this.systemDatabase = systemDatabase;
|
|
413
|
+
this.serializer = serializer;
|
|
414
|
+
this.logger = logger;
|
|
415
|
+
this.leading = leading;
|
|
416
|
+
this.#pollingIntervalMs = pollingIntervalMs ?? 30000;
|
|
417
|
+
}
|
|
418
|
+
/** Start the loop; a no-op while it runs. */
|
|
419
|
+
async initialize() {
|
|
420
|
+
if (this.#controller !== undefined)
|
|
421
|
+
return;
|
|
422
|
+
const controller = new AbortController();
|
|
423
|
+
this.#controller = controller;
|
|
424
|
+
const scheduler = new Scheduler(this.systemDatabase, this.serializer, this.logger);
|
|
425
|
+
// Only enqueues, never runs a workflow, so its retries may be abandoned when the loop stops.
|
|
426
|
+
this.#promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => schedulerLoop(controller.signal, scheduler, this.#pollingIntervalMs, this.leading));
|
|
427
|
+
await Promise.resolve();
|
|
428
|
+
}
|
|
429
|
+
/** Stop the loop and wait for it to end; a no-op if it isn't running. */
|
|
430
|
+
async destroy() {
|
|
431
|
+
const controller = this.#controller;
|
|
432
|
+
const promise = this.#promise;
|
|
433
|
+
// Cleared first, so an initialize while this one winds down starts a fresh loop.
|
|
434
|
+
this.#controller = undefined;
|
|
435
|
+
this.#promise = undefined;
|
|
436
|
+
controller?.abort();
|
|
437
|
+
if (promise !== undefined) {
|
|
438
|
+
await Promise.allSettled([promise]);
|
|
219
439
|
}
|
|
220
440
|
}
|
|
221
441
|
}
|
|
222
442
|
exports.DynamicSchedulerLoop = DynamicSchedulerLoop;
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
// Scheduled workflows are always enqueued to their owner's latest application version
|
|
228
|
-
const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
|
|
229
|
-
const internalStatus = {
|
|
443
|
+
/** The ENQUEUED status row of one scheduled run. */
|
|
444
|
+
async function scheduledStatus(definition, scheduledDate, workflowID, applicationVersion, owner, serializer) {
|
|
445
|
+
const serparam = await (0, serialization_1.serializeArgs)([scheduledDate, definition.context], undefined, serializer, undefined);
|
|
446
|
+
return {
|
|
230
447
|
workflowUUID: workflowID,
|
|
231
448
|
status: workflow_1.StatusString.ENQUEUED,
|
|
232
|
-
workflowName:
|
|
233
|
-
workflowClassName:
|
|
449
|
+
workflowName: definition.workflowName,
|
|
450
|
+
workflowClassName: definition.workflowClassName,
|
|
234
451
|
workflowConfigName: '',
|
|
235
|
-
applicationVersion
|
|
236
|
-
queueName:
|
|
452
|
+
applicationVersion,
|
|
453
|
+
queueName: definition.queueName || utils_1.INTERNAL_QUEUE_NAME,
|
|
237
454
|
authenticatedUser: '',
|
|
238
455
|
output: null,
|
|
239
456
|
error: null,
|
|
@@ -246,49 +463,72 @@ async function enqueueScheduledWorkflow(systemDatabase, serializer, sched, workf
|
|
|
246
463
|
priority: 0,
|
|
247
464
|
queuePartitionKey: undefined,
|
|
248
465
|
serialization: serparam.serialization,
|
|
249
|
-
scheduleName:
|
|
466
|
+
scheduleName: definition.scheduleName,
|
|
250
467
|
// Owned where possible: the internal queue's shared name cannot route an unclaimed row.
|
|
251
468
|
applicationName: owner,
|
|
252
|
-
shardKey:
|
|
469
|
+
shardKey: definition.shardKey,
|
|
253
470
|
};
|
|
254
|
-
await systemDatabase.initWorkflowStatus(internalStatus, null);
|
|
255
|
-
return;
|
|
256
471
|
}
|
|
472
|
+
/** The schedule's owner routes its runs, whoever fires them; an unclaimed one falls back to the firing handle, which may have no identity itself. */
|
|
473
|
+
function scheduleOwner(systemDatabase, definition) {
|
|
474
|
+
return definition.applicationName ?? systemDatabase.appName;
|
|
475
|
+
}
|
|
476
|
+
/** Enqueue these scheduled runs in one batch, skipping existing ones. Returns their workflow IDs. */
|
|
477
|
+
async function enqueueTicks(systemDatabase, serializer, ticks, stampLastFired) {
|
|
478
|
+
// Scheduled workflows are always enqueued to their owner's latest application version.
|
|
479
|
+
const versions = new Map();
|
|
480
|
+
const statuses = [];
|
|
481
|
+
const firedAt = new Map();
|
|
482
|
+
for (const [definition, at] of ticks) {
|
|
483
|
+
const owner = scheduleOwner(systemDatabase, definition);
|
|
484
|
+
let version = versions.get(owner);
|
|
485
|
+
if (version === undefined) {
|
|
486
|
+
version = (await systemDatabase.getLatestApplicationVersion(owner)).versionName;
|
|
487
|
+
versions.set(owner, version);
|
|
488
|
+
}
|
|
489
|
+
const scheduledDate = new Date(at);
|
|
490
|
+
const workflowID = `sched-${definition.scheduleName}-${scheduledDate.toISOString()}`;
|
|
491
|
+
statuses.push(await scheduledStatus(definition, scheduledDate, workflowID, version, owner, serializer));
|
|
492
|
+
if (stampLastFired) {
|
|
493
|
+
firedAt.set(workflowID, scheduledDate.toISOString());
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
await systemDatabase.enqueueScheduledWorkflows(statuses, firedAt);
|
|
497
|
+
return statuses.map((status) => status.workflowUUID);
|
|
498
|
+
}
|
|
499
|
+
exports.enqueueTicks = enqueueTicks;
|
|
257
500
|
async function triggerSchedule(systemDatabase, serializer, name) {
|
|
258
501
|
const sched = await systemDatabase.getSchedule(name);
|
|
259
502
|
if (!sched) {
|
|
260
503
|
throw new error_1.DBOSError(`Schedule "${name}" not found`);
|
|
261
504
|
}
|
|
262
|
-
const
|
|
505
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
263
506
|
const now = new Date();
|
|
264
507
|
const workflowID = `sched-${name}-trigger-${now.toISOString()}`;
|
|
265
|
-
|
|
508
|
+
const owner = scheduleOwner(systemDatabase, definition);
|
|
509
|
+
const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
|
|
510
|
+
await systemDatabase.initWorkflowStatus(await scheduledStatus(definition, now, workflowID, latestVersion.versionName, owner, serializer), null);
|
|
266
511
|
return workflowID;
|
|
267
512
|
}
|
|
268
513
|
exports.triggerSchedule = triggerSchedule;
|
|
514
|
+
/** Enqueue every scheduled execution between `start` and `end`. Returns their workflow IDs. */
|
|
269
515
|
async function backfillSchedule(systemDatabase, serializer, name, start, end) {
|
|
270
516
|
const sched = await systemDatabase.getSchedule(name);
|
|
271
517
|
if (!sched) {
|
|
272
518
|
throw new error_1.DBOSError(`Schedule "${name}" not found`);
|
|
273
519
|
}
|
|
274
|
-
const
|
|
275
|
-
const
|
|
520
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
521
|
+
const matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
|
|
522
|
+
const batchSize = exports.schedulerConfig.fireBatchSize;
|
|
276
523
|
const workflowIDs = [];
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
current = next.getTime();
|
|
284
|
-
continue;
|
|
524
|
+
// Computed a batch at a time, so a long backfill yields to the event loop between batches.
|
|
525
|
+
let next = nextTick(matcher, start.getTime());
|
|
526
|
+
while (next < end.getTime()) {
|
|
527
|
+
const batch = [];
|
|
528
|
+
for (; next < end.getTime() && batch.length < batchSize; next = nextTick(matcher, next)) {
|
|
529
|
+
batch.push([definition, next]);
|
|
285
530
|
}
|
|
286
|
-
|
|
287
|
-
break;
|
|
288
|
-
const workflowID = `sched-${name}-${next.toISOString()}`;
|
|
289
|
-
await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, next, context);
|
|
290
|
-
workflowIDs.push(workflowID);
|
|
291
|
-
current = next.getTime();
|
|
531
|
+
workflowIDs.push(...(await enqueueTicks(systemDatabase, serializer, batch, false)));
|
|
292
532
|
}
|
|
293
533
|
return workflowIDs;
|
|
294
534
|
}
|