@dbos-inc/dbos-sdk 5.4.11-preview → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +7 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/client.d.ts +26 -2
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +65 -1
- package/dist/src/client.js.map +1 -1
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +3 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +26 -1
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +50 -7
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +74 -5
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +139 -22
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/debugpoint.d.ts +1 -0
- package/dist/src/debugpoint.d.ts.map +1 -1
- package/dist/src/debugpoint.js +4 -2
- package/dist/src/debugpoint.js.map +1 -1
- package/dist/src/enterprise.d.ts +56 -1
- package/dist/src/enterprise.d.ts.map +1 -1
- package/dist/src/enterprise.js +26 -4
- package/dist/src/enterprise.js.map +1 -1
- package/dist/src/error.d.ts +42 -0
- package/dist/src/error.d.ts.map +1 -1
- package/dist/src/error.js +85 -1
- package/dist/src/error.js.map +1 -1
- package/dist/src/external_storage.d.ts +27 -0
- package/dist/src/external_storage.d.ts.map +1 -0
- package/dist/src/external_storage.js +28 -0
- package/dist/src/external_storage.js.map +1 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts +78 -3
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +414 -173
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +51 -10
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +418 -207
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/utils.d.ts +2 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +9 -1
- package/dist/src/utils.js.map +1 -1
- package/dist/src/wfqueue.js +1 -1
- package/dist/src/wfqueue.js.map +1 -1
- package/dist/src/workflow.d.ts +3 -1
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/src/workflow.js +16 -3
- package/dist/src/workflow.js.map +1 -1
- package/dist/src/workflow_management.d.ts +16 -4
- package/dist/src/workflow_management.d.ts.map +1 -1
- package/dist/src/workflow_management.js +74 -32
- package/dist/src/workflow_management.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.scheduleShardKey = exports.backfillSchedule = exports.triggerSchedule = exports.DynamicSchedulerLoop = exports.scheduleSignature = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
|
|
3
|
+
exports.scheduleShardKey = exports.backfillSchedule = exports.triggerSchedule = exports.enqueueTicks = exports.DynamicSchedulerLoop = exports.schedulerLoop = exports.Scheduler = exports.ScheduleEntry = exports.scheduleDefinition = exports.schedulerConfig = exports.scheduleSignature = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
|
|
4
4
|
const serialization_1 = require("../serialization");
|
|
5
5
|
const crypto_1 = require("crypto");
|
|
6
|
-
const __1 = require("..");
|
|
7
6
|
const utils_1 = require("../utils");
|
|
8
7
|
const crontab_1 = require("./crontab");
|
|
9
|
-
const dbos_executor_1 = require("../dbos-executor");
|
|
10
8
|
const error_1 = require("../error");
|
|
11
9
|
const workflow_1 = require("../workflow");
|
|
12
10
|
const context_1 = require("../context");
|
|
@@ -33,7 +31,7 @@ function createScheduleId() {
|
|
|
33
31
|
return (0, crypto_1.randomUUID)();
|
|
34
32
|
}
|
|
35
33
|
exports.createScheduleId = createScheduleId;
|
|
36
|
-
// Definition fields
|
|
34
|
+
// Definition fields an armed schedule captures; a change re-arms it. Runtime state (schedule_id, status, last_fired_at) is excluded so an unchanged re-apply doesn't re-arm it.
|
|
37
35
|
function scheduleSignature(sched) {
|
|
38
36
|
return JSON.stringify([
|
|
39
37
|
sched.workflowName,
|
|
@@ -46,193 +44,413 @@ function scheduleSignature(sched) {
|
|
|
46
44
|
]);
|
|
47
45
|
}
|
|
48
46
|
exports.scheduleSignature = scheduleSignature;
|
|
47
|
+
exports.schedulerConfig = {
|
|
48
|
+
/** Most scheduled runs enqueued in one transaction. */
|
|
49
|
+
fireBatchSize: 1000,
|
|
50
|
+
/** How long a standby keeps a tick no poll has shown fired, or three polling intervals if longer. */
|
|
51
|
+
bufferMinMs: 3_600_000,
|
|
52
|
+
/** How far back a starting scheduler arms each schedule, so a tick missed while its leader died still fires. */
|
|
53
|
+
startupLookbackMs: 30_000,
|
|
54
|
+
/** How often a scheduler that isn't firing checks whether it has taken over. */
|
|
55
|
+
standbyCheckMs: 1000,
|
|
56
|
+
};
|
|
49
57
|
// During the first minute after startup, poll for schedules every second so schedules
|
|
50
58
|
// registered around launch are picked up promptly rather than after a full polling interval.
|
|
51
59
|
const STARTUP_FAST_POLL_DURATION_MS = 60_000;
|
|
52
60
|
const STARTUP_FAST_POLL_INTERVAL_MS = 1_000;
|
|
53
|
-
// Timers can't sleep past ~24.8 days, so wait for distant
|
|
61
|
+
// Timers can't sleep past ~24.8 days, so wait for distant ticks in bounded chunks.
|
|
54
62
|
const MAX_SCHEDULE_SLEEP_MS = 60 * 60 * 1000;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
+
async function scheduleDefinition(sched, serializer) {
|
|
64
|
+
return {
|
|
65
|
+
scheduleName: sched.scheduleName,
|
|
66
|
+
workflowName: sched.workflowName,
|
|
67
|
+
workflowClassName: sched.workflowClassName,
|
|
68
|
+
context: await serializer.parse(sched.context),
|
|
69
|
+
queueName: sched.queueName ?? undefined,
|
|
70
|
+
applicationName: sched.applicationName,
|
|
71
|
+
shardKey: sched.shardKey,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
exports.scheduleDefinition = scheduleDefinition;
|
|
75
|
+
/** The first tick strictly after `after`; throws if the schedule can never fire again. */
|
|
76
|
+
function nextTick(matcher, after) {
|
|
77
|
+
const next = matcher.nextWakeupTime(after).getTime();
|
|
78
|
+
// TimeMatcher returns a non-matching instant only past its search horizon.
|
|
79
|
+
if (!matcher.match(next)) {
|
|
80
|
+
throw new error_1.DBOSError(`The schedule never fires after ${new Date(after).toISOString()}`);
|
|
63
81
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
82
|
+
return next;
|
|
83
|
+
}
|
|
84
|
+
/** An active schedule in the scheduler and its next tick. */
|
|
85
|
+
class ScheduleEntry {
|
|
86
|
+
definition;
|
|
87
|
+
matcher;
|
|
88
|
+
scheduleId;
|
|
89
|
+
// Definition snapshot; the scheduler re-arms the entry when it changes.
|
|
90
|
+
signature;
|
|
91
|
+
nextTime;
|
|
92
|
+
constructor(sched, definition, matcher, after) {
|
|
93
|
+
this.definition = definition;
|
|
94
|
+
this.matcher = matcher;
|
|
95
|
+
this.scheduleId = sched.scheduleId;
|
|
96
|
+
this.signature = scheduleSignature(sched);
|
|
97
|
+
this.nextTime = nextTick(matcher, after);
|
|
68
98
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
99
|
+
/** Arm `sched` at its first tick after `after`, by default now. Throws if it can't run. */
|
|
100
|
+
static async create(sched, matcher, serializer, after) {
|
|
101
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
102
|
+
return new ScheduleEntry(sched, definition, matcher, after ?? Date.now());
|
|
103
|
+
}
|
|
104
|
+
advance() {
|
|
105
|
+
this.nextTime = nextTick(this.matcher, this.nextTime);
|
|
106
|
+
}
|
|
107
|
+
advancePast(lastFired) {
|
|
108
|
+
while (this.nextTime <= lastFired) {
|
|
109
|
+
this.advance();
|
|
74
110
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
exports.ScheduleEntry = ScheduleEntry;
|
|
114
|
+
/** A min-heap of entries by next tick, ties in push order. */
|
|
115
|
+
class TickHeap {
|
|
116
|
+
#items = [];
|
|
117
|
+
#seq = 0;
|
|
118
|
+
get size() {
|
|
119
|
+
return this.#items.length;
|
|
120
|
+
}
|
|
121
|
+
/** The earliest tick, or Infinity if empty. */
|
|
122
|
+
peek() {
|
|
123
|
+
return this.#items[0]?.at ?? Infinity;
|
|
124
|
+
}
|
|
125
|
+
push(entry) {
|
|
126
|
+
const items = this.#items;
|
|
127
|
+
items.push({ at: entry.nextTime, seq: this.#seq++, entry });
|
|
128
|
+
let i = items.length - 1;
|
|
129
|
+
while (i > 0) {
|
|
130
|
+
const parent = (i - 1) >> 1;
|
|
131
|
+
if (!this.#before(i, parent))
|
|
132
|
+
break;
|
|
133
|
+
[items[i], items[parent]] = [items[parent], items[i]];
|
|
134
|
+
i = parent;
|
|
78
135
|
}
|
|
79
|
-
|
|
80
|
-
|
|
136
|
+
}
|
|
137
|
+
pop() {
|
|
138
|
+
const items = this.#items;
|
|
139
|
+
const top = items[0];
|
|
140
|
+
const last = items.pop();
|
|
141
|
+
if (items.length > 0) {
|
|
142
|
+
items[0] = last;
|
|
143
|
+
let i = 0;
|
|
144
|
+
while (true) {
|
|
145
|
+
const left = 2 * i + 1;
|
|
146
|
+
const right = left + 1;
|
|
147
|
+
let least = i;
|
|
148
|
+
if (left < items.length && this.#before(left, least))
|
|
149
|
+
least = left;
|
|
150
|
+
if (right < items.length && this.#before(right, least))
|
|
151
|
+
least = right;
|
|
152
|
+
if (least === i)
|
|
153
|
+
break;
|
|
154
|
+
[items[i], items[least]] = [items[least], items[i]];
|
|
155
|
+
i = least;
|
|
156
|
+
}
|
|
81
157
|
}
|
|
82
|
-
|
|
83
|
-
this.#scheduleLoops.clear();
|
|
158
|
+
return top.entry;
|
|
84
159
|
}
|
|
85
|
-
|
|
86
|
-
const
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
160
|
+
#before(a, b) {
|
|
161
|
+
const x = this.#items[a];
|
|
162
|
+
const y = this.#items[b];
|
|
163
|
+
return x.at < y.at || (x.at === y.at && x.seq < y.seq);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/** Fires every active schedule from one loop, enqueueing each tick's runs in batches. */
|
|
167
|
+
class Scheduler {
|
|
168
|
+
systemDatabase;
|
|
169
|
+
serializer;
|
|
170
|
+
logger;
|
|
171
|
+
// Active schedules keyed by schedule ID.
|
|
172
|
+
entries = new Map();
|
|
173
|
+
// One item per entry in `entries`; rebuilt by every poll.
|
|
174
|
+
heap = new TickHeap();
|
|
175
|
+
// Whether a poll has armed the schedules this scheduler started with.
|
|
176
|
+
#started = false;
|
|
177
|
+
constructor(systemDatabase, serializer, logger) {
|
|
178
|
+
this.systemDatabase = systemDatabase;
|
|
179
|
+
this.serializer = serializer;
|
|
180
|
+
this.logger = logger;
|
|
181
|
+
}
|
|
182
|
+
/** Sync the armed schedules with the database, then rebuild the heap from them. */
|
|
183
|
+
async poll() {
|
|
184
|
+
try {
|
|
185
|
+
await this.#reconcile();
|
|
186
|
+
}
|
|
187
|
+
finally {
|
|
188
|
+
// Even after a partial reconcile, so the heap never holds dropped entries or misses one.
|
|
189
|
+
this.heap = new TickHeap();
|
|
190
|
+
for (const entry of this.entries.values()) {
|
|
191
|
+
this.heap.push(entry);
|
|
97
192
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
async #reconcile() {
|
|
196
|
+
const schedules = await this.systemDatabase.listSchedules({ applicationName: this.systemDatabase.appName });
|
|
197
|
+
const currentIds = new Set(schedules.map((s) => s.scheduleId));
|
|
198
|
+
for (const scheduleId of this.entries.keys()) {
|
|
199
|
+
if (!currentIds.has(scheduleId)) {
|
|
200
|
+
this.entries.delete(scheduleId);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
for (const sched of schedules) {
|
|
204
|
+
const entry = this.entries.get(sched.scheduleId);
|
|
205
|
+
if (sched.status !== 'ACTIVE') {
|
|
206
|
+
// Paused: drop it; resuming re-arms it, backfilling if enabled.
|
|
207
|
+
this.entries.delete(sched.scheduleId);
|
|
101
208
|
continue;
|
|
102
209
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
entry.controller.abort();
|
|
109
|
-
this.#scheduleLoops.delete(name);
|
|
210
|
+
if (entry !== undefined) {
|
|
211
|
+
// Running: re-arm on a changed definition, without backfill.
|
|
212
|
+
if (entry.signature !== scheduleSignature(sched)) {
|
|
213
|
+
this.entries.delete(sched.scheduleId);
|
|
214
|
+
await this.#arm(sched);
|
|
110
215
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
for (const sched of schedules) {
|
|
114
|
-
const existing = this.#scheduleLoops.get(sched.scheduleName);
|
|
115
|
-
const signature = scheduleSignature(sched);
|
|
116
|
-
if (sched.status === 'PAUSED' && existing) {
|
|
117
|
-
// Paused but has a running loop — stop it
|
|
118
|
-
existing.controller.abort();
|
|
119
|
-
this.#scheduleLoops.delete(sched.scheduleName);
|
|
216
|
+
else {
|
|
217
|
+
this.#skipFired(entry, sched);
|
|
120
218
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const now = new Date();
|
|
134
|
-
if (lastFired < now) {
|
|
135
|
-
const executor = dbos_executor_1.DBOSExecutor.globalInstance;
|
|
136
|
-
await backfillSchedule(executor.systemDatabase, executor.serializer, sched.scheduleName, lastFired, now);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
catch (e) {
|
|
140
|
-
__1.DBOS.logger.warn(`Dynamic scheduler: error backfilling schedule "${sched.scheduleName}": ${e.message}`);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
// An abandoned backfill means shutdown began; a loop started now would outlive destroy().
|
|
144
|
-
if (signal.aborted)
|
|
145
|
-
return;
|
|
146
|
-
// Active and no running loop — start one
|
|
147
|
-
const controller = new AbortController();
|
|
148
|
-
const executor = dbos_executor_1.DBOSExecutor.globalInstance;
|
|
149
|
-
// Only enqueues, never runs the workflow, so its retries may be abandoned when the loop stops.
|
|
150
|
-
const promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => DynamicSchedulerLoop.#scheduleLoop(sched.scheduleName, sched.workflowName, sched.workflowClassName, sched.schedule, sched.context, executor.serializer, controller.signal, sched.cronTimezone ?? undefined, sched.queueName ?? undefined, sched.applicationName, sched.shardKey));
|
|
151
|
-
this.#scheduleLoops.set(sched.scheduleName, { controller, promise, signature });
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
// Not running: arm it, then backfill missed executions if enabled; a broken one is retried next poll.
|
|
222
|
+
const armed = await this.#arm(sched, this.#started ? undefined : startupAfter(sched));
|
|
223
|
+
if (armed === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
if (sched.automaticBackfill && sched.lastFiredAt) {
|
|
226
|
+
try {
|
|
227
|
+
const lastFired = new Date(sched.lastFiredAt);
|
|
228
|
+
const now = new Date();
|
|
229
|
+
if (lastFired < now) {
|
|
230
|
+
await backfillSchedule(this.systemDatabase, this.serializer, sched.scheduleName, lastFired, now);
|
|
152
231
|
}
|
|
153
232
|
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
this.logger.warn(`Exception during automatic backfill for schedule '${sched.scheduleName}': ${errorText(e)}`);
|
|
235
|
+
}
|
|
154
236
|
}
|
|
155
|
-
|
|
237
|
+
}
|
|
238
|
+
this.#started = true;
|
|
239
|
+
}
|
|
240
|
+
/** Pass over the ticks this schedule's last_fired_at shows some process enqueued. */
|
|
241
|
+
#skipFired(entry, sched) {
|
|
242
|
+
if (!sched.lastFiredAt)
|
|
243
|
+
return;
|
|
244
|
+
const lastFired = Date.parse(sched.lastFiredAt);
|
|
245
|
+
// Unreadable: the entry keeps its ticks, refired at worst.
|
|
246
|
+
if (Number.isNaN(lastFired))
|
|
247
|
+
return;
|
|
248
|
+
try {
|
|
249
|
+
entry.advancePast(lastFired);
|
|
250
|
+
}
|
|
251
|
+
catch (e) {
|
|
252
|
+
this.#drop(entry, e);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/** Register a new entry for this schedule, or log why it can't run and return undefined. */
|
|
256
|
+
async #arm(sched, after) {
|
|
257
|
+
let matcher;
|
|
258
|
+
try {
|
|
259
|
+
matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
|
|
260
|
+
}
|
|
261
|
+
catch (e) {
|
|
262
|
+
this.logger.error(`Cannot run schedule '${sched.scheduleName}'. Invalid crontab "${sched.schedule}": ${errorText(e)}`);
|
|
263
|
+
return undefined;
|
|
264
|
+
}
|
|
265
|
+
let entry;
|
|
266
|
+
try {
|
|
267
|
+
entry = await ScheduleEntry.create(sched, matcher, this.serializer, after);
|
|
268
|
+
}
|
|
269
|
+
catch (e) {
|
|
270
|
+
this.logger.error(`Cannot run schedule '${sched.scheduleName}': ${errorText(e)}`);
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
this.entries.set(entry.scheduleId, entry);
|
|
274
|
+
return entry;
|
|
275
|
+
}
|
|
276
|
+
/** Milliseconds until the earliest tick is `lag` milliseconds old, Infinity if none. */
|
|
277
|
+
msUntilDue(lag = 0) {
|
|
278
|
+
return this.heap.peek() + lag - Date.now();
|
|
279
|
+
}
|
|
280
|
+
/** Fire every due tick, batch by batch, until `signal` aborts. */
|
|
281
|
+
async fireDue(signal) {
|
|
282
|
+
const now = Date.now();
|
|
283
|
+
const due = [];
|
|
284
|
+
while (this.heap.size > 0 && this.heap.peek() <= now) {
|
|
285
|
+
due.push(this.heap.pop());
|
|
286
|
+
}
|
|
287
|
+
const batchSize = exports.schedulerConfig.fireBatchSize;
|
|
288
|
+
let fired = 0;
|
|
289
|
+
while (fired < due.length && !signal.aborted) {
|
|
290
|
+
await this.#fire(due.slice(fired, fired + batchSize), signal);
|
|
291
|
+
fired += batchSize;
|
|
292
|
+
}
|
|
293
|
+
for (const entry of due.slice(0, fired)) {
|
|
294
|
+
this.#advance(entry);
|
|
295
|
+
}
|
|
296
|
+
// Left unstamped, so the process that takes over fires them.
|
|
297
|
+
for (const entry of due.slice(fired)) {
|
|
298
|
+
this.heap.push(entry);
|
|
156
299
|
}
|
|
157
300
|
}
|
|
158
|
-
|
|
159
|
-
|
|
301
|
+
/** Pass over ticks older than `before` without firing them. */
|
|
302
|
+
skipDue(before) {
|
|
303
|
+
while (this.heap.size > 0 && this.heap.peek() <= before) {
|
|
304
|
+
this.#advance(this.heap.pop());
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
#advance(entry) {
|
|
308
|
+
// A late tick's successor may already be due; it fires on the next pass.
|
|
160
309
|
try {
|
|
161
|
-
|
|
310
|
+
entry.advance();
|
|
162
311
|
}
|
|
163
312
|
catch (e) {
|
|
164
|
-
|
|
313
|
+
this.#drop(entry, e);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
this.heap.push(entry);
|
|
317
|
+
}
|
|
318
|
+
#drop(entry, e) {
|
|
319
|
+
// Dropped; the next poll tries to arm it again.
|
|
320
|
+
this.entries.delete(entry.scheduleId);
|
|
321
|
+
this.logger.error(`Cannot run schedule '${entry.definition.scheduleName}': ${errorText(e)}`);
|
|
322
|
+
}
|
|
323
|
+
async #fire(batch, signal) {
|
|
324
|
+
try {
|
|
325
|
+
await enqueueTicks(this.systemDatabase, this.serializer, batch.map((entry) => [entry.definition, entry.nextTime]), true);
|
|
165
326
|
return;
|
|
166
327
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
workflowClassName,
|
|
172
|
-
schedule: cronExpression,
|
|
173
|
-
status: 'ACTIVE',
|
|
174
|
-
context: serializedContext,
|
|
175
|
-
lastFiredAt: null,
|
|
176
|
-
automaticBackfill: false,
|
|
177
|
-
cronTimezone: cronTimezone ?? null,
|
|
178
|
-
queueName: queueName ?? null,
|
|
179
|
-
applicationName,
|
|
180
|
-
shardKey,
|
|
181
|
-
};
|
|
182
|
-
let lastExec = new Date().setMilliseconds(0);
|
|
183
|
-
while (!signal.aborted) {
|
|
184
|
-
const nextExec = timeMatcher.nextWakeupTime(lastExec).getTime();
|
|
185
|
-
const untilNext = nextExec - Date.now();
|
|
186
|
-
// Apply jitter to prevent thundering herd
|
|
187
|
-
const wakeTime = untilNext > 0 ? nextExec + Math.random() * Math.min(untilNext / 10, 10000) : nextExec;
|
|
188
|
-
while (!signal.aborted && Date.now() < wakeTime) {
|
|
189
|
-
await (0, utils_1.interruptibleSleep)(Math.min(wakeTime - Date.now(), MAX_SCHEDULE_SLEEP_MS), signal);
|
|
328
|
+
catch (e) {
|
|
329
|
+
if (batch.length === 1) {
|
|
330
|
+
this.logger.warn(`Exception in schedule '${batch[0].definition.scheduleName}': ${errorText(e)}`);
|
|
331
|
+
return;
|
|
190
332
|
}
|
|
333
|
+
// A stopping loop's retries are cut short, so retrying each schedule would only log a warning per schedule.
|
|
191
334
|
if (signal.aborted) {
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
// If TimeMatcher did not find the next occurrence of the schedule yet,
|
|
195
|
-
// we must have it determine the next wakeup time.
|
|
196
|
-
if (!timeMatcher.match(nextExec)) {
|
|
197
|
-
lastExec = nextExec;
|
|
198
|
-
continue;
|
|
335
|
+
this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows: ${errorText(e)}`);
|
|
336
|
+
return;
|
|
199
337
|
}
|
|
200
|
-
|
|
201
|
-
|
|
338
|
+
// Logged here, as the per-schedule retries may succeed and never show it.
|
|
339
|
+
this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows, retrying each schedule alone: ${errorText(e)}`);
|
|
340
|
+
}
|
|
341
|
+
// One failing schedule must not hold back the rest of its batch.
|
|
342
|
+
for (const entry of batch) {
|
|
343
|
+
if (signal.aborted)
|
|
344
|
+
return;
|
|
345
|
+
await this.#fire([entry], signal);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
exports.Scheduler = Scheduler;
|
|
350
|
+
/** Where a starting scheduler arms a schedule: past its last fired tick, back at most the startup look-back. Later arms (creates, resumes) start from now. */
|
|
351
|
+
function startupAfter(sched) {
|
|
352
|
+
if (!sched.lastFiredAt)
|
|
353
|
+
return undefined;
|
|
354
|
+
const lastFired = Date.parse(sched.lastFiredAt);
|
|
355
|
+
if (Number.isNaN(lastFired))
|
|
356
|
+
return undefined;
|
|
357
|
+
return Math.max(lastFired, Date.now() - exports.schedulerConfig.startupLookbackMs);
|
|
358
|
+
}
|
|
359
|
+
function errorText(e) {
|
|
360
|
+
return e instanceof Error ? (e.stack ?? e.message) : String(e);
|
|
361
|
+
}
|
|
362
|
+
/** Poll schedules until aborted, firing only while `leading()`; a standby keeps each tick until a poll shows it fired, so a taker fires all it missed. */
|
|
363
|
+
async function schedulerLoop(signal, scheduler, pollingIntervalMs, leading) {
|
|
364
|
+
const bufferMs = Math.max(exports.schedulerConfig.bufferMinMs, 3 * pollingIntervalMs);
|
|
365
|
+
const startupDeadline = performance.now() + STARTUP_FAST_POLL_DURATION_MS;
|
|
366
|
+
const pollTimeout = () => performance.now() < startupDeadline
|
|
367
|
+
? Math.min(STARTUP_FAST_POLL_INTERVAL_MS, pollingIntervalMs)
|
|
368
|
+
: pollingIntervalMs;
|
|
369
|
+
let nextPoll = performance.now();
|
|
370
|
+
while (!signal.aborted) {
|
|
371
|
+
if (performance.now() >= nextPoll) {
|
|
202
372
|
try {
|
|
203
|
-
|
|
204
|
-
const existing = await __1.DBOS.getWorkflowStatus(workflowID, { shardKey });
|
|
205
|
-
if (existing) {
|
|
206
|
-
lastExec = nextExec;
|
|
207
|
-
continue;
|
|
208
|
-
}
|
|
209
|
-
const context = await serializer.parse(serializedContext);
|
|
210
|
-
const systemDatabase = dbos_executor_1.DBOSExecutor.globalInstance.systemDatabase;
|
|
211
|
-
await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, date, context);
|
|
212
|
-
await systemDatabase.updateLastFiredAt(scheduleName, date.toISOString());
|
|
373
|
+
await scheduler.poll();
|
|
213
374
|
}
|
|
214
375
|
catch (e) {
|
|
215
|
-
|
|
376
|
+
scheduler.logger.warn(`Exception polling schedules: ${errorText(e)}`);
|
|
216
377
|
}
|
|
217
|
-
|
|
378
|
+
nextPoll = performance.now() + pollTimeout();
|
|
379
|
+
}
|
|
380
|
+
// Checked once per pass: a holder losing the lock mid-pass still fires all it took.
|
|
381
|
+
const firing = leading();
|
|
382
|
+
try {
|
|
383
|
+
if (firing) {
|
|
384
|
+
await scheduler.fireDue(signal);
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
scheduler.skipDue(Date.now() - bufferMs);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
scheduler.logger.warn(`Exception firing schedules: ${errorText(e)}`);
|
|
392
|
+
}
|
|
393
|
+
const untilPoll = nextPoll - performance.now();
|
|
394
|
+
const wait = firing
|
|
395
|
+
? Math.min(untilPoll, scheduler.msUntilDue())
|
|
396
|
+
: Math.min(untilPoll, scheduler.msUntilDue(bufferMs), exports.schedulerConfig.standbyCheckMs);
|
|
397
|
+
// Always yields, so a pass that finds more due work never starves the event loop.
|
|
398
|
+
await (0, utils_1.interruptibleSleep)(Math.min(Math.max(wait, 0), MAX_SCHEDULE_SLEEP_MS), signal);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
exports.schedulerLoop = schedulerLoop;
|
|
402
|
+
/** Runs the scheduler loop from `initialize` until `destroy`, firing while `leading()`. */
|
|
403
|
+
class DynamicSchedulerLoop {
|
|
404
|
+
systemDatabase;
|
|
405
|
+
serializer;
|
|
406
|
+
logger;
|
|
407
|
+
leading;
|
|
408
|
+
#controller;
|
|
409
|
+
#promise;
|
|
410
|
+
#pollingIntervalMs;
|
|
411
|
+
constructor(systemDatabase, serializer, logger, pollingIntervalMs, leading) {
|
|
412
|
+
this.systemDatabase = systemDatabase;
|
|
413
|
+
this.serializer = serializer;
|
|
414
|
+
this.logger = logger;
|
|
415
|
+
this.leading = leading;
|
|
416
|
+
this.#pollingIntervalMs = pollingIntervalMs ?? 30000;
|
|
417
|
+
}
|
|
418
|
+
/** Start the loop; a no-op while it runs. */
|
|
419
|
+
async initialize() {
|
|
420
|
+
if (this.#controller !== undefined)
|
|
421
|
+
return;
|
|
422
|
+
const controller = new AbortController();
|
|
423
|
+
this.#controller = controller;
|
|
424
|
+
const scheduler = new Scheduler(this.systemDatabase, this.serializer, this.logger);
|
|
425
|
+
// Only enqueues, never runs a workflow, so its retries may be abandoned when the loop stops.
|
|
426
|
+
this.#promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => schedulerLoop(controller.signal, scheduler, this.#pollingIntervalMs, this.leading));
|
|
427
|
+
await Promise.resolve();
|
|
428
|
+
}
|
|
429
|
+
/** Stop the loop and wait for it to end; a no-op if it isn't running. */
|
|
430
|
+
async destroy() {
|
|
431
|
+
const controller = this.#controller;
|
|
432
|
+
const promise = this.#promise;
|
|
433
|
+
// Cleared first, so an initialize while this one winds down starts a fresh loop.
|
|
434
|
+
this.#controller = undefined;
|
|
435
|
+
this.#promise = undefined;
|
|
436
|
+
controller?.abort();
|
|
437
|
+
if (promise !== undefined) {
|
|
438
|
+
await Promise.allSettled([promise]);
|
|
218
439
|
}
|
|
219
440
|
}
|
|
220
441
|
}
|
|
221
442
|
exports.DynamicSchedulerLoop = DynamicSchedulerLoop;
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
// Scheduled workflows are always enqueued to their owner's latest application version
|
|
227
|
-
const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
|
|
228
|
-
const internalStatus = {
|
|
443
|
+
/** The ENQUEUED status row of one scheduled run. */
|
|
444
|
+
async function scheduledStatus(definition, scheduledDate, workflowID, applicationVersion, owner, serializer) {
|
|
445
|
+
const serparam = await (0, serialization_1.serializeArgs)([scheduledDate, definition.context], undefined, serializer, undefined);
|
|
446
|
+
return {
|
|
229
447
|
workflowUUID: workflowID,
|
|
230
448
|
status: workflow_1.StatusString.ENQUEUED,
|
|
231
|
-
workflowName:
|
|
232
|
-
workflowClassName:
|
|
449
|
+
workflowName: definition.workflowName,
|
|
450
|
+
workflowClassName: definition.workflowClassName,
|
|
233
451
|
workflowConfigName: '',
|
|
234
|
-
applicationVersion
|
|
235
|
-
queueName:
|
|
452
|
+
applicationVersion,
|
|
453
|
+
queueName: definition.queueName || utils_1.INTERNAL_QUEUE_NAME,
|
|
236
454
|
authenticatedUser: '',
|
|
237
455
|
output: null,
|
|
238
456
|
error: null,
|
|
@@ -245,49 +463,72 @@ async function enqueueScheduledWorkflow(systemDatabase, serializer, sched, workf
|
|
|
245
463
|
priority: 0,
|
|
246
464
|
queuePartitionKey: undefined,
|
|
247
465
|
serialization: serparam.serialization,
|
|
248
|
-
scheduleName:
|
|
466
|
+
scheduleName: definition.scheduleName,
|
|
249
467
|
// Owned where possible: the internal queue's shared name cannot route an unclaimed row.
|
|
250
468
|
applicationName: owner,
|
|
251
|
-
shardKey:
|
|
469
|
+
shardKey: definition.shardKey,
|
|
252
470
|
};
|
|
253
|
-
await systemDatabase.initWorkflowStatus(internalStatus, null);
|
|
254
|
-
return;
|
|
255
471
|
}
|
|
472
|
+
/** The schedule's owner routes its runs, whoever fires them; an unclaimed one falls back to the firing handle, which may have no identity itself. */
|
|
473
|
+
function scheduleOwner(systemDatabase, definition) {
|
|
474
|
+
return definition.applicationName ?? systemDatabase.appName;
|
|
475
|
+
}
|
|
476
|
+
/** Enqueue these scheduled runs in one batch, skipping existing ones. Returns their workflow IDs. */
|
|
477
|
+
async function enqueueTicks(systemDatabase, serializer, ticks, stampLastFired) {
|
|
478
|
+
// Scheduled workflows are always enqueued to their owner's latest application version.
|
|
479
|
+
const versions = new Map();
|
|
480
|
+
const statuses = [];
|
|
481
|
+
const firedAt = new Map();
|
|
482
|
+
for (const [definition, at] of ticks) {
|
|
483
|
+
const owner = scheduleOwner(systemDatabase, definition);
|
|
484
|
+
let version = versions.get(owner);
|
|
485
|
+
if (version === undefined) {
|
|
486
|
+
version = (await systemDatabase.getLatestApplicationVersion(owner)).versionName;
|
|
487
|
+
versions.set(owner, version);
|
|
488
|
+
}
|
|
489
|
+
const scheduledDate = new Date(at);
|
|
490
|
+
const workflowID = `sched-${definition.scheduleName}-${scheduledDate.toISOString()}`;
|
|
491
|
+
statuses.push(await scheduledStatus(definition, scheduledDate, workflowID, version, owner, serializer));
|
|
492
|
+
if (stampLastFired) {
|
|
493
|
+
firedAt.set(workflowID, scheduledDate.toISOString());
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
await systemDatabase.enqueueScheduledWorkflows(statuses, firedAt);
|
|
497
|
+
return statuses.map((status) => status.workflowUUID);
|
|
498
|
+
}
|
|
499
|
+
exports.enqueueTicks = enqueueTicks;
|
|
256
500
|
async function triggerSchedule(systemDatabase, serializer, name) {
|
|
257
501
|
const sched = await systemDatabase.getSchedule(name);
|
|
258
502
|
if (!sched) {
|
|
259
503
|
throw new error_1.DBOSError(`Schedule "${name}" not found`);
|
|
260
504
|
}
|
|
261
|
-
const
|
|
505
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
262
506
|
const now = new Date();
|
|
263
507
|
const workflowID = `sched-${name}-trigger-${now.toISOString()}`;
|
|
264
|
-
|
|
508
|
+
const owner = scheduleOwner(systemDatabase, definition);
|
|
509
|
+
const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
|
|
510
|
+
await systemDatabase.initWorkflowStatus(await scheduledStatus(definition, now, workflowID, latestVersion.versionName, owner, serializer), null);
|
|
265
511
|
return workflowID;
|
|
266
512
|
}
|
|
267
513
|
exports.triggerSchedule = triggerSchedule;
|
|
514
|
+
/** Enqueue every scheduled execution between `start` and `end`. Returns their workflow IDs. */
|
|
268
515
|
async function backfillSchedule(systemDatabase, serializer, name, start, end) {
|
|
269
516
|
const sched = await systemDatabase.getSchedule(name);
|
|
270
517
|
if (!sched) {
|
|
271
518
|
throw new error_1.DBOSError(`Schedule "${name}" not found`);
|
|
272
519
|
}
|
|
273
|
-
const
|
|
274
|
-
const
|
|
520
|
+
const definition = await scheduleDefinition(sched, serializer);
|
|
521
|
+
const matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
|
|
522
|
+
const batchSize = exports.schedulerConfig.fireBatchSize;
|
|
275
523
|
const workflowIDs = [];
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
current = next.getTime();
|
|
283
|
-
continue;
|
|
524
|
+
// Computed a batch at a time, so a long backfill yields to the event loop between batches.
|
|
525
|
+
let next = nextTick(matcher, start.getTime());
|
|
526
|
+
while (next < end.getTime()) {
|
|
527
|
+
const batch = [];
|
|
528
|
+
for (; next < end.getTime() && batch.length < batchSize; next = nextTick(matcher, next)) {
|
|
529
|
+
batch.push([definition, next]);
|
|
284
530
|
}
|
|
285
|
-
|
|
286
|
-
break;
|
|
287
|
-
const workflowID = `sched-${name}-${next.toISOString()}`;
|
|
288
|
-
await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, next, context);
|
|
289
|
-
workflowIDs.push(workflowID);
|
|
290
|
-
current = next.getTime();
|
|
531
|
+
workflowIDs.push(...(await enqueueTicks(systemDatabase, serializer, batch, false)));
|
|
291
532
|
}
|
|
292
533
|
return workflowIDs;
|
|
293
534
|
}
|