@dbos-inc/dbos-sdk 5.4.10-preview → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/dist/schemas/system_db_schema.d.ts +7 -0
  2. package/dist/schemas/system_db_schema.d.ts.map +1 -1
  3. package/dist/schemas/system_db_schema.js.map +1 -1
  4. package/dist/src/cli/cli.js +56 -13
  5. package/dist/src/cli/cli.js.map +1 -1
  6. package/dist/src/client.d.ts +44 -10
  7. package/dist/src/client.d.ts.map +1 -1
  8. package/dist/src/client.js +140 -44
  9. package/dist/src/client.js.map +1 -1
  10. package/dist/src/config.d.ts +1 -0
  11. package/dist/src/config.d.ts.map +1 -1
  12. package/dist/src/config.js +4 -0
  13. package/dist/src/config.js.map +1 -1
  14. package/dist/src/context.d.ts +6 -0
  15. package/dist/src/context.d.ts.map +1 -1
  16. package/dist/src/context.js +15 -1
  17. package/dist/src/context.js.map +1 -1
  18. package/dist/src/datasource.d.ts +2 -0
  19. package/dist/src/datasource.d.ts.map +1 -1
  20. package/dist/src/datasource.js +7 -1
  21. package/dist/src/datasource.js.map +1 -1
  22. package/dist/src/dbos-executor.d.ts +32 -4
  23. package/dist/src/dbos-executor.d.ts.map +1 -1
  24. package/dist/src/dbos-executor.js +102 -33
  25. package/dist/src/dbos-executor.js.map +1 -1
  26. package/dist/src/dbos.d.ts +104 -15
  27. package/dist/src/dbos.d.ts.map +1 -1
  28. package/dist/src/dbos.js +275 -86
  29. package/dist/src/dbos.js.map +1 -1
  30. package/dist/src/debugpoint.d.ts +1 -0
  31. package/dist/src/debugpoint.d.ts.map +1 -1
  32. package/dist/src/debugpoint.js +4 -2
  33. package/dist/src/debugpoint.js.map +1 -1
  34. package/dist/src/enqueue_options.d.ts +2 -0
  35. package/dist/src/enqueue_options.d.ts.map +1 -1
  36. package/dist/src/enqueue_options.js +3 -0
  37. package/dist/src/enqueue_options.js.map +1 -1
  38. package/dist/src/enqueue_workflow.d.ts.map +1 -1
  39. package/dist/src/enqueue_workflow.js +11 -5
  40. package/dist/src/enqueue_workflow.js.map +1 -1
  41. package/dist/src/enterprise.d.ts +78 -3
  42. package/dist/src/enterprise.d.ts.map +1 -1
  43. package/dist/src/enterprise.js +104 -45
  44. package/dist/src/enterprise.js.map +1 -1
  45. package/dist/src/error.d.ts +42 -0
  46. package/dist/src/error.d.ts.map +1 -1
  47. package/dist/src/error.js +85 -1
  48. package/dist/src/error.js.map +1 -1
  49. package/dist/src/eventreceiver.d.ts +2 -0
  50. package/dist/src/eventreceiver.d.ts.map +1 -1
  51. package/dist/src/eventreceiver.js +7 -1
  52. package/dist/src/eventreceiver.js.map +1 -1
  53. package/dist/src/external_storage.d.ts +27 -0
  54. package/dist/src/external_storage.d.ts.map +1 -0
  55. package/dist/src/external_storage.js +28 -0
  56. package/dist/src/external_storage.js.map +1 -0
  57. package/dist/src/index.d.ts +4 -3
  58. package/dist/src/index.d.ts.map +1 -1
  59. package/dist/src/index.js +2 -1
  60. package/dist/src/index.js.map +1 -1
  61. package/dist/src/scheduler/scheduler.d.ts +83 -3
  62. package/dist/src/scheduler/scheduler.d.ts.map +1 -1
  63. package/dist/src/scheduler/scheduler.js +422 -171
  64. package/dist/src/scheduler/scheduler.js.map +1 -1
  65. package/dist/src/shard_keys.d.ts +13 -0
  66. package/dist/src/shard_keys.d.ts.map +1 -0
  67. package/dist/src/shard_keys.js +55 -0
  68. package/dist/src/shard_keys.js.map +1 -0
  69. package/dist/src/streams.d.ts +2 -0
  70. package/dist/src/streams.d.ts.map +1 -1
  71. package/dist/src/streams.js +5 -2
  72. package/dist/src/streams.js.map +1 -1
  73. package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
  74. package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
  75. package/dist/src/system_database.d.ts +185 -10
  76. package/dist/src/system_database.d.ts.map +1 -1
  77. package/dist/src/system_database.js +1429 -626
  78. package/dist/src/system_database.js.map +1 -1
  79. package/dist/src/utils.d.ts +2 -0
  80. package/dist/src/utils.d.ts.map +1 -1
  81. package/dist/src/utils.js +9 -1
  82. package/dist/src/utils.js.map +1 -1
  83. package/dist/src/wfqueue.d.ts +1 -2
  84. package/dist/src/wfqueue.d.ts.map +1 -1
  85. package/dist/src/wfqueue.js +83 -63
  86. package/dist/src/wfqueue.js.map +1 -1
  87. package/dist/src/workflow.d.ts +15 -3
  88. package/dist/src/workflow.d.ts.map +1 -1
  89. package/dist/src/workflow.js +30 -12
  90. package/dist/src/workflow.js.map +1 -1
  91. package/dist/src/workflow_management.d.ts +17 -3
  92. package/dist/src/workflow_management.d.ts.map +1 -1
  93. package/dist/src/workflow_management.js +93 -20
  94. package/dist/src/workflow_management.js.map +1 -1
  95. package/dist/tsconfig.tsbuildinfo +1 -1
  96. package/package.json +2 -2
@@ -1,12 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.backfillSchedule = exports.triggerSchedule = exports.DynamicSchedulerLoop = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
3
+ exports.scheduleShardKey = exports.backfillSchedule = exports.triggerSchedule = exports.enqueueTicks = exports.DynamicSchedulerLoop = exports.schedulerLoop = exports.Scheduler = exports.ScheduleEntry = exports.scheduleDefinition = exports.schedulerConfig = exports.scheduleSignature = exports.createScheduleId = exports.toWorkflowSchedule = void 0;
4
4
  const serialization_1 = require("../serialization");
5
5
  const crypto_1 = require("crypto");
6
- const __1 = require("..");
7
6
  const utils_1 = require("../utils");
8
7
  const crontab_1 = require("./crontab");
9
- const dbos_executor_1 = require("../dbos-executor");
10
8
  const error_1 = require("../error");
11
9
  const workflow_1 = require("../workflow");
12
10
  const context_1 = require("../context");
@@ -25,6 +23,7 @@ async function toWorkflowSchedule(internal, serializer) {
25
23
  cronTimezone: internal.cronTimezone,
26
24
  queueName: internal.queueName,
27
25
  applicationName: internal.applicationName,
26
+ shardKey: internal.shardKey,
28
27
  };
29
28
  }
30
29
  exports.toWorkflowSchedule = toWorkflowSchedule;
@@ -32,7 +31,7 @@ function createScheduleId() {
32
31
  return (0, crypto_1.randomUUID)();
33
32
  }
34
33
  exports.createScheduleId = createScheduleId;
35
- // Definition fields a running loop captures at start; a change requires restarting it. Runtime state (schedule_id, status, last_fired_at) is excluded so an unchanged re-apply doesn't restart the loop.
34
+ // Definition fields an armed schedule captures; a change re-arms it. Runtime state (schedule_id, status, last_fired_at) is excluded so an unchanged re-apply doesn't re-arm it.
36
35
  function scheduleSignature(sched) {
37
36
  return JSON.stringify([
38
37
  sched.workflowName,
@@ -41,194 +40,417 @@ function scheduleSignature(sched) {
41
40
  sched.context,
42
41
  sched.cronTimezone ?? null,
43
42
  sched.queueName ?? null,
43
+ sched.shardKey ?? null,
44
44
  ]);
45
45
  }
46
+ exports.scheduleSignature = scheduleSignature;
47
+ exports.schedulerConfig = {
48
+ /** Most scheduled runs enqueued in one transaction. */
49
+ fireBatchSize: 1000,
50
+ /** How long a standby keeps a tick no poll has shown fired, or three polling intervals if longer. */
51
+ bufferMinMs: 3_600_000,
52
+ /** How far back a starting scheduler arms each schedule, so a tick missed while its leader died still fires. */
53
+ startupLookbackMs: 30_000,
54
+ /** How often a scheduler that isn't firing checks whether it has taken over. */
55
+ standbyCheckMs: 1000,
56
+ };
46
57
  // During the first minute after startup, poll for schedules every second so schedules
47
58
  // registered around launch are picked up promptly rather than after a full polling interval.
48
59
  const STARTUP_FAST_POLL_DURATION_MS = 60_000;
49
60
  const STARTUP_FAST_POLL_INTERVAL_MS = 1_000;
50
- // Timers can't sleep past ~24.8 days, so wait for distant runs in bounded chunks.
61
+ // Timers can't sleep past ~24.8 days, so wait for distant ticks in bounded chunks.
51
62
  const MAX_SCHEDULE_SLEEP_MS = 60 * 60 * 1000;
52
- class DynamicSchedulerLoop {
53
- #mainController = new AbortController();
54
- #pollingPromise;
55
- #scheduleLoops = new Map();
56
- #pollingIntervalMs;
57
- constructor(pollingIntervalMs) {
58
- this.#pollingIntervalMs = pollingIntervalMs ?? 30000;
59
- __1.DBOS.registerLifecycleCallback(this);
63
+ async function scheduleDefinition(sched, serializer) {
64
+ return {
65
+ scheduleName: sched.scheduleName,
66
+ workflowName: sched.workflowName,
67
+ workflowClassName: sched.workflowClassName,
68
+ context: await serializer.parse(sched.context),
69
+ queueName: sched.queueName ?? undefined,
70
+ applicationName: sched.applicationName,
71
+ shardKey: sched.shardKey,
72
+ };
73
+ }
74
+ exports.scheduleDefinition = scheduleDefinition;
75
+ /** The first tick strictly after `after`; throws if the schedule can never fire again. */
76
+ function nextTick(matcher, after) {
77
+ const next = matcher.nextWakeupTime(after).getTime();
78
+ // TimeMatcher returns a non-matching instant only past its search horizon.
79
+ if (!matcher.match(next)) {
80
+ throw new error_1.DBOSError(`The schedule never fires after ${new Date(after).toISOString()}`);
60
81
  }
61
- async initialize() {
62
- const signal = this.#mainController.signal;
63
- this.#pollingPromise = (0, context_1.runWithAbortableDbRetries)(signal, () => this.#pollingLoop(signal));
64
- await Promise.resolve();
82
+ return next;
83
+ }
84
+ /** An active schedule in the scheduler and its next tick. */
85
+ class ScheduleEntry {
86
+ definition;
87
+ matcher;
88
+ scheduleId;
89
+ // Definition snapshot; the scheduler re-arms the entry when it changes.
90
+ signature;
91
+ nextTime;
92
+ constructor(sched, definition, matcher, after) {
93
+ this.definition = definition;
94
+ this.matcher = matcher;
95
+ this.scheduleId = sched.scheduleId;
96
+ this.signature = scheduleSignature(sched);
97
+ this.nextTime = nextTick(matcher, after);
65
98
  }
66
- async destroy() {
67
- this.#mainController.abort();
68
- // Abort all per-schedule loops
69
- for (const entry of this.#scheduleLoops.values()) {
70
- entry.controller.abort();
99
+ /** Arm `sched` at its first tick after `after`, by default now. Throws if it can't run. */
100
+ static async create(sched, matcher, serializer, after) {
101
+ const definition = await scheduleDefinition(sched, serializer);
102
+ return new ScheduleEntry(sched, definition, matcher, after ?? Date.now());
103
+ }
104
+ advance() {
105
+ this.nextTime = nextTick(this.matcher, this.nextTime);
106
+ }
107
+ advancePast(lastFired) {
108
+ while (this.nextTime <= lastFired) {
109
+ this.advance();
71
110
  }
72
- const allPromises = [];
73
- if (this.#pollingPromise) {
74
- allPromises.push(this.#pollingPromise);
111
+ }
112
+ }
113
+ exports.ScheduleEntry = ScheduleEntry;
114
+ /** A min-heap of entries by next tick, ties in push order. */
115
+ class TickHeap {
116
+ #items = [];
117
+ #seq = 0;
118
+ get size() {
119
+ return this.#items.length;
120
+ }
121
+ /** The earliest tick, or Infinity if empty. */
122
+ peek() {
123
+ return this.#items[0]?.at ?? Infinity;
124
+ }
125
+ push(entry) {
126
+ const items = this.#items;
127
+ items.push({ at: entry.nextTime, seq: this.#seq++, entry });
128
+ let i = items.length - 1;
129
+ while (i > 0) {
130
+ const parent = (i - 1) >> 1;
131
+ if (!this.#before(i, parent))
132
+ break;
133
+ [items[i], items[parent]] = [items[parent], items[i]];
134
+ i = parent;
75
135
  }
76
- for (const entry of this.#scheduleLoops.values()) {
77
- allPromises.push(entry.promise);
136
+ }
137
+ pop() {
138
+ const items = this.#items;
139
+ const top = items[0];
140
+ const last = items.pop();
141
+ if (items.length > 0) {
142
+ items[0] = last;
143
+ let i = 0;
144
+ while (true) {
145
+ const left = 2 * i + 1;
146
+ const right = left + 1;
147
+ let least = i;
148
+ if (left < items.length && this.#before(left, least))
149
+ least = left;
150
+ if (right < items.length && this.#before(right, least))
151
+ least = right;
152
+ if (least === i)
153
+ break;
154
+ [items[i], items[least]] = [items[least], items[i]];
155
+ i = least;
156
+ }
78
157
  }
79
- await Promise.allSettled(allPromises);
80
- this.#scheduleLoops.clear();
158
+ return top.entry;
81
159
  }
82
- async #pollingLoop(signal) {
83
- const startupDeadline = Date.now() + STARTUP_FAST_POLL_DURATION_MS;
84
- const pollTimeout = () => Date.now() < startupDeadline
85
- ? Math.min(STARTUP_FAST_POLL_INTERVAL_MS, this.#pollingIntervalMs)
86
- : this.#pollingIntervalMs;
87
- while (!signal.aborted) {
88
- let schedules;
89
- try {
90
- const executor = dbos_executor_1.DBOSExecutor.globalInstance;
91
- schedules = await executor.systemDatabase.listSchedules({
92
- applicationName: executor.systemDatabase.appName,
93
- });
160
+ #before(a, b) {
161
+ const x = this.#items[a];
162
+ const y = this.#items[b];
163
+ return x.at < y.at || (x.at === y.at && x.seq < y.seq);
164
+ }
165
+ }
166
+ /** Fires every active schedule from one loop, enqueueing each tick's runs in batches. */
167
+ class Scheduler {
168
+ systemDatabase;
169
+ serializer;
170
+ logger;
171
+ // Active schedules keyed by schedule ID.
172
+ entries = new Map();
173
+ // One item per entry in `entries`; rebuilt by every poll.
174
+ heap = new TickHeap();
175
+ // Whether a poll has armed the schedules this scheduler started with.
176
+ #started = false;
177
+ constructor(systemDatabase, serializer, logger) {
178
+ this.systemDatabase = systemDatabase;
179
+ this.serializer = serializer;
180
+ this.logger = logger;
181
+ }
182
+ /** Sync the armed schedules with the database, then rebuild the heap from them. */
183
+ async poll() {
184
+ try {
185
+ await this.#reconcile();
186
+ }
187
+ finally {
188
+ // Even after a partial reconcile, so the heap never holds dropped entries or misses one.
189
+ this.heap = new TickHeap();
190
+ for (const entry of this.entries.values()) {
191
+ this.heap.push(entry);
94
192
  }
95
- catch (e) {
96
- __1.DBOS.logger.warn(`Dynamic scheduler: error listing schedules: ${e.message}`);
97
- await (0, utils_1.interruptibleSleep)(pollTimeout(), signal);
193
+ }
194
+ }
195
+ async #reconcile() {
196
+ const schedules = await this.systemDatabase.listSchedules({ applicationName: this.systemDatabase.appName });
197
+ const currentIds = new Set(schedules.map((s) => s.scheduleId));
198
+ for (const scheduleId of this.entries.keys()) {
199
+ if (!currentIds.has(scheduleId)) {
200
+ this.entries.delete(scheduleId);
201
+ }
202
+ }
203
+ for (const sched of schedules) {
204
+ const entry = this.entries.get(sched.scheduleId);
205
+ if (sched.status !== 'ACTIVE') {
206
+ // Paused: drop it; resuming re-arms it, backfilling if enabled.
207
+ this.entries.delete(sched.scheduleId);
98
208
  continue;
99
209
  }
100
- // Build set of current schedule names
101
- const currentNames = new Set(schedules.map((s) => s.scheduleName));
102
- // Stop loops for deleted schedules
103
- for (const [name, entry] of this.#scheduleLoops) {
104
- if (!currentNames.has(name)) {
105
- entry.controller.abort();
106
- this.#scheduleLoops.delete(name);
210
+ if (entry !== undefined) {
211
+ // Running: re-arm on a changed definition, without backfill.
212
+ if (entry.signature !== scheduleSignature(sched)) {
213
+ this.entries.delete(sched.scheduleId);
214
+ await this.#arm(sched);
107
215
  }
108
- }
109
- // Process each schedule
110
- for (const sched of schedules) {
111
- const existing = this.#scheduleLoops.get(sched.scheduleName);
112
- const signature = scheduleSignature(sched);
113
- if (sched.status === 'PAUSED' && existing) {
114
- // Paused but has a running loop — stop it
115
- existing.controller.abort();
116
- this.#scheduleLoops.delete(sched.scheduleName);
216
+ else {
217
+ this.#skipFired(entry, sched);
117
218
  }
118
- else if (sched.status === 'ACTIVE') {
119
- // If the schedule's definition changed, restart the loop so it fires with the new definition.
120
- if (existing && existing.signature !== signature) {
121
- existing.controller.abort();
122
- this.#scheduleLoops.delete(sched.scheduleName);
123
- }
124
- if (!this.#scheduleLoops.has(sched.scheduleName)) {
125
- // Automatic backfill: if enabled and lastFiredAt is set,
126
- // backfill missed executions before starting the thread.
127
- if (sched.automaticBackfill && sched.lastFiredAt) {
128
- try {
129
- const lastFired = new Date(sched.lastFiredAt);
130
- const now = new Date();
131
- if (lastFired < now) {
132
- const executor = dbos_executor_1.DBOSExecutor.globalInstance;
133
- await backfillSchedule(executor.systemDatabase, executor.serializer, sched.scheduleName, lastFired, now);
134
- }
135
- }
136
- catch (e) {
137
- __1.DBOS.logger.warn(`Dynamic scheduler: error backfilling schedule "${sched.scheduleName}": ${e.message}`);
138
- }
139
- }
140
- // An abandoned backfill means shutdown began; a loop started now would outlive destroy().
141
- if (signal.aborted)
142
- return;
143
- // Active and no running loop — start one
144
- const controller = new AbortController();
145
- const executor = dbos_executor_1.DBOSExecutor.globalInstance;
146
- // Only enqueues, never runs the workflow, so its retries may be abandoned when the loop stops.
147
- const promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => DynamicSchedulerLoop.#scheduleLoop(sched.scheduleName, sched.workflowName, sched.workflowClassName, sched.schedule, sched.context, executor.serializer, controller.signal, sched.cronTimezone ?? undefined, sched.queueName ?? undefined, sched.applicationName));
148
- this.#scheduleLoops.set(sched.scheduleName, { controller, promise, signature });
219
+ continue;
220
+ }
221
+ // Not running: arm it, then backfill missed executions if enabled; a broken one is retried next poll.
222
+ const armed = await this.#arm(sched, this.#started ? undefined : startupAfter(sched));
223
+ if (armed === undefined)
224
+ continue;
225
+ if (sched.automaticBackfill && sched.lastFiredAt) {
226
+ try {
227
+ const lastFired = new Date(sched.lastFiredAt);
228
+ const now = new Date();
229
+ if (lastFired < now) {
230
+ await backfillSchedule(this.systemDatabase, this.serializer, sched.scheduleName, lastFired, now);
149
231
  }
150
232
  }
233
+ catch (e) {
234
+ this.logger.warn(`Exception during automatic backfill for schedule '${sched.scheduleName}': ${errorText(e)}`);
235
+ }
151
236
  }
152
- await (0, utils_1.interruptibleSleep)(pollTimeout(), signal);
153
237
  }
238
+ this.#started = true;
154
239
  }
155
- static async #scheduleLoop(scheduleName, workflowName, workflowClassName, cronExpression, serializedContext, serializer, signal, cronTimezone, queueName, applicationName) {
156
- let timeMatcher;
240
+ /** Pass over the ticks this schedule's last_fired_at shows some process enqueued. */
241
+ #skipFired(entry, sched) {
242
+ if (!sched.lastFiredAt)
243
+ return;
244
+ const lastFired = Date.parse(sched.lastFiredAt);
245
+ // Unreadable: the entry keeps its ticks, refired at worst.
246
+ if (Number.isNaN(lastFired))
247
+ return;
157
248
  try {
158
- timeMatcher = new crontab_1.TimeMatcher(cronExpression, cronTimezone);
249
+ entry.advancePast(lastFired);
159
250
  }
160
251
  catch (e) {
161
- __1.DBOS.logger.warn(`Dynamic scheduler: not running invalid schedule "${scheduleName}" ("${cronExpression}"): ${e.message}`);
252
+ this.#drop(entry, e);
253
+ }
254
+ }
255
+ /** Register a new entry for this schedule, or log why it can't run and return undefined. */
256
+ async #arm(sched, after) {
257
+ let matcher;
258
+ try {
259
+ matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
260
+ }
261
+ catch (e) {
262
+ this.logger.error(`Cannot run schedule '${sched.scheduleName}'. Invalid crontab "${sched.schedule}": ${errorText(e)}`);
263
+ return undefined;
264
+ }
265
+ let entry;
266
+ try {
267
+ entry = await ScheduleEntry.create(sched, matcher, this.serializer, after);
268
+ }
269
+ catch (e) {
270
+ this.logger.error(`Cannot run schedule '${sched.scheduleName}': ${errorText(e)}`);
271
+ return undefined;
272
+ }
273
+ this.entries.set(entry.scheduleId, entry);
274
+ return entry;
275
+ }
276
+ /** Milliseconds until the earliest tick is `lag` milliseconds old, Infinity if none. */
277
+ msUntilDue(lag = 0) {
278
+ return this.heap.peek() + lag - Date.now();
279
+ }
280
+ /** Fire every due tick, batch by batch, until `signal` aborts. */
281
+ async fireDue(signal) {
282
+ const now = Date.now();
283
+ const due = [];
284
+ while (this.heap.size > 0 && this.heap.peek() <= now) {
285
+ due.push(this.heap.pop());
286
+ }
287
+ const batchSize = exports.schedulerConfig.fireBatchSize;
288
+ let fired = 0;
289
+ while (fired < due.length && !signal.aborted) {
290
+ await this.#fire(due.slice(fired, fired + batchSize), signal);
291
+ fired += batchSize;
292
+ }
293
+ for (const entry of due.slice(0, fired)) {
294
+ this.#advance(entry);
295
+ }
296
+ // Left unstamped, so the process that takes over fires them.
297
+ for (const entry of due.slice(fired)) {
298
+ this.heap.push(entry);
299
+ }
300
+ }
301
+ /** Pass over ticks older than `before` without firing them. */
302
+ skipDue(before) {
303
+ while (this.heap.size > 0 && this.heap.peek() <= before) {
304
+ this.#advance(this.heap.pop());
305
+ }
306
+ }
307
+ #advance(entry) {
308
+ // A late tick's successor may already be due; it fires on the next pass.
309
+ try {
310
+ entry.advance();
311
+ }
312
+ catch (e) {
313
+ this.#drop(entry, e);
162
314
  return;
163
315
  }
164
- const sched = {
165
- scheduleId: '',
166
- scheduleName,
167
- workflowName,
168
- workflowClassName,
169
- schedule: cronExpression,
170
- status: 'ACTIVE',
171
- context: serializedContext,
172
- lastFiredAt: null,
173
- automaticBackfill: false,
174
- cronTimezone: cronTimezone ?? null,
175
- queueName: queueName ?? null,
176
- applicationName,
177
- };
178
- let lastExec = new Date().setMilliseconds(0);
179
- while (!signal.aborted) {
180
- const nextExec = timeMatcher.nextWakeupTime(lastExec).getTime();
181
- const untilNext = nextExec - Date.now();
182
- // Apply jitter to prevent thundering herd
183
- const wakeTime = untilNext > 0 ? nextExec + Math.random() * Math.min(untilNext / 10, 10000) : nextExec;
184
- while (!signal.aborted && Date.now() < wakeTime) {
185
- await (0, utils_1.interruptibleSleep)(Math.min(wakeTime - Date.now(), MAX_SCHEDULE_SLEEP_MS), signal);
316
+ this.heap.push(entry);
317
+ }
318
+ #drop(entry, e) {
319
+ // Dropped; the next poll tries to arm it again.
320
+ this.entries.delete(entry.scheduleId);
321
+ this.logger.error(`Cannot run schedule '${entry.definition.scheduleName}': ${errorText(e)}`);
322
+ }
323
+ async #fire(batch, signal) {
324
+ try {
325
+ await enqueueTicks(this.systemDatabase, this.serializer, batch.map((entry) => [entry.definition, entry.nextTime]), true);
326
+ return;
327
+ }
328
+ catch (e) {
329
+ if (batch.length === 1) {
330
+ this.logger.warn(`Exception in schedule '${batch[0].definition.scheduleName}': ${errorText(e)}`);
331
+ return;
186
332
  }
333
+ // A stopping loop's retries are cut short, so retrying each schedule would only log a warning per schedule.
187
334
  if (signal.aborted) {
188
- break;
189
- }
190
- // If TimeMatcher did not find the next occurrence of the schedule yet,
191
- // we must have it determine the next wakeup time.
192
- if (!timeMatcher.match(nextExec)) {
193
- lastExec = nextExec;
194
- continue;
335
+ this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows: ${errorText(e)}`);
336
+ return;
195
337
  }
196
- const date = new Date(nextExec);
197
- const workflowID = `sched-${scheduleName}-${date.toISOString()}`;
338
+ // Logged here, as the per-schedule retries may succeed and never show it.
339
+ this.logger.warn(`Exception enqueueing ${batch.length} scheduled workflows, retrying each schedule alone: ${errorText(e)}`);
340
+ }
341
+ // One failing schedule must not hold back the rest of its batch.
342
+ for (const entry of batch) {
343
+ if (signal.aborted)
344
+ return;
345
+ await this.#fire([entry], signal);
346
+ }
347
+ }
348
+ }
349
+ exports.Scheduler = Scheduler;
350
+ /** Where a starting scheduler arms a schedule: past its last fired tick, back at most the startup look-back. Later arms (creates, resumes) start from now. */
351
+ function startupAfter(sched) {
352
+ if (!sched.lastFiredAt)
353
+ return undefined;
354
+ const lastFired = Date.parse(sched.lastFiredAt);
355
+ if (Number.isNaN(lastFired))
356
+ return undefined;
357
+ return Math.max(lastFired, Date.now() - exports.schedulerConfig.startupLookbackMs);
358
+ }
359
+ function errorText(e) {
360
+ return e instanceof Error ? (e.stack ?? e.message) : String(e);
361
+ }
362
+ /** Poll schedules until aborted, firing only while `leading()`; a standby keeps each tick until a poll shows it fired, so a taker fires all it missed. */
363
+ async function schedulerLoop(signal, scheduler, pollingIntervalMs, leading) {
364
+ const bufferMs = Math.max(exports.schedulerConfig.bufferMinMs, 3 * pollingIntervalMs);
365
+ const startupDeadline = performance.now() + STARTUP_FAST_POLL_DURATION_MS;
366
+ const pollTimeout = () => performance.now() < startupDeadline
367
+ ? Math.min(STARTUP_FAST_POLL_INTERVAL_MS, pollingIntervalMs)
368
+ : pollingIntervalMs;
369
+ let nextPoll = performance.now();
370
+ while (!signal.aborted) {
371
+ if (performance.now() >= nextPoll) {
198
372
  try {
199
- // Idempotency check -- for performance only, not needed for correctness
200
- const existing = await __1.DBOS.getWorkflowStatus(workflowID);
201
- if (existing) {
202
- lastExec = nextExec;
203
- continue;
204
- }
205
- const context = await serializer.parse(serializedContext);
206
- const systemDatabase = dbos_executor_1.DBOSExecutor.globalInstance.systemDatabase;
207
- await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, date, context);
208
- await systemDatabase.updateLastFiredAt(scheduleName, date.toISOString());
373
+ await scheduler.poll();
209
374
  }
210
375
  catch (e) {
211
- __1.DBOS.logger.warn(`Dynamic scheduler: error firing workflow for schedule "${scheduleName}": ${e.message}`);
376
+ scheduler.logger.warn(`Exception polling schedules: ${errorText(e)}`);
377
+ }
378
+ nextPoll = performance.now() + pollTimeout();
379
+ }
380
+ // Checked once per pass: a holder losing the lock mid-pass still fires all it took.
381
+ const firing = leading();
382
+ try {
383
+ if (firing) {
384
+ await scheduler.fireDue(signal);
212
385
  }
213
- lastExec = nextExec;
386
+ else {
387
+ scheduler.skipDue(Date.now() - bufferMs);
388
+ }
389
+ }
390
+ catch (e) {
391
+ scheduler.logger.warn(`Exception firing schedules: ${errorText(e)}`);
392
+ }
393
+ const untilPoll = nextPoll - performance.now();
394
+ const wait = firing
395
+ ? Math.min(untilPoll, scheduler.msUntilDue())
396
+ : Math.min(untilPoll, scheduler.msUntilDue(bufferMs), exports.schedulerConfig.standbyCheckMs);
397
+ // Always yields, so a pass that finds more due work never starves the event loop.
398
+ await (0, utils_1.interruptibleSleep)(Math.min(Math.max(wait, 0), MAX_SCHEDULE_SLEEP_MS), signal);
399
+ }
400
+ }
401
+ exports.schedulerLoop = schedulerLoop;
402
+ /** Runs the scheduler loop from `initialize` until `destroy`, firing while `leading()`. */
403
+ class DynamicSchedulerLoop {
404
+ systemDatabase;
405
+ serializer;
406
+ logger;
407
+ leading;
408
+ #controller;
409
+ #promise;
410
+ #pollingIntervalMs;
411
+ constructor(systemDatabase, serializer, logger, pollingIntervalMs, leading) {
412
+ this.systemDatabase = systemDatabase;
413
+ this.serializer = serializer;
414
+ this.logger = logger;
415
+ this.leading = leading;
416
+ this.#pollingIntervalMs = pollingIntervalMs ?? 30000;
417
+ }
418
+ /** Start the loop; a no-op while it runs. */
419
+ async initialize() {
420
+ if (this.#controller !== undefined)
421
+ return;
422
+ const controller = new AbortController();
423
+ this.#controller = controller;
424
+ const scheduler = new Scheduler(this.systemDatabase, this.serializer, this.logger);
425
+ // Only enqueues, never runs a workflow, so its retries may be abandoned when the loop stops.
426
+ this.#promise = (0, context_1.runWithAbortableDbRetries)(controller.signal, () => schedulerLoop(controller.signal, scheduler, this.#pollingIntervalMs, this.leading));
427
+ await Promise.resolve();
428
+ }
429
+ /** Stop the loop and wait for it to end; a no-op if it isn't running. */
430
+ async destroy() {
431
+ const controller = this.#controller;
432
+ const promise = this.#promise;
433
+ // Cleared first, so an initialize while this one winds down starts a fresh loop.
434
+ this.#controller = undefined;
435
+ this.#promise = undefined;
436
+ controller?.abort();
437
+ if (promise !== undefined) {
438
+ await Promise.allSettled([promise]);
214
439
  }
215
440
  }
216
441
  }
217
442
  exports.DynamicSchedulerLoop = DynamicSchedulerLoop;
218
- async function enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, scheduledDate, context) {
219
- const serparam = await (0, serialization_1.serializeArgs)([scheduledDate, context], undefined, serializer, undefined);
220
- // The schedule's owner routes its runs, whoever fires them; an unclaimed one falls back to the firing handle, which may have no identity itself.
221
- const owner = sched.applicationName ?? systemDatabase.appName;
222
- // Scheduled workflows are always enqueued to their owner's latest application version
223
- const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
224
- const internalStatus = {
443
+ /** The ENQUEUED status row of one scheduled run. */
444
+ async function scheduledStatus(definition, scheduledDate, workflowID, applicationVersion, owner, serializer) {
445
+ const serparam = await (0, serialization_1.serializeArgs)([scheduledDate, definition.context], undefined, serializer, undefined);
446
+ return {
225
447
  workflowUUID: workflowID,
226
448
  status: workflow_1.StatusString.ENQUEUED,
227
- workflowName: sched.workflowName,
228
- workflowClassName: sched.workflowClassName,
449
+ workflowName: definition.workflowName,
450
+ workflowClassName: definition.workflowClassName,
229
451
  workflowConfigName: '',
230
- applicationVersion: latestVersion.versionName,
231
- queueName: sched.queueName || utils_1.INTERNAL_QUEUE_NAME,
452
+ applicationVersion,
453
+ queueName: definition.queueName || utils_1.INTERNAL_QUEUE_NAME,
232
454
  authenticatedUser: '',
233
455
  output: null,
234
456
  error: null,
@@ -241,50 +463,79 @@ async function enqueueScheduledWorkflow(systemDatabase, serializer, sched, workf
241
463
  priority: 0,
242
464
  queuePartitionKey: undefined,
243
465
  serialization: serparam.serialization,
244
- scheduleName: sched.scheduleName,
466
+ scheduleName: definition.scheduleName,
245
467
  // Owned where possible: the internal queue's shared name cannot route an unclaimed row.
246
468
  applicationName: owner,
469
+ shardKey: definition.shardKey,
247
470
  };
248
- await systemDatabase.initWorkflowStatus(internalStatus, null);
249
- return;
250
471
  }
472
+ /** The schedule's owner routes its runs, whoever fires them; an unclaimed one falls back to the firing handle, which may have no identity itself. */
473
+ function scheduleOwner(systemDatabase, definition) {
474
+ return definition.applicationName ?? systemDatabase.appName;
475
+ }
476
+ /** Enqueue these scheduled runs in one batch, skipping existing ones. Returns their workflow IDs. */
477
+ async function enqueueTicks(systemDatabase, serializer, ticks, stampLastFired) {
478
+ // Scheduled workflows are always enqueued to their owner's latest application version.
479
+ const versions = new Map();
480
+ const statuses = [];
481
+ const firedAt = new Map();
482
+ for (const [definition, at] of ticks) {
483
+ const owner = scheduleOwner(systemDatabase, definition);
484
+ let version = versions.get(owner);
485
+ if (version === undefined) {
486
+ version = (await systemDatabase.getLatestApplicationVersion(owner)).versionName;
487
+ versions.set(owner, version);
488
+ }
489
+ const scheduledDate = new Date(at);
490
+ const workflowID = `sched-${definition.scheduleName}-${scheduledDate.toISOString()}`;
491
+ statuses.push(await scheduledStatus(definition, scheduledDate, workflowID, version, owner, serializer));
492
+ if (stampLastFired) {
493
+ firedAt.set(workflowID, scheduledDate.toISOString());
494
+ }
495
+ }
496
+ await systemDatabase.enqueueScheduledWorkflows(statuses, firedAt);
497
+ return statuses.map((status) => status.workflowUUID);
498
+ }
499
+ exports.enqueueTicks = enqueueTicks;
251
500
  async function triggerSchedule(systemDatabase, serializer, name) {
252
501
  const sched = await systemDatabase.getSchedule(name);
253
502
  if (!sched) {
254
503
  throw new error_1.DBOSError(`Schedule "${name}" not found`);
255
504
  }
256
- const context = await serializer.parse(sched.context);
505
+ const definition = await scheduleDefinition(sched, serializer);
257
506
  const now = new Date();
258
507
  const workflowID = `sched-${name}-trigger-${now.toISOString()}`;
259
- await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, now, context);
508
+ const owner = scheduleOwner(systemDatabase, definition);
509
+ const latestVersion = await systemDatabase.getLatestApplicationVersion(owner);
510
+ await systemDatabase.initWorkflowStatus(await scheduledStatus(definition, now, workflowID, latestVersion.versionName, owner, serializer), null);
260
511
  return workflowID;
261
512
  }
262
513
  exports.triggerSchedule = triggerSchedule;
514
+ /** Enqueue every scheduled execution between `start` and `end`. Returns their workflow IDs. */
263
515
  async function backfillSchedule(systemDatabase, serializer, name, start, end) {
264
516
  const sched = await systemDatabase.getSchedule(name);
265
517
  if (!sched) {
266
518
  throw new error_1.DBOSError(`Schedule "${name}" not found`);
267
519
  }
268
- const context = await serializer.parse(sched.context);
269
- const timeMatcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
520
+ const definition = await scheduleDefinition(sched, serializer);
521
+ const matcher = new crontab_1.TimeMatcher(sched.schedule, sched.cronTimezone ?? undefined);
522
+ const batchSize = exports.schedulerConfig.fireBatchSize;
270
523
  const workflowIDs = [];
271
- let current = start.getTime();
272
- while (current < end.getTime()) {
273
- const next = timeMatcher.nextWakeupTime(current);
274
- // If TimeMatcher did not find the next occurrence of the schedule yet,
275
- // we must have it determine the next wakeup time.
276
- if (!timeMatcher.match(next)) {
277
- current = next.getTime();
278
- continue;
524
+ // Computed a batch at a time, so a long backfill yields to the event loop between batches.
525
+ let next = nextTick(matcher, start.getTime());
526
+ while (next < end.getTime()) {
527
+ const batch = [];
528
+ for (; next < end.getTime() && batch.length < batchSize; next = nextTick(matcher, next)) {
529
+ batch.push([definition, next]);
279
530
  }
280
- if (next.getTime() >= end.getTime())
281
- break;
282
- const workflowID = `sched-${name}-${next.toISOString()}`;
283
- await enqueueScheduledWorkflow(systemDatabase, serializer, sched, workflowID, next, context);
284
- workflowIDs.push(workflowID);
285
- current = next.getTime();
531
+ workflowIDs.push(...(await enqueueTicks(systemDatabase, serializer, batch, false)));
286
532
  }
287
533
  return workflowIDs;
288
534
  }
289
535
  exports.backfillSchedule = backfillSchedule;
536
+ /** The shard key a schedule's workflows run under. */
537
+ async function scheduleShardKey(systemDatabase, name) {
538
+ return (await systemDatabase.getSchedule(name))?.shardKey;
539
+ }
540
+ exports.scheduleShardKey = scheduleShardKey;
290
541
  //# sourceMappingURL=scheduler.js.map