durability 5.0.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,4 @@
1
+ import { type SerializedError } from './errors.js';
1
2
  import { type DurabilityLifecycleEvent, type Emit, type LifecycleEntity } from './lifecycle.js';
2
3
  import { type MigrationCapability } from './migrations.js';
3
4
  import { type AttemptPolicy } from './policy.js';
@@ -13,6 +14,33 @@ export type DurabilitySchedulerOptions = {
13
14
  alarmMinDelayMs?: number;
14
15
  /** Time before a running alarm hands unfinished work to a new alarm. Defaults to 14 minutes. */
15
16
  alarmHandoffMs?: number;
17
+ /**
18
+ * Experimental. Start due work from in-memory timers at its exact scheduled
19
+ * time, independent of `alarmMinDelayMs`, and let newly due work run without
20
+ * waiting behind long-running handlers. The physical alarm stays armed as the
21
+ * eviction fallback. Off by default.
22
+ */
23
+ liveTimers?: boolean;
24
+ /**
25
+ * Called each time an alarm run fails and the scheduler re-arms its own
26
+ * fallback alarm instead of letting the platform retry, for example to send
27
+ * the error to an external service. Best effort: it runs after the fallback is
28
+ * armed, a returned promise is passed to `waitUntil`, and a throw or rejection
29
+ * is only logged, so it never changes scheduling.
30
+ */
31
+ onAlarmRetry?: (info: AlarmRetryInfo) => void | Promise<void>;
32
+ };
33
+ /** Details of a failed alarm run passed to `onAlarmRetry`. */
34
+ export type AlarmRetryInfo = {
35
+ error: SerializedError;
36
+ /** Delay until the re-armed fallback alarm fires. */
37
+ retryInMs: number;
38
+ /** The platform's retry count for this alarm invocation, when it provided one. */
39
+ retryCount?: number;
40
+ /** Whether the platform marked this invocation as a retry, when it provided one. */
41
+ isRetry?: boolean;
42
+ /** The owning Durable Object's ID, when the context carries one. */
43
+ objectId?: string;
16
44
  };
17
45
  /** Configuration for a scheduler shared by several helpers. */
18
46
  export type DurabilitySchedulerConfig = DurabilitySchedulerOptions & {
@@ -34,6 +62,12 @@ export type Participant = {
34
62
  runDue(now: number, info: AlarmInvocationInfo | undefined): Promise<void>;
35
63
  /** Earliest due time, for tables without `status` and `next_attempt_at` columns. */
36
64
  nextPendingAt?(sql: SqlStorage): number | undefined;
65
+ /**
66
+ * True while the participant runs its own due work from an alarm invocation
67
+ * it keeps open. The in-memory timer then ignores its pending time, which only
68
+ * arms the physical alarm as the eviction fallback.
69
+ */
70
+ holding?(): boolean;
37
71
  };
38
72
  /**
39
73
  * Called after a record may have settled. Successful completions pass the
@@ -85,6 +119,8 @@ export declare class Engine {
85
119
  private readonly alarmConcurrency;
86
120
  private readonly alarmMinDelayMs;
87
121
  private readonly alarmHandoffMs;
122
+ readonly liveTimers: boolean;
123
+ private readonly onAlarmRetry;
88
124
  private readonly participants;
89
125
  private readonly migrated;
90
126
  private availablePermits;
@@ -92,6 +128,7 @@ export declare class Engine {
92
128
  private alarmRefresh;
93
129
  private earlyAlarm;
94
130
  private readonly watchers;
131
+ private readonly activeByTable;
95
132
  constructor(context: DurabilityContext, options: DurabilitySchedulerOptions);
96
133
  register(participant: Participant): void;
97
134
  waitUntil(promise: Promise<unknown>, onError: (error: unknown) => void): void;
@@ -113,7 +150,19 @@ export declare class Engine {
113
150
  * transaction, then publishes the in-memory timer only after commit.
114
151
  */
115
152
  transaction<T>(write: (sql: SqlStorage) => T): Promise<T>;
116
- /** Returns the earliest pending time and maintains its bounded physical fallback. */
153
+ private trackActive;
154
+ private releaseActive;
155
+ /**
156
+ * The earliest pending time for one participant: `physical` arms the physical
157
+ * alarm (eviction fallback) and includes active work; `timer` drives the
158
+ * in-memory timer and, under liveTimers, skips work that is already executing
159
+ * so a still-due active row cannot re-fire a zero-delay timer in a loop.
160
+ */
161
+ private pendingTimes;
162
+ /**
163
+ * Arms the physical alarm from the earliest pending time (the eviction
164
+ * fallback) and returns the earliest time the in-memory timer should fire.
165
+ */
117
166
  private reconcile;
118
167
  private scheduleEarlyAlarm;
119
168
  /** Coalesces concurrent refresh requests into one reconciliation transaction. */
@@ -137,6 +186,7 @@ export declare class Engine {
137
186
  * because the platform drops an alarm after a few failed deliveries.
138
187
  */
139
188
  alarm(info?: AlarmInvocationInfo): Promise<void>;
189
+ private notifyAlarmRetry;
140
190
  /**
141
191
  * Runs every participant's due work, hands off to a fresh alarm if the
142
192
  * invocation approaches its wall-time limit, and re-arms the physical alarm.
package/dist/scheduler.js CHANGED
@@ -8,7 +8,11 @@ import { deleteMemos, deleteStaleMemos } from './memoize.js';
8
8
  import { claimAttempt, exhaustAttempts, updateAttempt, } from './storage.js';
9
9
  const minAlarmDelayMs = 1000;
10
10
  const maxAlarmDelayMs = 15000;
11
- const nextAttemptRowSchema = z.object({
11
+ // setTimeout treats delays past this as zero, so anything further out can only
12
+ // be served by the physical alarm fallback.
13
+ const maxTimerMs = 2 ** 31 - 1;
14
+ const pendingHeadRowSchema = z.object({
15
+ pk: z.string(),
12
16
  next_attempt_at: z.number().int().nonnegative(),
13
17
  });
14
18
  const countRowSchema = z.object({ count: z.number().int().nonnegative() });
@@ -60,12 +64,21 @@ export class Engine {
60
64
  this.migrated = new Set();
61
65
  this.permitWaiters = [];
62
66
  this.watchers = new Map();
67
+ // Keys whose attempt is currently executing, per table. The in-memory timer
68
+ // skips these so a still-due active row cannot drive a zero-delay timer loop.
69
+ this.activeByTable = new Map();
63
70
  this.context = context;
64
71
  this.storage = traceStorage(context.storage);
65
72
  this.sql = this.storage.sql;
66
73
  this.alarmConcurrency = options.alarmConcurrency ?? 10;
67
74
  this.alarmMinDelayMs = options.alarmMinDelayMs ?? 15000;
68
75
  this.alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
76
+ this.liveTimers = options.liveTimers ?? false;
77
+ this.onAlarmRetry = options.onAlarmRetry;
78
+ if (this.onAlarmRetry !== undefined &&
79
+ typeof this.onAlarmRetry !== 'function') {
80
+ throw new TypeError('onAlarmRetry must be a function');
81
+ }
69
82
  assertPositiveInteger('alarmConcurrency', this.alarmConcurrency);
70
83
  assertPositiveInteger('alarmMinDelayMs', this.alarmMinDelayMs);
71
84
  if (this.alarmMinDelayMs < minAlarmDelayMs ||
@@ -134,39 +147,91 @@ export class Engine {
134
147
  this.scheduleEarlyAlarm(nextPendingAt);
135
148
  return result;
136
149
  }
137
- /** Returns the earliest pending time and maintains its bounded physical fallback. */
150
+ trackActive(table, key) {
151
+ let keys = this.activeByTable.get(table);
152
+ if (!keys) {
153
+ keys = new Set();
154
+ this.activeByTable.set(table, keys);
155
+ }
156
+ keys.add(key);
157
+ }
158
+ releaseActive(table, key) {
159
+ const keys = this.activeByTable.get(table);
160
+ if (!keys) {
161
+ return;
162
+ }
163
+ keys.delete(key);
164
+ if (keys.size === 0) {
165
+ this.activeByTable.delete(table);
166
+ }
167
+ }
168
+ /**
169
+ * The earliest pending time for one participant: `physical` arms the physical
170
+ * alarm (eviction fallback) and includes active work; `timer` drives the
171
+ * in-memory timer and, under liveTimers, skips work that is already executing
172
+ * so a still-due active row cannot re-fire a zero-delay timer in a loop.
173
+ */
174
+ pendingTimes(participant) {
175
+ if (participant.nextPendingAt) {
176
+ const at = participant.nextPendingAt(this.sql);
177
+ if (at === undefined) {
178
+ return {};
179
+ }
180
+ return participant.holding?.()
181
+ ? { physical: at }
182
+ : { physical: at, timer: at };
183
+ }
184
+ const active = this.activeByTable.get(participant.table.name);
185
+ const limit = this.liveTimers ? (active?.size ?? 0) + 1 : 1;
186
+ const rows = pendingHeadRowSchema.array().parse(this.sql
187
+ .exec(`SELECT ${participant.table.key} AS pk, next_attempt_at
188
+ FROM ${participant.table.name}
189
+ WHERE status = 'pending'
190
+ ORDER BY next_attempt_at ASC
191
+ LIMIT ?`, limit)
192
+ .toArray());
193
+ const physical = rows[0]?.next_attempt_at;
194
+ if (!this.liveTimers) {
195
+ if (physical === undefined) {
196
+ return {};
197
+ }
198
+ return { physical, timer: physical };
199
+ }
200
+ const eligible = rows.find((row) => !active?.has(row.pk));
201
+ return { physical, timer: eligible?.next_attempt_at };
202
+ }
203
+ /**
204
+ * Arms the physical alarm from the earliest pending time (the eviction
205
+ * fallback) and returns the earliest time the in-memory timer should fire.
206
+ */
138
207
  async reconcile(physicalAlarm) {
139
- const scheduled = this.participants.flatMap((participant) => {
140
- if (participant.nextPendingAt) {
141
- const at = participant.nextPendingAt(this.sql);
142
- return at === undefined ? [] : [at];
208
+ const physicalTimes = [];
209
+ const timerTimes = [];
210
+ for (const participant of this.participants) {
211
+ const { physical, timer } = this.pendingTimes(participant);
212
+ if (physical !== undefined) {
213
+ physicalTimes.push(physical);
143
214
  }
144
- return nextAttemptRowSchema
145
- .array()
146
- .parse(this.sql
147
- .exec(`SELECT next_attempt_at FROM ${participant.table.name}
148
- WHERE status = 'pending'
149
- ORDER BY next_attempt_at ASC
150
- LIMIT 1`)
151
- .toArray())
152
- .map((row) => row.next_attempt_at);
153
- });
154
- const nextAlarmAt = scheduled.length === 0 ? undefined : Math.min(...scheduled);
215
+ if (timer !== undefined) {
216
+ timerTimes.push(timer);
217
+ }
218
+ }
219
+ const physicalNextAt = physicalTimes.length === 0 ? undefined : Math.min(...physicalTimes);
155
220
  const currentAlarm = await physicalAlarm.getAlarm();
156
- if (nextAlarmAt === undefined) {
221
+ if (physicalNextAt === undefined) {
157
222
  if (currentAlarm !== null) {
158
223
  await physicalAlarm.deleteAlarm();
159
224
  }
160
225
  return undefined;
161
226
  }
162
- const target = Math.max(nextAlarmAt, Date.now() + this.alarmMinDelayMs);
227
+ const target = Math.max(physicalNextAt, Date.now() + this.alarmMinDelayMs);
163
228
  // Preserve a fallback inside this range so frequent reconciliation cannot push it later.
164
229
  if (currentAlarm === null ||
165
- currentAlarm < nextAlarmAt ||
230
+ currentAlarm < physicalNextAt ||
166
231
  currentAlarm > target) {
167
232
  await physicalAlarm.setAlarm(target);
168
233
  }
169
- return nextAlarmAt;
234
+ return timerTimes.length === 0 ? undefined : Math.min(...timerTimes);
170
235
  }
171
236
  scheduleEarlyAlarm(timestamp) {
172
237
  if (this.earlyAlarm?.timestamp === timestamp) {
@@ -176,8 +241,18 @@ export class Engine {
176
241
  clearTimeout(this.earlyAlarm.timer);
177
242
  this.earlyAlarm = undefined;
178
243
  }
179
- if (timestamp === undefined ||
180
- timestamp >= Date.now() + this.alarmMinDelayMs) {
244
+ if (timestamp === undefined) {
245
+ return;
246
+ }
247
+ const delay = timestamp - Date.now();
248
+ // Without liveTimers the in-memory timer only covers the sub-min-delay
249
+ // window; the physical alarm handles everything beyond it. With liveTimers
250
+ // it fires at the exact due time no matter how far out, except past the
251
+ // timer's range where only the physical fallback can serve it.
252
+ if (!this.liveTimers && delay >= this.alarmMinDelayMs) {
253
+ return;
254
+ }
255
+ if (delay > maxTimerMs) {
181
256
  return;
182
257
  }
183
258
  const timer = setTimeout(() => {
@@ -189,7 +264,7 @@ export class Engine {
189
264
  const onError = (error) => reportFailure('durability.background_execution.failed', { scheduledTime: timestamp }, error);
190
265
  const execution = this.alarm().catch(onError);
191
266
  this.waitUntil(execution, onError);
192
- }, Math.max(timestamp - Date.now(), 0));
267
+ }, Math.max(delay, 0));
193
268
  this.earlyAlarm = { timestamp, timer };
194
269
  }
195
270
  /** Coalesces concurrent refresh requests into one reconciliation transaction. */
@@ -312,6 +387,35 @@ export class Engine {
312
387
  throw error;
313
388
  }
314
389
  reportFailure('durability.alarm.failed', { retryInMs: this.alarmMinDelayMs }, error);
390
+ this.notifyAlarmRetry(error, info);
391
+ }
392
+ }
393
+ notifyAlarmRetry(failure, invocation) {
394
+ const hook = this.onAlarmRetry;
395
+ if (!hook) {
396
+ return;
397
+ }
398
+ const objectId = this.objectId();
399
+ const info = {
400
+ error: serializeError(failure),
401
+ retryInMs: this.alarmMinDelayMs,
402
+ ...(invocation === undefined
403
+ ? {}
404
+ : {
405
+ retryCount: invocation.retryCount,
406
+ isRetry: invocation.isRetry,
407
+ }),
408
+ ...(objectId === undefined ? {} : { objectId }),
409
+ };
410
+ const report = (hookError) => reportFailure('durability.alarm.retry_hook_failed', { retryInMs: info.retryInMs }, hookError);
411
+ try {
412
+ const result = hook(info);
413
+ if (result !== undefined) {
414
+ this.waitUntil(Promise.resolve(result).catch(report), report);
415
+ }
416
+ }
417
+ catch (hookError) {
418
+ report(hookError);
315
419
  }
316
420
  }
317
421
  /**
@@ -381,6 +485,7 @@ export class Engine {
381
485
  settled,
382
486
  };
383
487
  spec.active.set(key, activeExecution);
488
+ this.trackActive(spec.table.name, key);
384
489
  let releasePermit = eagerPermit;
385
490
  let handlerStarted = false;
386
491
  let handlerSettled = false;
@@ -462,6 +567,12 @@ export class Engine {
462
567
  if (spec.active.get(key) === activeExecution) {
463
568
  spec.active.delete(key);
464
569
  }
570
+ this.releaseActive(spec.table.name, key);
571
+ // Re-arm the in-memory timer now that this key is eligible again, so work
572
+ // the timer skipped while it executed is picked up at its exact time.
573
+ if (this.liveTimers) {
574
+ void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
575
+ }
465
576
  });
466
577
  return execution;
467
578
  }
@@ -208,7 +208,9 @@ export class QueueStore {
208
208
  this.sql.exec(`INSERT INTO durability_semaphore_queue_keys (key, next_request_at, queued)
209
209
  VALUES (?, ?, ?)
210
210
  ON CONFLICT(key) DO UPDATE SET
211
- next_request_at = excluded.next_request_at, queued = excluded.queued`, row.key, row.next_request_at, row.queued);
211
+ next_request_at = excluded.next_request_at, queued = excluded.queued
212
+ WHERE next_request_at != excluded.next_request_at
213
+ OR queued != excluded.queued`, row.key, row.next_request_at, row.queued);
212
214
  }
213
215
  removeKey(key) {
214
216
  this.sql.exec('DELETE FROM durability_semaphore_queue_keys WHERE key = ?', key);
@@ -221,24 +223,26 @@ export class QueueStore {
221
223
  .toArray());
222
224
  }
223
225
  nextPendingAt(checksFinished) {
224
- const candidates = [
225
- this.sql
226
- .exec(`SELECT next_request_at AS at FROM durability_semaphore_queue_keys
227
- WHERE queued > 0 ORDER BY next_request_at ASC LIMIT 1`)
228
- .toArray()[0]?.['at'],
229
- this.sql
230
- .exec(`SELECT next_attempt_at AS at FROM durability_semaphore_queue_items
231
- WHERE status = 'admitted' AND (notified = 0 OR ?)
232
- ORDER BY next_attempt_at ASC LIMIT 1`, checksFinished ? 1 : 0)
233
- .toArray()[0]?.['at'],
234
- // An idle lease is due when it stops at its expiry, not before.
235
- this.sql
236
- .exec(`SELECT MIN(CASE WHEN released < drawn THEN renew_at
237
- ELSE MAX(renew_at, expires_at) END) AS at
238
- FROM durability_semaphore_queue_leases WHERE confirmed = 0`)
239
- .toArray()[0]?.['at'],
240
- ].filter((value) => typeof value === 'number');
241
- return candidates.length === 0 ? undefined : Math.min(...candidates);
226
+ // An idle lease is due when it stops at its expiry, not before.
227
+ const at = this.sql
228
+ .exec(`SELECT MIN(at) AS at FROM (
229
+ SELECT (
230
+ SELECT next_request_at FROM durability_semaphore_queue_keys
231
+ WHERE queued > 0 ORDER BY next_request_at ASC LIMIT 1
232
+ ) AS at
233
+ UNION ALL
234
+ SELECT (
235
+ SELECT next_attempt_at FROM durability_semaphore_queue_items
236
+ WHERE status = 'admitted' AND (notified = 0 OR ?)
237
+ ORDER BY next_attempt_at ASC LIMIT 1
238
+ ) AS at
239
+ UNION ALL
240
+ SELECT MIN(CASE WHEN released < drawn THEN renew_at
241
+ ELSE MAX(renew_at, expires_at) END) AS at
242
+ FROM durability_semaphore_queue_leases WHERE confirmed = 0
243
+ )`, checksFinished ? 1 : 0)
244
+ .toArray()[0]?.['at'];
245
+ return typeof at === 'number' ? at : undefined;
242
246
  }
243
247
  items(query, ...bindings) {
244
248
  return queueItemRowSchema
package/dist/storage.d.ts CHANGED
@@ -13,8 +13,11 @@ export type AttemptRef = {
13
13
  attempt: number;
14
14
  };
15
15
  export declare const pendingGeneration: (table: DurableTable) => string;
16
- /** Starts the next attempt and returns its number, unless the generation is gone or out of attempts. */
17
- export declare const claimAttempt: (sql: SqlStorage, table: DurableTable, key: string, generation: string, maxAttempts: number) => number | undefined;
16
+ /**
17
+ * Starts the next attempt, optionally persisting its deadline in the same write.
18
+ * Returns undefined when the generation is gone or out of attempts.
19
+ */
20
+ export declare const claimAttempt: (sql: SqlStorage, table: DurableTable, key: string, generation: string, maxAttempts: number, nextAttemptAt?: number) => number | undefined;
18
21
  /**
19
22
  * Applies `assignments` (a SQL `SET` list) to the record if the attempt still
20
23
  * owns it, and reports whether it did.
package/dist/storage.js CHANGED
@@ -5,13 +5,19 @@ const attemptRowSchema = z.object({
5
5
  // Every write is conditional on the record's generation and pending status, so a
6
6
  // stale execution can never overwrite a replacement or a later attempt.
7
7
  export const pendingGeneration = (table) => `${table.key} = ? AND generation_id = ? AND status = 'pending'`;
8
- /** Starts the next attempt and returns its number, unless the generation is gone or out of attempts. */
9
- export const claimAttempt = (sql, table, key, generation, maxAttempts) => attemptRowSchema.optional().parse(sql
10
- .exec(`UPDATE ${table.name}
11
- SET attempt = attempt + 1
8
+ /**
9
+ * Starts the next attempt, optionally persisting its deadline in the same write.
10
+ * Returns undefined when the generation is gone or out of attempts.
11
+ */
12
+ export const claimAttempt = (sql, table, key, generation, maxAttempts, nextAttemptAt) => {
13
+ const deadlineAssignment = nextAttemptAt === undefined ? '' : ', next_attempt_at = ?';
14
+ return attemptRowSchema.optional().parse(sql
15
+ .exec(`UPDATE ${table.name}
16
+ SET attempt = attempt + 1${deadlineAssignment}
12
17
  WHERE ${pendingGeneration(table)} AND attempt < ?
13
- RETURNING attempt`, key, generation, maxAttempts)
14
- .toArray()[0])?.attempt;
18
+ RETURNING attempt`, ...(nextAttemptAt === undefined ? [] : [nextAttemptAt]), key, generation, maxAttempts)
19
+ .toArray()[0])?.attempt;
20
+ };
15
21
  /**
16
22
  * Applies `assignments` (a SQL `SET` list) to the record if the attempt still
17
23
  * owns it, and reports whether it did.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "durability",
3
- "version": "5.0.0",
3
+ "version": "5.1.0",
4
4
  "description": "Alarm-backed durable operations and typed WebSocket sync for Cloudflare Durable Objects",
5
5
  "type": "module",
6
6
  "license": "ISC",