durability 4.3.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +245 -20
- package/dist/durability-alarms.d.ts +25 -3
- package/dist/durability-alarms.js +50 -8
- package/dist/durability.d.ts +30 -5
- package/dist/durability.js +42 -14
- package/dist/errors.d.ts +29 -0
- package/dist/errors.js +47 -0
- package/dist/experimental.d.ts +3 -0
- package/dist/experimental.js +3 -0
- package/dist/fanout.d.ts +6 -0
- package/dist/fanout.js +10 -6
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/lifecycle.d.ts +6 -1
- package/dist/log.js +35 -12
- package/dist/memoize.d.ts +34 -0
- package/dist/memoize.js +136 -0
- package/dist/migrations.d.ts +27 -1
- package/dist/migrations.js +93 -1
- package/dist/outbox.js +2 -3
- package/dist/schedule.d.ts +157 -0
- package/dist/schedule.js +651 -0
- package/dist/scheduler.d.ts +53 -1
- package/dist/scheduler.js +164 -30
- package/dist/semaphore-queue-store.js +23 -19
- package/dist/storage.d.ts +6 -2
- package/dist/storage.js +13 -7
- package/dist/tracked-source.d.ts +2 -0
- package/package.json +2 -1
package/dist/scheduler.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type SerializedError } from './errors.js';
|
|
1
2
|
import { type DurabilityLifecycleEvent, type Emit, type LifecycleEntity } from './lifecycle.js';
|
|
2
3
|
import { type MigrationCapability } from './migrations.js';
|
|
3
4
|
import { type AttemptPolicy } from './policy.js';
|
|
@@ -13,6 +14,33 @@ export type DurabilitySchedulerOptions = {
|
|
|
13
14
|
alarmMinDelayMs?: number;
|
|
14
15
|
/** Time before a running alarm hands unfinished work to a new alarm. Defaults to 14 minutes. */
|
|
15
16
|
alarmHandoffMs?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Experimental. Start due work from in-memory timers at its exact scheduled
|
|
19
|
+
* time, independent of `alarmMinDelayMs`, and let newly due work run without
|
|
20
|
+
* waiting behind long-running handlers. The physical alarm stays armed as the
|
|
21
|
+
* eviction fallback. Off by default.
|
|
22
|
+
*/
|
|
23
|
+
liveTimers?: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Called each time an alarm run fails and the scheduler re-arms its own
|
|
26
|
+
* fallback alarm instead of letting the platform retry, for example to send
|
|
27
|
+
* the error to an external service. Best effort: it runs after the fallback is
|
|
28
|
+
* armed, a returned promise is passed to `waitUntil`, and a throw or rejection
|
|
29
|
+
* is only logged, so it never changes scheduling.
|
|
30
|
+
*/
|
|
31
|
+
onAlarmRetry?: (info: AlarmRetryInfo) => void | Promise<void>;
|
|
32
|
+
};
|
|
33
|
+
/** Details of a failed alarm run passed to `onAlarmRetry`. */
|
|
34
|
+
export type AlarmRetryInfo = {
|
|
35
|
+
error: SerializedError;
|
|
36
|
+
/** Delay until the re-armed fallback alarm fires. */
|
|
37
|
+
retryInMs: number;
|
|
38
|
+
/** The platform's retry count for this alarm invocation, when it provided one. */
|
|
39
|
+
retryCount?: number;
|
|
40
|
+
/** Whether the platform marked this invocation as a retry, when it provided one. */
|
|
41
|
+
isRetry?: boolean;
|
|
42
|
+
/** The owning Durable Object's ID, when the context carries one. */
|
|
43
|
+
objectId?: string;
|
|
16
44
|
};
|
|
17
45
|
/** Configuration for a scheduler shared by several helpers. */
|
|
18
46
|
export type DurabilitySchedulerConfig = DurabilitySchedulerOptions & {
|
|
@@ -34,6 +62,12 @@ export type Participant = {
|
|
|
34
62
|
runDue(now: number, info: AlarmInvocationInfo | undefined): Promise<void>;
|
|
35
63
|
/** Earliest due time, for tables without `status` and `next_attempt_at` columns. */
|
|
36
64
|
nextPendingAt?(sql: SqlStorage): number | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* True while the participant runs its own due work from an alarm invocation
|
|
67
|
+
* it keeps open. The in-memory timer then ignores its pending time, which only
|
|
68
|
+
* arms the physical alarm as the eviction fallback.
|
|
69
|
+
*/
|
|
70
|
+
holding?(): boolean;
|
|
37
71
|
};
|
|
38
72
|
/**
|
|
39
73
|
* Called after a record may have settled. Successful completions pass the
|
|
@@ -60,6 +94,8 @@ export type ExecutionSpec = {
|
|
|
60
94
|
active: Map<string, ActiveExecution>;
|
|
61
95
|
policy: AttemptPolicy;
|
|
62
96
|
emit: Emit;
|
|
97
|
+
/** When true, the completed event carries the handler result; omitted by default to keep results out of telemetry. */
|
|
98
|
+
includeResult?: boolean;
|
|
63
99
|
exhaustedError: () => Error;
|
|
64
100
|
timeoutError: () => Error;
|
|
65
101
|
invoke: (attempt: number, signal: AbortSignal) => unknown;
|
|
@@ -83,6 +119,8 @@ export declare class Engine {
|
|
|
83
119
|
private readonly alarmConcurrency;
|
|
84
120
|
private readonly alarmMinDelayMs;
|
|
85
121
|
private readonly alarmHandoffMs;
|
|
122
|
+
readonly liveTimers: boolean;
|
|
123
|
+
private readonly onAlarmRetry;
|
|
86
124
|
private readonly participants;
|
|
87
125
|
private readonly migrated;
|
|
88
126
|
private availablePermits;
|
|
@@ -90,6 +128,7 @@ export declare class Engine {
|
|
|
90
128
|
private alarmRefresh;
|
|
91
129
|
private earlyAlarm;
|
|
92
130
|
private readonly watchers;
|
|
131
|
+
private readonly activeByTable;
|
|
93
132
|
constructor(context: DurabilityContext, options: DurabilitySchedulerOptions);
|
|
94
133
|
register(participant: Participant): void;
|
|
95
134
|
waitUntil(promise: Promise<unknown>, onError: (error: unknown) => void): void;
|
|
@@ -111,7 +150,19 @@ export declare class Engine {
|
|
|
111
150
|
* transaction, then publishes the in-memory timer only after commit.
|
|
112
151
|
*/
|
|
113
152
|
transaction<T>(write: (sql: SqlStorage) => T): Promise<T>;
|
|
114
|
-
|
|
153
|
+
private trackActive;
|
|
154
|
+
private releaseActive;
|
|
155
|
+
/**
|
|
156
|
+
* The earliest pending time for one participant: `physical` arms the physical
|
|
157
|
+
* alarm (eviction fallback) and includes active work; `timer` drives the
|
|
158
|
+
* in-memory timer and, under liveTimers, skips work that is already executing
|
|
159
|
+
* so a still-due active row cannot re-fire a zero-delay timer in a loop.
|
|
160
|
+
*/
|
|
161
|
+
private pendingTimes;
|
|
162
|
+
/**
|
|
163
|
+
* Arms the physical alarm from the earliest pending time (the eviction
|
|
164
|
+
* fallback) and returns the earliest time the in-memory timer should fire.
|
|
165
|
+
*/
|
|
115
166
|
private reconcile;
|
|
116
167
|
private scheduleEarlyAlarm;
|
|
117
168
|
/** Coalesces concurrent refresh requests into one reconciliation transaction. */
|
|
@@ -135,6 +186,7 @@ export declare class Engine {
|
|
|
135
186
|
* because the platform drops an alarm after a few failed deliveries.
|
|
136
187
|
*/
|
|
137
188
|
alarm(info?: AlarmInvocationInfo): Promise<void>;
|
|
189
|
+
private notifyAlarmRetry;
|
|
138
190
|
/**
|
|
139
191
|
* Runs every participant's due work, hands off to a fresh alarm if the
|
|
140
192
|
* invocation approaches its wall-time limit, and re-arms the physical alarm.
|
package/dist/scheduler.js
CHANGED
|
@@ -2,12 +2,17 @@ import { z } from 'zod';
|
|
|
2
2
|
import { trace, traceMethods, traceStorage } from './dev.js';
|
|
3
3
|
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from './errors.js';
|
|
4
4
|
import { createEmitter, } from './lifecycle.js';
|
|
5
|
-
import { migrateCapability } from './migrations.js';
|
|
5
|
+
import { isSchemaCurrent, migrateCapability, } from './migrations.js';
|
|
6
6
|
import { assertPositiveInteger } from './policy.js';
|
|
7
|
+
import { deleteMemos, deleteStaleMemos } from './memoize.js';
|
|
7
8
|
import { claimAttempt, exhaustAttempts, updateAttempt, } from './storage.js';
|
|
8
9
|
const minAlarmDelayMs = 1000;
|
|
9
10
|
const maxAlarmDelayMs = 15000;
|
|
10
|
-
|
|
11
|
+
// setTimeout treats delays past this as zero, so anything further out can only
|
|
12
|
+
// be served by the physical alarm fallback.
|
|
13
|
+
const maxTimerMs = 2 ** 31 - 1;
|
|
14
|
+
const pendingHeadRowSchema = z.object({
|
|
15
|
+
pk: z.string(),
|
|
11
16
|
next_attempt_at: z.number().int().nonnegative(),
|
|
12
17
|
});
|
|
13
18
|
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
@@ -59,12 +64,21 @@ export class Engine {
|
|
|
59
64
|
this.migrated = new Set();
|
|
60
65
|
this.permitWaiters = [];
|
|
61
66
|
this.watchers = new Map();
|
|
67
|
+
// Keys whose attempt is currently executing, per table. The in-memory timer
|
|
68
|
+
// skips these so a still-due active row cannot drive a zero-delay timer loop.
|
|
69
|
+
this.activeByTable = new Map();
|
|
62
70
|
this.context = context;
|
|
63
71
|
this.storage = traceStorage(context.storage);
|
|
64
72
|
this.sql = this.storage.sql;
|
|
65
73
|
this.alarmConcurrency = options.alarmConcurrency ?? 10;
|
|
66
74
|
this.alarmMinDelayMs = options.alarmMinDelayMs ?? 15000;
|
|
67
75
|
this.alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
|
|
76
|
+
this.liveTimers = options.liveTimers ?? false;
|
|
77
|
+
this.onAlarmRetry = options.onAlarmRetry;
|
|
78
|
+
if (this.onAlarmRetry !== undefined &&
|
|
79
|
+
typeof this.onAlarmRetry !== 'function') {
|
|
80
|
+
throw new TypeError('onAlarmRetry must be a function');
|
|
81
|
+
}
|
|
68
82
|
assertPositiveInteger('alarmConcurrency', this.alarmConcurrency);
|
|
69
83
|
assertPositiveInteger('alarmMinDelayMs', this.alarmMinDelayMs);
|
|
70
84
|
if (this.alarmMinDelayMs < minAlarmDelayMs ||
|
|
@@ -110,7 +124,9 @@ export class Engine {
|
|
|
110
124
|
if (this.migrated.has(capability)) {
|
|
111
125
|
return;
|
|
112
126
|
}
|
|
113
|
-
|
|
127
|
+
if (!isSchemaCurrent(this.sql, capability, migrations)) {
|
|
128
|
+
migrateCapability(this.storage, capability, migrations, undefined);
|
|
129
|
+
}
|
|
114
130
|
this.migrated.add(capability);
|
|
115
131
|
}
|
|
116
132
|
/** Runs synchronous writes atomically without touching the physical alarm. */
|
|
@@ -131,39 +147,91 @@ export class Engine {
|
|
|
131
147
|
this.scheduleEarlyAlarm(nextPendingAt);
|
|
132
148
|
return result;
|
|
133
149
|
}
|
|
134
|
-
|
|
150
|
+
trackActive(table, key) {
|
|
151
|
+
let keys = this.activeByTable.get(table);
|
|
152
|
+
if (!keys) {
|
|
153
|
+
keys = new Set();
|
|
154
|
+
this.activeByTable.set(table, keys);
|
|
155
|
+
}
|
|
156
|
+
keys.add(key);
|
|
157
|
+
}
|
|
158
|
+
releaseActive(table, key) {
|
|
159
|
+
const keys = this.activeByTable.get(table);
|
|
160
|
+
if (!keys) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
keys.delete(key);
|
|
164
|
+
if (keys.size === 0) {
|
|
165
|
+
this.activeByTable.delete(table);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* The earliest pending time for one participant: `physical` arms the physical
|
|
170
|
+
* alarm (eviction fallback) and includes active work; `timer` drives the
|
|
171
|
+
* in-memory timer and, under liveTimers, skips work that is already executing
|
|
172
|
+
* so a still-due active row cannot re-fire a zero-delay timer in a loop.
|
|
173
|
+
*/
|
|
174
|
+
pendingTimes(participant) {
|
|
175
|
+
if (participant.nextPendingAt) {
|
|
176
|
+
const at = participant.nextPendingAt(this.sql);
|
|
177
|
+
if (at === undefined) {
|
|
178
|
+
return {};
|
|
179
|
+
}
|
|
180
|
+
return participant.holding?.()
|
|
181
|
+
? { physical: at }
|
|
182
|
+
: { physical: at, timer: at };
|
|
183
|
+
}
|
|
184
|
+
const active = this.activeByTable.get(participant.table.name);
|
|
185
|
+
const limit = this.liveTimers ? (active?.size ?? 0) + 1 : 1;
|
|
186
|
+
const rows = pendingHeadRowSchema.array().parse(this.sql
|
|
187
|
+
.exec(`SELECT ${participant.table.key} AS pk, next_attempt_at
|
|
188
|
+
FROM ${participant.table.name}
|
|
189
|
+
WHERE status = 'pending'
|
|
190
|
+
ORDER BY next_attempt_at ASC
|
|
191
|
+
LIMIT ?`, limit)
|
|
192
|
+
.toArray());
|
|
193
|
+
const physical = rows[0]?.next_attempt_at;
|
|
194
|
+
if (!this.liveTimers) {
|
|
195
|
+
if (physical === undefined) {
|
|
196
|
+
return {};
|
|
197
|
+
}
|
|
198
|
+
return { physical, timer: physical };
|
|
199
|
+
}
|
|
200
|
+
const eligible = rows.find((row) => !active?.has(row.pk));
|
|
201
|
+
return { physical, timer: eligible?.next_attempt_at };
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Arms the physical alarm from the earliest pending time (the eviction
|
|
205
|
+
* fallback) and returns the earliest time the in-memory timer should fire.
|
|
206
|
+
*/
|
|
135
207
|
async reconcile(physicalAlarm) {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
208
|
+
const physicalTimes = [];
|
|
209
|
+
const timerTimes = [];
|
|
210
|
+
for (const participant of this.participants) {
|
|
211
|
+
const { physical, timer } = this.pendingTimes(participant);
|
|
212
|
+
if (physical !== undefined) {
|
|
213
|
+
physicalTimes.push(physical);
|
|
140
214
|
}
|
|
141
|
-
|
|
142
|
-
.
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
ORDER BY next_attempt_at ASC
|
|
147
|
-
LIMIT 1`)
|
|
148
|
-
.toArray())
|
|
149
|
-
.map((row) => row.next_attempt_at);
|
|
150
|
-
});
|
|
151
|
-
const nextAlarmAt = scheduled.length === 0 ? undefined : Math.min(...scheduled);
|
|
215
|
+
if (timer !== undefined) {
|
|
216
|
+
timerTimes.push(timer);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
const physicalNextAt = physicalTimes.length === 0 ? undefined : Math.min(...physicalTimes);
|
|
152
220
|
const currentAlarm = await physicalAlarm.getAlarm();
|
|
153
|
-
if (
|
|
221
|
+
if (physicalNextAt === undefined) {
|
|
154
222
|
if (currentAlarm !== null) {
|
|
155
223
|
await physicalAlarm.deleteAlarm();
|
|
156
224
|
}
|
|
157
225
|
return undefined;
|
|
158
226
|
}
|
|
159
|
-
const target = Math.max(
|
|
227
|
+
const target = Math.max(physicalNextAt, Date.now() + this.alarmMinDelayMs);
|
|
160
228
|
// Preserve a fallback inside this range so frequent reconciliation cannot push it later.
|
|
161
229
|
if (currentAlarm === null ||
|
|
162
|
-
currentAlarm <
|
|
230
|
+
currentAlarm < physicalNextAt ||
|
|
163
231
|
currentAlarm > target) {
|
|
164
232
|
await physicalAlarm.setAlarm(target);
|
|
165
233
|
}
|
|
166
|
-
return
|
|
234
|
+
return timerTimes.length === 0 ? undefined : Math.min(...timerTimes);
|
|
167
235
|
}
|
|
168
236
|
scheduleEarlyAlarm(timestamp) {
|
|
169
237
|
if (this.earlyAlarm?.timestamp === timestamp) {
|
|
@@ -173,8 +241,18 @@ export class Engine {
|
|
|
173
241
|
clearTimeout(this.earlyAlarm.timer);
|
|
174
242
|
this.earlyAlarm = undefined;
|
|
175
243
|
}
|
|
176
|
-
if (timestamp === undefined
|
|
177
|
-
|
|
244
|
+
if (timestamp === undefined) {
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const delay = timestamp - Date.now();
|
|
248
|
+
// Without liveTimers the in-memory timer only covers the sub-min-delay
|
|
249
|
+
// window; the physical alarm handles everything beyond it. With liveTimers
|
|
250
|
+
// it fires at the exact due time no matter how far out, except past the
|
|
251
|
+
// timer's range where only the physical fallback can serve it.
|
|
252
|
+
if (!this.liveTimers && delay >= this.alarmMinDelayMs) {
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (delay > maxTimerMs) {
|
|
178
256
|
return;
|
|
179
257
|
}
|
|
180
258
|
const timer = setTimeout(() => {
|
|
@@ -186,7 +264,7 @@ export class Engine {
|
|
|
186
264
|
const onError = (error) => reportFailure('durability.background_execution.failed', { scheduledTime: timestamp }, error);
|
|
187
265
|
const execution = this.alarm().catch(onError);
|
|
188
266
|
this.waitUntil(execution, onError);
|
|
189
|
-
}, Math.max(
|
|
267
|
+
}, Math.max(delay, 0));
|
|
190
268
|
this.earlyAlarm = { timestamp, timer };
|
|
191
269
|
}
|
|
192
270
|
/** Coalesces concurrent refresh requests into one reconciliation transaction. */
|
|
@@ -217,6 +295,7 @@ export class Engine {
|
|
|
217
295
|
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE created_at < ?`, before)
|
|
218
296
|
.toArray()[0]);
|
|
219
297
|
sql.exec(`DELETE FROM ${table.name} WHERE created_at < ?`, before);
|
|
298
|
+
deleteStaleMemos(sql, table);
|
|
220
299
|
return count;
|
|
221
300
|
});
|
|
222
301
|
if (removed > 0) {
|
|
@@ -308,6 +387,35 @@ export class Engine {
|
|
|
308
387
|
throw error;
|
|
309
388
|
}
|
|
310
389
|
reportFailure('durability.alarm.failed', { retryInMs: this.alarmMinDelayMs }, error);
|
|
390
|
+
this.notifyAlarmRetry(error, info);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
notifyAlarmRetry(failure, invocation) {
|
|
394
|
+
const hook = this.onAlarmRetry;
|
|
395
|
+
if (!hook) {
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
const objectId = this.objectId();
|
|
399
|
+
const info = {
|
|
400
|
+
error: serializeError(failure),
|
|
401
|
+
retryInMs: this.alarmMinDelayMs,
|
|
402
|
+
...(invocation === undefined
|
|
403
|
+
? {}
|
|
404
|
+
: {
|
|
405
|
+
retryCount: invocation.retryCount,
|
|
406
|
+
isRetry: invocation.isRetry,
|
|
407
|
+
}),
|
|
408
|
+
...(objectId === undefined ? {} : { objectId }),
|
|
409
|
+
};
|
|
410
|
+
const report = (hookError) => reportFailure('durability.alarm.retry_hook_failed', { retryInMs: info.retryInMs }, hookError);
|
|
411
|
+
try {
|
|
412
|
+
const result = hook(info);
|
|
413
|
+
if (result !== undefined) {
|
|
414
|
+
this.waitUntil(Promise.resolve(result).catch(report), report);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
catch (hookError) {
|
|
418
|
+
report(hookError);
|
|
311
419
|
}
|
|
312
420
|
}
|
|
313
421
|
/**
|
|
@@ -377,6 +485,7 @@ export class Engine {
|
|
|
377
485
|
settled,
|
|
378
486
|
};
|
|
379
487
|
spec.active.set(key, activeExecution);
|
|
488
|
+
this.trackActive(spec.table.name, key);
|
|
380
489
|
let releasePermit = eagerPermit;
|
|
381
490
|
let handlerStarted = false;
|
|
382
491
|
let handlerSettled = false;
|
|
@@ -421,7 +530,14 @@ export class Engine {
|
|
|
421
530
|
}, policy.attemptTimeoutMs);
|
|
422
531
|
});
|
|
423
532
|
const result = await Promise.race([handlerResult, timeoutResult]);
|
|
424
|
-
|
|
533
|
+
// One transaction: a failed memo cleanup must undo the completion, not orphan the memos.
|
|
534
|
+
const timestamp = this.transactionSync((sql) => {
|
|
535
|
+
const completedAt = spec.complete(result, attempt);
|
|
536
|
+
if (completedAt !== undefined) {
|
|
537
|
+
deleteMemos(sql, spec.table, key);
|
|
538
|
+
}
|
|
539
|
+
return completedAt;
|
|
540
|
+
});
|
|
425
541
|
if (timestamp !== undefined) {
|
|
426
542
|
this.notify(spec.table, key, row);
|
|
427
543
|
spec.emit({
|
|
@@ -431,7 +547,7 @@ export class Engine {
|
|
|
431
547
|
attempt,
|
|
432
548
|
durationMs: Math.max(0, timestamp - startedAt),
|
|
433
549
|
outcome: 'completed',
|
|
434
|
-
result,
|
|
550
|
+
...(spec.includeResult ? { result } : {}),
|
|
435
551
|
});
|
|
436
552
|
}
|
|
437
553
|
}
|
|
@@ -451,13 +567,25 @@ export class Engine {
|
|
|
451
567
|
if (spec.active.get(key) === activeExecution) {
|
|
452
568
|
spec.active.delete(key);
|
|
453
569
|
}
|
|
570
|
+
this.releaseActive(spec.table.name, key);
|
|
571
|
+
// Re-arm the in-memory timer now that this key is eligible again, so work
|
|
572
|
+
// the timer skipped while it executed is picked up at its exact time.
|
|
573
|
+
if (this.liveTimers) {
|
|
574
|
+
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
575
|
+
}
|
|
454
576
|
});
|
|
455
577
|
return execution;
|
|
456
578
|
}
|
|
457
579
|
async failExhausted(spec) {
|
|
458
580
|
const { key, row, policy } = spec;
|
|
459
581
|
const exhausted = serializeError(spec.exhaustedError());
|
|
460
|
-
const attempt = await this.transaction((sql) =>
|
|
582
|
+
const attempt = await this.transaction((sql) => {
|
|
583
|
+
const exhaustedAttempt = exhaustAttempts(sql, spec.table, key, row.generation_id, policy.maxAttempts, exhausted);
|
|
584
|
+
if (exhaustedAttempt !== undefined) {
|
|
585
|
+
deleteMemos(sql, spec.table, key);
|
|
586
|
+
}
|
|
587
|
+
return exhaustedAttempt;
|
|
588
|
+
});
|
|
461
589
|
if (attempt === undefined) {
|
|
462
590
|
return;
|
|
463
591
|
}
|
|
@@ -475,7 +603,13 @@ export class Engine {
|
|
|
475
603
|
const { key, row } = spec;
|
|
476
604
|
const decision = decideRetry(spec, attempt, caught);
|
|
477
605
|
const serializedError = serializeError(decision.error);
|
|
478
|
-
const updated = await this.transaction((sql) =>
|
|
606
|
+
const updated = await this.transaction((sql) => {
|
|
607
|
+
const owned = updateAttempt(sql, spec.table, { key, generation: row.generation_id, attempt }, 'status = ?, next_attempt_at = ?, last_error = ?, last_error_name = ?', decision.terminal ? 'failed' : 'pending', decision.nextAttemptAt, serializedError.message, serializedError.name);
|
|
608
|
+
if (owned && decision.terminal) {
|
|
609
|
+
deleteMemos(sql, spec.table, key);
|
|
610
|
+
}
|
|
611
|
+
return owned;
|
|
612
|
+
});
|
|
479
613
|
if (!updated) {
|
|
480
614
|
return;
|
|
481
615
|
}
|
|
@@ -208,7 +208,9 @@ export class QueueStore {
|
|
|
208
208
|
this.sql.exec(`INSERT INTO durability_semaphore_queue_keys (key, next_request_at, queued)
|
|
209
209
|
VALUES (?, ?, ?)
|
|
210
210
|
ON CONFLICT(key) DO UPDATE SET
|
|
211
|
-
next_request_at = excluded.next_request_at, queued = excluded.queued
|
|
211
|
+
next_request_at = excluded.next_request_at, queued = excluded.queued
|
|
212
|
+
WHERE next_request_at != excluded.next_request_at
|
|
213
|
+
OR queued != excluded.queued`, row.key, row.next_request_at, row.queued);
|
|
212
214
|
}
|
|
213
215
|
removeKey(key) {
|
|
214
216
|
this.sql.exec('DELETE FROM durability_semaphore_queue_keys WHERE key = ?', key);
|
|
@@ -221,24 +223,26 @@ export class QueueStore {
|
|
|
221
223
|
.toArray());
|
|
222
224
|
}
|
|
223
225
|
nextPendingAt(checksFinished) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
226
|
+
// An idle lease is due when it stops at its expiry, not before.
|
|
227
|
+
const at = this.sql
|
|
228
|
+
.exec(`SELECT MIN(at) AS at FROM (
|
|
229
|
+
SELECT (
|
|
230
|
+
SELECT next_request_at FROM durability_semaphore_queue_keys
|
|
231
|
+
WHERE queued > 0 ORDER BY next_request_at ASC LIMIT 1
|
|
232
|
+
) AS at
|
|
233
|
+
UNION ALL
|
|
234
|
+
SELECT (
|
|
235
|
+
SELECT next_attempt_at FROM durability_semaphore_queue_items
|
|
236
|
+
WHERE status = 'admitted' AND (notified = 0 OR ?)
|
|
237
|
+
ORDER BY next_attempt_at ASC LIMIT 1
|
|
238
|
+
) AS at
|
|
239
|
+
UNION ALL
|
|
240
|
+
SELECT MIN(CASE WHEN released < drawn THEN renew_at
|
|
241
|
+
ELSE MAX(renew_at, expires_at) END) AS at
|
|
242
|
+
FROM durability_semaphore_queue_leases WHERE confirmed = 0
|
|
243
|
+
)`, checksFinished ? 1 : 0)
|
|
244
|
+
.toArray()[0]?.['at'];
|
|
245
|
+
return typeof at === 'number' ? at : undefined;
|
|
242
246
|
}
|
|
243
247
|
items(query, ...bindings) {
|
|
244
248
|
return queueItemRowSchema
|
package/dist/storage.d.ts
CHANGED
|
@@ -12,8 +12,12 @@ export type AttemptRef = {
|
|
|
12
12
|
generation: string;
|
|
13
13
|
attempt: number;
|
|
14
14
|
};
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
export declare const pendingGeneration: (table: DurableTable) => string;
|
|
16
|
+
/**
|
|
17
|
+
* Starts the next attempt, optionally persisting its deadline in the same write.
|
|
18
|
+
* Returns undefined when the generation is gone or out of attempts.
|
|
19
|
+
*/
|
|
20
|
+
export declare const claimAttempt: (sql: SqlStorage, table: DurableTable, key: string, generation: string, maxAttempts: number, nextAttemptAt?: number) => number | undefined;
|
|
17
21
|
/**
|
|
18
22
|
* Applies `assignments` (a SQL `SET` list) to the record if the attempt still
|
|
19
23
|
* owns it, and reports whether it did.
|
package/dist/storage.js
CHANGED
|
@@ -4,14 +4,20 @@ const attemptRowSchema = z.object({
|
|
|
4
4
|
});
|
|
5
5
|
// Every write is conditional on the record's generation and pending status, so a
|
|
6
6
|
// stale execution can never overwrite a replacement or a later attempt.
|
|
7
|
-
const pendingGeneration = (table) => `${table.key} = ? AND generation_id = ? AND status = 'pending'`;
|
|
8
|
-
/**
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
export const pendingGeneration = (table) => `${table.key} = ? AND generation_id = ? AND status = 'pending'`;
|
|
8
|
+
/**
|
|
9
|
+
* Starts the next attempt, optionally persisting its deadline in the same write.
|
|
10
|
+
* Returns undefined when the generation is gone or out of attempts.
|
|
11
|
+
*/
|
|
12
|
+
export const claimAttempt = (sql, table, key, generation, maxAttempts, nextAttemptAt) => {
|
|
13
|
+
const deadlineAssignment = nextAttemptAt === undefined ? '' : ', next_attempt_at = ?';
|
|
14
|
+
return attemptRowSchema.optional().parse(sql
|
|
15
|
+
.exec(`UPDATE ${table.name}
|
|
16
|
+
SET attempt = attempt + 1${deadlineAssignment}
|
|
12
17
|
WHERE ${pendingGeneration(table)} AND attempt < ?
|
|
13
|
-
RETURNING attempt`, key, generation, maxAttempts)
|
|
14
|
-
|
|
18
|
+
RETURNING attempt`, ...(nextAttemptAt === undefined ? [] : [nextAttemptAt]), key, generation, maxAttempts)
|
|
19
|
+
.toArray()[0])?.attempt;
|
|
20
|
+
};
|
|
15
21
|
/**
|
|
16
22
|
* Applies `assignments` (a SQL `SET` list) to the record if the attempt still
|
|
17
23
|
* owns it, and reports whether it did.
|
package/dist/tracked-source.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export type TrackedState<Result> = {
|
|
|
10
10
|
attempt?: number;
|
|
11
11
|
nextAttemptAt?: number;
|
|
12
12
|
lastError?: string | null;
|
|
13
|
+
/** Keys whose memoized results the next attempt reuses, in the order they were saved. */
|
|
14
|
+
memoized?: string[];
|
|
13
15
|
} | {
|
|
14
16
|
status: 'completed';
|
|
15
17
|
result: Result;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "durability",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.1.0",
|
|
4
4
|
"description": "Alarm-backed durable operations and typed WebSocket sync for Cloudflare Durable Objects",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "ISC",
|
|
@@ -86,6 +86,7 @@
|
|
|
86
86
|
"@durability/typescript-config": "0.0.0"
|
|
87
87
|
},
|
|
88
88
|
"scripts": {
|
|
89
|
+
"bench:load": "vitest run --config vitest.bench.config.ts",
|
|
89
90
|
"build": "rm -rf dist && tsc -p tsconfig.build.json",
|
|
90
91
|
"lint": "oxlint",
|
|
91
92
|
"test": "vitest run && vitest run --config vitest.workers.config.ts",
|