durability 5.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -7
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +23 -6
- package/dist/{durability-alarms.js → alarms/index.js} +96 -52
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/errors.d.ts +15 -0
- package/dist/errors.js +24 -0
- package/dist/experimental.d.ts +11 -8
- package/dist/experimental.js +8 -5
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +55 -49
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +15 -3
- package/dist/{log.js → log/index.js} +152 -124
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +14 -11
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +1 -103
- package/dist/migrations.js +1 -570
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +63 -46
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +18 -63
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/schedule/index.d.ts +165 -0
- package/dist/schedule/index.js +667 -0
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +122 -22
- package/dist/scheduler/index.js +925 -0
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +2 -2
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/{semaphore-protocol.d.ts → semaphore/protocol.d.ts} +16 -8
- package/dist/{semaphore-protocol.js → semaphore/protocol.js} +37 -14
- package/dist/{semaphore-queue-store.d.ts → semaphore/queue-store.d.ts} +12 -6
- package/dist/{semaphore-queue-store.js → semaphore/queue-store.js} +87 -75
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +51 -24
- package/dist/{semaphore-store.d.ts → semaphore/store.d.ts} +8 -3
- package/dist/{semaphore-store.js → semaphore/store.js} +35 -33
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +9 -2
- package/dist/storage.js +24 -11
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +92 -58
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +2 -2
- package/package.json +11 -11
- package/dist/scheduler.js +0 -577
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{topology-snapshot.d.ts → topology/snapshot.d.ts} +0 -0
- /package/dist/{topology-snapshot.js → topology/snapshot.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -0,0 +1,925 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { trace, traceMethods, traceStorage } from '../dev.js';
|
|
3
|
+
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from '../errors.js';
|
|
4
|
+
import { createEmitter, } from '../lifecycle.js';
|
|
5
|
+
import { isSchemaCurrent, migrateCapability, } from '../migrations.js';
|
|
6
|
+
import { assertPositiveInteger } from '../policy.js';
|
|
7
|
+
import { deleteMemos, deleteStaleMemos } from '../memoize/index.js';
|
|
8
|
+
import { channelBindings, channelClause, claimAttempt, exhaustAttempts, tableOwner, updateAttempt, } from '../storage.js';
|
|
9
|
+
const minAlarmDelayMs = 1000;
|
|
10
|
+
const failedRunRetryMs = 1000;
|
|
11
|
+
const defaultAlarmConcurrency = 10;
|
|
12
|
+
const schedulerOptionKeys = [
|
|
13
|
+
'alarmConcurrency',
|
|
14
|
+
'alarmMinDelayMs',
|
|
15
|
+
'alarmHandoffMs',
|
|
16
|
+
'liveTimers',
|
|
17
|
+
'onAlarmRetry',
|
|
18
|
+
];
|
|
19
|
+
const maxAlarmDelayMs = 15000;
|
|
20
|
+
// setTimeout treats delays past this as zero, so anything further out can only
|
|
21
|
+
// be served by the physical alarm fallback.
|
|
22
|
+
const maxTimerMs = 2 ** 31 - 1;
|
|
23
|
+
const pendingHeadRowSchema = z.object({
|
|
24
|
+
pk: z.string(),
|
|
25
|
+
channel: z.string().optional(),
|
|
26
|
+
next_attempt_at: z.number().int().nonnegative(),
|
|
27
|
+
});
|
|
28
|
+
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
29
|
+
/** The channel every helper uses when it is not given one. */
|
|
30
|
+
export const defaultChannel = 'default';
|
|
31
|
+
/** How long records of a channel no instance attached wait before they are looked at again. */
|
|
32
|
+
const unattachedChannelRetryMs = 60000;
|
|
33
|
+
const channelRowSchema = z.object({ channel: z.string() });
|
|
34
|
+
const activeKey = (channel, key) => JSON.stringify([channel ?? null, key]);
|
|
35
|
+
/** Rejects a channel name a helper would otherwise store rows under. */
|
|
36
|
+
export const assertChannel = (helper, channel) => {
|
|
37
|
+
if (typeof channel !== 'string' || channel.length === 0) {
|
|
38
|
+
throw new TypeError(`${helper} channel must be a non-empty string`);
|
|
39
|
+
}
|
|
40
|
+
return channel;
|
|
41
|
+
};
|
|
42
|
+
const decideRetry = (spec, attempt, error) => {
|
|
43
|
+
const terminal = isNonRetryable(error) ||
|
|
44
|
+
attempt >= spec.policy.maxAttempts ||
|
|
45
|
+
spec.isTerminal(error);
|
|
46
|
+
if (terminal) {
|
|
47
|
+
const timestamp = Date.now();
|
|
48
|
+
return { error, terminal, timestamp, nextAttemptAt: timestamp };
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
const rawDelay = spec.policy.delay(attempt);
|
|
52
|
+
if (!Number.isFinite(rawDelay) || rawDelay < 0) {
|
|
53
|
+
throw new DurableRetryPolicyError(spec.label);
|
|
54
|
+
}
|
|
55
|
+
const delay = Math.round(rawDelay);
|
|
56
|
+
const timestamp = Date.now();
|
|
57
|
+
if (!Number.isSafeInteger(delay) ||
|
|
58
|
+
!Number.isSafeInteger(timestamp + delay)) {
|
|
59
|
+
throw new DurableRetryPolicyError(spec.label);
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
error,
|
|
63
|
+
terminal: false,
|
|
64
|
+
timestamp,
|
|
65
|
+
nextAttemptAt: timestamp + delay,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
catch (policyError) {
|
|
69
|
+
const timestamp = Date.now();
|
|
70
|
+
return {
|
|
71
|
+
error: isErrorInstance(policyError, DurableRetryPolicyError)
|
|
72
|
+
? policyError
|
|
73
|
+
: new DurableRetryPolicyError(spec.label, policyError),
|
|
74
|
+
terminal: true,
|
|
75
|
+
timestamp,
|
|
76
|
+
nextAttemptAt: timestamp,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* Shared machinery behind {@link DurabilityScheduler}: transactions, the physical
|
|
82
|
+
* alarm, the concurrency pool, and the attempt lifecycle. Package-internal.
|
|
83
|
+
*/
|
|
84
|
+
export class Engine {
|
|
85
|
+
constructor(context, options) {
|
|
86
|
+
this.alarmConcurrency = defaultAlarmConcurrency;
|
|
87
|
+
this.alarmMinDelayMs = 15000;
|
|
88
|
+
this.alarmHandoffMs = 14 * 60000;
|
|
89
|
+
this.liveTimers = false;
|
|
90
|
+
// Only options someone set can conflict; an omitted one takes any value.
|
|
91
|
+
this.explicitOptions = {};
|
|
92
|
+
this.participants = [];
|
|
93
|
+
this.channelsByTable = new Map();
|
|
94
|
+
this.claimedChannels = new Map();
|
|
95
|
+
this.reportedMissingChannels = new Set();
|
|
96
|
+
this.migrated = new Set();
|
|
97
|
+
this.availablePermits = defaultAlarmConcurrency;
|
|
98
|
+
this.permitWaiters = [];
|
|
99
|
+
this.alarmRefreshRequested = false;
|
|
100
|
+
this.failedRuns = 0;
|
|
101
|
+
this.watchers = new Map();
|
|
102
|
+
// Keys whose attempt is currently executing, per table. The in-memory timer
|
|
103
|
+
// skips these so a still-due active row cannot drive a zero-delay timer loop.
|
|
104
|
+
this.activeByTable = new Map();
|
|
105
|
+
this.context = context;
|
|
106
|
+
this.storage = traceStorage(context.storage);
|
|
107
|
+
this.sql = this.storage.sql;
|
|
108
|
+
this.configure(options);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Joins one more helper or scheduler handle to this object's scheduler. The
|
|
112
|
+
* first context that can `waitUntil` is kept, so a bare `{ storage }` context
|
|
113
|
+
* attached first does not hide it from the rest.
|
|
114
|
+
*/
|
|
115
|
+
attach(context, options) {
|
|
116
|
+
if (this.context.waitUntil === undefined &&
|
|
117
|
+
context.waitUntil !== undefined) {
|
|
118
|
+
this.context = context;
|
|
119
|
+
}
|
|
120
|
+
this.configure(options);
|
|
121
|
+
}
|
|
122
|
+
configure(options) {
|
|
123
|
+
const merged = { ...this.explicitOptions };
|
|
124
|
+
for (const key of schedulerOptionKeys) {
|
|
125
|
+
const value = options[key];
|
|
126
|
+
if (value === undefined) {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
const current = this.explicitOptions[key];
|
|
130
|
+
if (current !== undefined && current !== value) {
|
|
131
|
+
throw new Error(`Scheduler option "${key}" is already set to a different value for this Durable Object`);
|
|
132
|
+
}
|
|
133
|
+
Object.assign(merged, { [key]: value });
|
|
134
|
+
}
|
|
135
|
+
const alarmConcurrency = merged.alarmConcurrency ?? defaultAlarmConcurrency;
|
|
136
|
+
const alarmMinDelayMs = merged.alarmMinDelayMs ?? 15000;
|
|
137
|
+
const alarmHandoffMs = merged.alarmHandoffMs ?? 14 * 60000;
|
|
138
|
+
if (merged.onAlarmRetry !== undefined &&
|
|
139
|
+
typeof merged.onAlarmRetry !== 'function') {
|
|
140
|
+
throw new TypeError('onAlarmRetry must be a function');
|
|
141
|
+
}
|
|
142
|
+
assertPositiveInteger('alarmConcurrency', alarmConcurrency);
|
|
143
|
+
assertPositiveInteger('alarmMinDelayMs', alarmMinDelayMs);
|
|
144
|
+
if (alarmMinDelayMs < minAlarmDelayMs ||
|
|
145
|
+
alarmMinDelayMs > maxAlarmDelayMs) {
|
|
146
|
+
throw new RangeError(`alarmMinDelayMs must be between ${minAlarmDelayMs} and ${maxAlarmDelayMs}`);
|
|
147
|
+
}
|
|
148
|
+
assertPositiveInteger('alarmHandoffMs', alarmHandoffMs);
|
|
149
|
+
// Permits already handed out stay out; raising the limit frees more now.
|
|
150
|
+
this.availablePermits += alarmConcurrency - this.alarmConcurrency;
|
|
151
|
+
this.alarmConcurrency = alarmConcurrency;
|
|
152
|
+
this.alarmMinDelayMs = alarmMinDelayMs;
|
|
153
|
+
this.alarmHandoffMs = alarmHandoffMs;
|
|
154
|
+
this.liveTimers = merged.liveTimers ?? false;
|
|
155
|
+
this.onAlarmRetry = merged.onAlarmRetry;
|
|
156
|
+
this.explicitOptions = merged;
|
|
157
|
+
while (this.availablePermits > 0 && this.permitWaiters.length > 0) {
|
|
158
|
+
const waiter = this.permitWaiters.shift();
|
|
159
|
+
this.availablePermits -= 1;
|
|
160
|
+
waiter(this.createRelease());
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
register(participant) {
|
|
164
|
+
const { name } = participant.table;
|
|
165
|
+
if (this.participants.some(({ table }) => table.name === name)) {
|
|
166
|
+
throw new Error(`A ${name} helper is already attached to this scheduler`);
|
|
167
|
+
}
|
|
168
|
+
this.participants.push(participant);
|
|
169
|
+
}
|
|
170
|
+
/** Reserves an instance identity, including for helpers that do not schedule work. */
|
|
171
|
+
claimChannel(kind, channel, helper) {
|
|
172
|
+
const claimed = this.claimedChannels.get(kind) ?? new Set();
|
|
173
|
+
if (claimed.has(channel)) {
|
|
174
|
+
throw new Error(`${helper} for channel "${channel}" is already attached to this scheduler`);
|
|
175
|
+
}
|
|
176
|
+
claimed.add(channel);
|
|
177
|
+
this.claimedChannels.set(kind, claimed);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Attaches one channel of a table several helper instances share. The first
|
|
181
|
+
* attachment registers the table; each run then runs every attached
|
|
182
|
+
* channel's due rows and retains the rows of channels nothing attached.
|
|
183
|
+
*/
|
|
184
|
+
registerChannel(participant) {
|
|
185
|
+
const { table, channel, helper } = participant;
|
|
186
|
+
let channels = this.channelsByTable.get(table.name);
|
|
187
|
+
if (!channels) {
|
|
188
|
+
const created = new Map();
|
|
189
|
+
this.register({
|
|
190
|
+
table: { name: table.name, key: table.key },
|
|
191
|
+
runDue: async (now, info) => {
|
|
192
|
+
await this.postponeUnattachedChannels(table, created);
|
|
193
|
+
await Promise.all([...created.values()].map((attached) => attached.runDue(now, info)));
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
this.channelsByTable.set(table.name, created);
|
|
197
|
+
channels = created;
|
|
198
|
+
}
|
|
199
|
+
this.claimChannel(table.name, channel, helper);
|
|
200
|
+
channels.set(channel, {
|
|
201
|
+
...participant,
|
|
202
|
+
table: { ...table, channel },
|
|
203
|
+
});
|
|
204
|
+
if (participant.nextPendingAt !== undefined) {
|
|
205
|
+
// Missing custom channels are dormant. Defer until construction finishes,
|
|
206
|
+
// then restore their alarm without requiring a new application write.
|
|
207
|
+
const onError = (error) => reportFailure('durability.channel_resume_failed', { table: table.name, channel }, error);
|
|
208
|
+
const resumed = Promise.resolve()
|
|
209
|
+
.then(() => {
|
|
210
|
+
if (participant.nextPendingAt?.(this.sql) !== undefined) {
|
|
211
|
+
return this.scheduleNextAlarm();
|
|
212
|
+
}
|
|
213
|
+
return undefined;
|
|
214
|
+
})
|
|
215
|
+
.catch(async (error) => {
|
|
216
|
+
if (await this.armFallbackAlarm()) {
|
|
217
|
+
this.retryFailedRun();
|
|
218
|
+
}
|
|
219
|
+
onError(error);
|
|
220
|
+
});
|
|
221
|
+
this.waitUntil(resumed, onError);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Due rows of a channel no instance attached, such as one a deploy removed
|
|
226
|
+
* or a library not yet constructed, would keep the alarm awake forever. They
|
|
227
|
+
* are kept and looked at again later, since their owner may return.
|
|
228
|
+
*/
|
|
229
|
+
async postponeUnattachedChannels(table, channels) {
|
|
230
|
+
const now = Date.now();
|
|
231
|
+
const customTiming = [...channels.values()].some((participant) => participant.nextPendingAt !== undefined);
|
|
232
|
+
const channelTables = new Set([
|
|
233
|
+
table.name,
|
|
234
|
+
...[...channels.values()].flatMap((participant) => participant.channelTables ?? []),
|
|
235
|
+
]);
|
|
236
|
+
const query = customTiming
|
|
237
|
+
? [...channelTables]
|
|
238
|
+
.map((name) => `SELECT channel FROM ${name}`)
|
|
239
|
+
.join(' UNION ')
|
|
240
|
+
: `SELECT DISTINCT channel FROM ${table.name}
|
|
241
|
+
WHERE status = 'pending' AND next_attempt_at <= ?`;
|
|
242
|
+
const unattached = channelRowSchema
|
|
243
|
+
.array()
|
|
244
|
+
.parse(this.sql.exec(query, ...(customTiming ? [] : [now])).toArray())
|
|
245
|
+
.map((row) => row.channel)
|
|
246
|
+
.filter((channel) => !channels.has(channel));
|
|
247
|
+
if (unattached.length === 0) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
for (const channel of unattached) {
|
|
251
|
+
const identity = JSON.stringify([table.name, channel]);
|
|
252
|
+
if (customTiming && this.reportedMissingChannels.has(identity)) {
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
this.reportedMissingChannels.add(identity);
|
|
256
|
+
reportFailure('durability.channel_missing', { table: table.name, channel }, new Error(`Channel "${channel}" of ${table.name} has no helper attached; retaining its records`));
|
|
257
|
+
}
|
|
258
|
+
// Custom timing may represent leases or recurring configurations, not pending
|
|
259
|
+
// attempts. Retain those records unchanged until their channel returns.
|
|
260
|
+
if (customTiming) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
await this.transaction((sql) => {
|
|
264
|
+
for (const channel of unattached) {
|
|
265
|
+
sql.exec(`UPDATE ${table.name} SET next_attempt_at = ?
|
|
266
|
+
WHERE channel = ? AND status = 'pending' AND next_attempt_at <= ?`, now + unattachedChannelRetryMs, channel, now);
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
waitUntil(promise, onError) {
|
|
271
|
+
if (!this.context.waitUntil) {
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
try {
|
|
275
|
+
this.context.waitUntil(promise);
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
onError(error);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* The owning Durable Object's ID. The public context type omits it, but the
|
|
283
|
+
* `DurableObjectState` applications pass at runtime carries it.
|
|
284
|
+
*/
|
|
285
|
+
objectId() {
|
|
286
|
+
const id = Reflect.get(this.context, 'id');
|
|
287
|
+
return id === undefined || id === null ? undefined : String(id);
|
|
288
|
+
}
|
|
289
|
+
emitter(hook) {
|
|
290
|
+
return createEmitter(hook, (promise) => this.context.waitUntil?.(promise));
|
|
291
|
+
}
|
|
292
|
+
/** Applies a helper's SQLite migrations. */
|
|
293
|
+
migrateSchema(capability, migrations) {
|
|
294
|
+
// Helpers of one kind on one scheduler share their tables; checking the
|
|
295
|
+
// schema again for each would only add statements to every construction.
|
|
296
|
+
if (this.migrated.has(capability)) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
if (!isSchemaCurrent(this.sql, capability, migrations)) {
|
|
300
|
+
migrateCapability(this.storage, capability, migrations, undefined);
|
|
301
|
+
}
|
|
302
|
+
this.migrated.add(capability);
|
|
303
|
+
}
|
|
304
|
+
/** Runs synchronous writes atomically without touching the physical alarm. */
|
|
305
|
+
transactionSync(write) {
|
|
306
|
+
return this.storage.transactionSync(() => write(this.sql));
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Runs the writes and reconciles the physical alarm in one storage
|
|
310
|
+
* transaction, then publishes the in-memory timer only after commit.
|
|
311
|
+
*/
|
|
312
|
+
async transaction(write) {
|
|
313
|
+
let nextPendingAt;
|
|
314
|
+
const result = await this.storage.transaction(async (transaction) => {
|
|
315
|
+
const value = write(this.sql);
|
|
316
|
+
nextPendingAt = await this.reconcile(transaction);
|
|
317
|
+
return value;
|
|
318
|
+
});
|
|
319
|
+
this.scheduleEarlyAlarm(nextPendingAt);
|
|
320
|
+
return result;
|
|
321
|
+
}
|
|
322
|
+
trackActive(table, key) {
|
|
323
|
+
let keys = this.activeByTable.get(table);
|
|
324
|
+
if (!keys) {
|
|
325
|
+
keys = new Set();
|
|
326
|
+
this.activeByTable.set(table, keys);
|
|
327
|
+
}
|
|
328
|
+
keys.add(key);
|
|
329
|
+
}
|
|
330
|
+
releaseActive(table, key) {
|
|
331
|
+
const keys = this.activeByTable.get(table);
|
|
332
|
+
if (!keys) {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
keys.delete(key);
|
|
336
|
+
if (keys.size === 0) {
|
|
337
|
+
this.activeByTable.delete(table);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* The earliest pending time for one participant: `physical` arms the physical
|
|
342
|
+
* alarm (eviction fallback) and includes active work; `timer` drives the
|
|
343
|
+
* in-memory timer and skips work that is already executing
|
|
344
|
+
* so a still-due active row cannot re-fire a zero-delay timer in a loop.
|
|
345
|
+
*/
|
|
346
|
+
pendingTimes(participant) {
|
|
347
|
+
const channels = this.channelsByTable.get(participant.table.name);
|
|
348
|
+
if (participant.table.channel === undefined &&
|
|
349
|
+
channels !== undefined &&
|
|
350
|
+
[...channels.values()].some((channel) => channel.nextPendingAt !== undefined)) {
|
|
351
|
+
let physical;
|
|
352
|
+
let timer;
|
|
353
|
+
for (const channel of channels.values()) {
|
|
354
|
+
const next = this.pendingTimes(channel);
|
|
355
|
+
if (next.physical !== undefined) {
|
|
356
|
+
physical = Math.min(physical ?? Infinity, next.physical);
|
|
357
|
+
}
|
|
358
|
+
if (next.timer !== undefined) {
|
|
359
|
+
timer = Math.min(timer ?? Infinity, next.timer);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return { physical, timer };
|
|
363
|
+
}
|
|
364
|
+
if (participant.nextPendingAt) {
|
|
365
|
+
const at = participant.nextPendingAt(this.sql);
|
|
366
|
+
if (at === undefined) {
|
|
367
|
+
return {};
|
|
368
|
+
}
|
|
369
|
+
return participant.holding?.()
|
|
370
|
+
? { physical: at }
|
|
371
|
+
: {
|
|
372
|
+
physical: at,
|
|
373
|
+
timer: participant.nextTimerAt
|
|
374
|
+
? participant.nextTimerAt(this.sql)
|
|
375
|
+
: at,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
const active = this.activeByTable.get(participant.table.name);
|
|
379
|
+
const partitioned = this.channelsByTable.has(participant.table.name);
|
|
380
|
+
const limit = (active?.size ?? 0) + 1;
|
|
381
|
+
const rows = pendingHeadRowSchema.array().parse(this.sql
|
|
382
|
+
.exec(`SELECT ${participant.table.key} AS pk, next_attempt_at${partitioned ? ', channel' : ''}
|
|
383
|
+
FROM ${participant.table.name}
|
|
384
|
+
WHERE status = 'pending'
|
|
385
|
+
ORDER BY next_attempt_at ASC
|
|
386
|
+
LIMIT ?`, limit)
|
|
387
|
+
.toArray());
|
|
388
|
+
const physical = rows[0]?.next_attempt_at;
|
|
389
|
+
const eligible = rows.find((row) => !active?.has(activeKey(row.channel, row.pk)));
|
|
390
|
+
return { physical, timer: eligible?.next_attempt_at };
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Arms the physical alarm from the earliest pending time (the eviction
|
|
394
|
+
* fallback) and returns the earliest time the in-memory timer should fire.
|
|
395
|
+
*/
|
|
396
|
+
async reconcile(physicalAlarm) {
|
|
397
|
+
const physicalTimes = [];
|
|
398
|
+
const timerTimes = [];
|
|
399
|
+
for (const participant of this.participants) {
|
|
400
|
+
const { physical, timer } = this.pendingTimes(participant);
|
|
401
|
+
if (physical !== undefined) {
|
|
402
|
+
physicalTimes.push(physical);
|
|
403
|
+
}
|
|
404
|
+
if (timer !== undefined) {
|
|
405
|
+
timerTimes.push(timer);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
const physicalNextAt = physicalTimes.length === 0 ? undefined : Math.min(...physicalTimes);
|
|
409
|
+
const currentAlarm = await physicalAlarm.getAlarm();
|
|
410
|
+
if (physicalNextAt === undefined) {
|
|
411
|
+
if (currentAlarm !== null) {
|
|
412
|
+
await physicalAlarm.deleteAlarm();
|
|
413
|
+
}
|
|
414
|
+
return undefined;
|
|
415
|
+
}
|
|
416
|
+
const target = Math.max(physicalNextAt, Date.now() + this.alarmMinDelayMs);
|
|
417
|
+
// Preserve a fallback inside this range so frequent reconciliation cannot push it later.
|
|
418
|
+
if (currentAlarm === null ||
|
|
419
|
+
currentAlarm < physicalNextAt ||
|
|
420
|
+
currentAlarm > target) {
|
|
421
|
+
await physicalAlarm.setAlarm(target);
|
|
422
|
+
}
|
|
423
|
+
return timerTimes.length === 0 ? undefined : Math.min(...timerTimes);
|
|
424
|
+
}
|
|
425
|
+
scheduleEarlyAlarm(timestamp) {
|
|
426
|
+
if (this.earlyAlarm?.timestamp === timestamp) {
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
if (this.earlyAlarm) {
|
|
430
|
+
clearTimeout(this.earlyAlarm.timer);
|
|
431
|
+
this.earlyAlarm = undefined;
|
|
432
|
+
}
|
|
433
|
+
if (timestamp === undefined) {
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
const delay = timestamp - Date.now();
|
|
437
|
+
// Without liveTimers the in-memory timer only covers the sub-min-delay
|
|
438
|
+
// window; the physical alarm handles everything beyond it. With liveTimers
|
|
439
|
+
// it fires at the exact due time no matter how far out, except past the
|
|
440
|
+
// timer's range where only the physical fallback can serve it.
|
|
441
|
+
if (!this.liveTimers && delay >= this.alarmMinDelayMs) {
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
if (delay > maxTimerMs) {
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
const timer = setTimeout(() => {
|
|
448
|
+
if (this.earlyAlarm?.timer !== timer) {
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
this.earlyAlarm = undefined;
|
|
452
|
+
trace('scheduler.early_alarm', { scheduledTime: timestamp });
|
|
453
|
+
const onError = (error) => reportFailure('durability.background_execution.failed', { scheduledTime: timestamp }, error);
|
|
454
|
+
const execution = this.alarm().catch(onError);
|
|
455
|
+
this.waitUntil(execution, onError);
|
|
456
|
+
}, Math.max(delay, 0));
|
|
457
|
+
this.earlyAlarm = { timestamp, timer };
|
|
458
|
+
}
|
|
459
|
+
/** Coalesces refreshes without dropping requests arriving after the pending-work read. */
|
|
460
|
+
scheduleNextAlarm() {
|
|
461
|
+
this.alarmRefreshRequested = true;
|
|
462
|
+
if (this.alarmRefresh) {
|
|
463
|
+
return this.alarmRefresh;
|
|
464
|
+
}
|
|
465
|
+
this.alarmRefresh = (async () => {
|
|
466
|
+
try {
|
|
467
|
+
do {
|
|
468
|
+
// eslint-disable-next-line no-await-in-loop
|
|
469
|
+
await this.transaction(() => {
|
|
470
|
+
this.alarmRefreshRequested = false;
|
|
471
|
+
});
|
|
472
|
+
} while (this.alarmRefreshRequested);
|
|
473
|
+
}
|
|
474
|
+
finally {
|
|
475
|
+
this.alarmRefresh = undefined;
|
|
476
|
+
}
|
|
477
|
+
})();
|
|
478
|
+
return this.alarmRefresh;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Aborts matching active handlers, then deletes one kind's records created
|
|
482
|
+
* before the timestamp and reconciles the alarm in the same transaction.
|
|
483
|
+
*/
|
|
484
|
+
async purge(table, before, active, abortReason) {
|
|
485
|
+
if (!Number.isSafeInteger(before) || before < 0) {
|
|
486
|
+
throw new RangeError('timestamp must be a non-negative safe integer');
|
|
487
|
+
}
|
|
488
|
+
for (const running of active.values()) {
|
|
489
|
+
if (running.createdAt < before) {
|
|
490
|
+
running.controller?.abort(new Error(abortReason));
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
const where = `created_at < ?${channelClause(table)}`;
|
|
494
|
+
const params = [before, ...channelBindings(table)];
|
|
495
|
+
const removed = await this.transaction((sql) => {
|
|
496
|
+
const { count } = countRowSchema.parse(sql
|
|
497
|
+
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE ${where}`, ...params)
|
|
498
|
+
.toArray()[0]);
|
|
499
|
+
sql.exec(`DELETE FROM ${table.name} WHERE ${where}`, ...params);
|
|
500
|
+
deleteStaleMemos(sql, table);
|
|
501
|
+
return count;
|
|
502
|
+
});
|
|
503
|
+
if (removed > 0) {
|
|
504
|
+
for (const key of this.watchers.get(tableOwner(table))?.keys() ?? []) {
|
|
505
|
+
this.notify(table, key);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
return removed;
|
|
509
|
+
}
|
|
510
|
+
/** In-memory only: listeners never survive eviction. */
|
|
511
|
+
watch(table, key, listener) {
|
|
512
|
+
const owner = tableOwner(table);
|
|
513
|
+
const byKey = this.watchers.get(owner) ?? new Map();
|
|
514
|
+
this.watchers.set(owner, byKey);
|
|
515
|
+
const listeners = byKey.get(key) ?? new Set();
|
|
516
|
+
byKey.set(key, listeners);
|
|
517
|
+
listeners.add(listener);
|
|
518
|
+
return () => {
|
|
519
|
+
listeners.delete(listener);
|
|
520
|
+
if (listeners.size === 0 && byKey.get(key) === listeners) {
|
|
521
|
+
byKey.delete(key);
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
/** Call only after the change has committed. */
|
|
526
|
+
notify(table, key, completed) {
|
|
527
|
+
for (const listener of this.watchers.get(tableOwner(table))?.get(key) ??
|
|
528
|
+
[]) {
|
|
529
|
+
listener(completed);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
tryAcquirePermit() {
|
|
533
|
+
if (this.availablePermits <= 0) {
|
|
534
|
+
return undefined;
|
|
535
|
+
}
|
|
536
|
+
this.availablePermits -= 1;
|
|
537
|
+
return this.createRelease();
|
|
538
|
+
}
|
|
539
|
+
acquirePermit() {
|
|
540
|
+
const release = this.tryAcquirePermit();
|
|
541
|
+
if (release) {
|
|
542
|
+
return Promise.resolve(release);
|
|
543
|
+
}
|
|
544
|
+
return new Promise((resolve) => {
|
|
545
|
+
this.permitWaiters.push(resolve);
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
createRelease() {
|
|
549
|
+
let released = false;
|
|
550
|
+
return () => {
|
|
551
|
+
if (released) {
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
released = true;
|
|
555
|
+
this.availablePermits += 1;
|
|
556
|
+
if (this.availablePermits <= 0) {
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
const waiter = this.permitWaiters.shift();
|
|
560
|
+
if (waiter) {
|
|
561
|
+
this.availablePermits -= 1;
|
|
562
|
+
waiter(this.createRelease());
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
async runConcurrent(items, worker) {
|
|
567
|
+
let nextIndex = 0;
|
|
568
|
+
await Promise.all(Array.from({ length: Math.min(this.alarmConcurrency, items.length) }, async () => {
|
|
569
|
+
while (nextIndex < items.length) {
|
|
570
|
+
const item = items[nextIndex++];
|
|
571
|
+
if (item !== undefined) {
|
|
572
|
+
// eslint-disable-next-line no-await-in-loop
|
|
573
|
+
await worker(item);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}));
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Runs due work and re-arms the physical alarm. A failure that would
|
|
580
|
+
* otherwise propagate to the platform is turned into a retry of our own,
|
|
581
|
+
* because the platform drops an alarm after a few failed deliveries.
|
|
582
|
+
*/
|
|
583
|
+
async alarm(info) {
|
|
584
|
+
try {
|
|
585
|
+
await this.runAlarm(info);
|
|
586
|
+
this.failedRuns = 0;
|
|
587
|
+
}
|
|
588
|
+
catch (error) {
|
|
589
|
+
// Throwing lets the platform retry, but it gives up after a few failures
|
|
590
|
+
// and drops the alarm, stranding pending work. Keeping our own alarm
|
|
591
|
+
// armed turns the failure into a retry we control instead.
|
|
592
|
+
if (!(await this.armFallbackAlarm())) {
|
|
593
|
+
throw error;
|
|
594
|
+
}
|
|
595
|
+
reportFailure('durability.alarm.failed', { retryInMs: this.alarmMinDelayMs }, error);
|
|
596
|
+
this.notifyAlarmRetry(error, info);
|
|
597
|
+
this.retryFailedRun();
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* A transient storage fault would otherwise leave all due work to the
|
|
602
|
+
* fallback alarm, up to `alarmMinDelayMs` away: longer than a holder's
|
|
603
|
+
* grant window, for one. The in-memory timer retries sooner, backing off to
|
|
604
|
+
* that fallback while runs keep failing.
|
|
605
|
+
*/
|
|
606
|
+
retryFailedRun() {
|
|
607
|
+
this.failedRuns += 1;
|
|
608
|
+
const delay = Math.min(failedRunRetryMs * 2 ** (this.failedRuns - 1), this.alarmMinDelayMs);
|
|
609
|
+
this.scheduleEarlyAlarm(Date.now() + delay);
|
|
610
|
+
}
|
|
611
|
+
notifyAlarmRetry(failure, invocation) {
|
|
612
|
+
const hook = this.onAlarmRetry;
|
|
613
|
+
if (!hook) {
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
const objectId = this.objectId();
|
|
617
|
+
const info = {
|
|
618
|
+
error: serializeError(failure),
|
|
619
|
+
retryInMs: this.alarmMinDelayMs,
|
|
620
|
+
...(invocation === undefined
|
|
621
|
+
? {}
|
|
622
|
+
: {
|
|
623
|
+
retryCount: invocation.retryCount,
|
|
624
|
+
isRetry: invocation.isRetry,
|
|
625
|
+
}),
|
|
626
|
+
...(objectId === undefined ? {} : { objectId }),
|
|
627
|
+
};
|
|
628
|
+
const report = (hookError) => reportFailure('durability.alarm.retry_hook_failed', { retryInMs: info.retryInMs }, hookError);
|
|
629
|
+
try {
|
|
630
|
+
const result = hook(info);
|
|
631
|
+
if (result !== undefined) {
|
|
632
|
+
this.waitUntil(Promise.resolve(result).catch(report), report);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
catch (hookError) {
|
|
636
|
+
report(hookError);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Runs every participant's due work, hands off to a fresh alarm if the
|
|
641
|
+
* invocation approaches its wall-time limit, and re-arms the physical alarm.
|
|
642
|
+
*/
|
|
643
|
+
async runAlarm(info) {
|
|
644
|
+
const startedAt = Date.now();
|
|
645
|
+
const execution = Promise.all(this.participants.map((participant) => participant.runDue(startedAt, info))).then(() => undefined);
|
|
646
|
+
void execution.catch(() => undefined);
|
|
647
|
+
const handoff = Symbol('alarm handoff');
|
|
648
|
+
const remaining = this.alarmHandoffMs - (Date.now() - startedAt);
|
|
649
|
+
let timeout;
|
|
650
|
+
let result;
|
|
651
|
+
try {
|
|
652
|
+
result = await Promise.race([
|
|
653
|
+
execution,
|
|
654
|
+
new Promise((resolve) => {
|
|
655
|
+
timeout = setTimeout(() => resolve(handoff), Math.max(remaining, 0));
|
|
656
|
+
}),
|
|
657
|
+
]);
|
|
658
|
+
}
|
|
659
|
+
finally {
|
|
660
|
+
if (timeout !== undefined) {
|
|
661
|
+
clearTimeout(timeout);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
if (result === handoff) {
|
|
665
|
+
await this.armForHandoff(Date.now());
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
await this.scheduleNextAlarm();
|
|
669
|
+
}
|
|
670
|
+
/** Arms the physical alarm directly, outside any transaction; the next run reconciles properly. */
|
|
671
|
+
async armFallbackAlarm() {
|
|
672
|
+
try {
|
|
673
|
+
await this.storage.setAlarm(Date.now() + this.alarmMinDelayMs);
|
|
674
|
+
return true;
|
|
675
|
+
}
|
|
676
|
+
catch {
|
|
677
|
+
return false;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
armForHandoff(timestamp) {
|
|
681
|
+
return this.storage.transaction(async (physicalAlarm) => {
|
|
682
|
+
const currentAlarm = await physicalAlarm.getAlarm();
|
|
683
|
+
const target = Math.max(timestamp, Date.now() + this.alarmMinDelayMs);
|
|
684
|
+
if (currentAlarm === null || currentAlarm > target) {
|
|
685
|
+
await physicalAlarm.setAlarm(target);
|
|
686
|
+
}
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Registers the execution in its active map synchronously, then claims and
|
|
691
|
+
* runs one attempt in the background. The returned promise settles when the
|
|
692
|
+
* attempt's outcome has been persisted, not when a timed-out handler settles.
|
|
693
|
+
*/
|
|
694
|
+
startExecution(spec, eagerPermit) {
|
|
695
|
+
const { key, row, policy } = spec;
|
|
696
|
+
let settleActive;
|
|
697
|
+
const settled = new Promise((resolve) => {
|
|
698
|
+
settleActive = resolve;
|
|
699
|
+
});
|
|
700
|
+
const activeExecution = {
|
|
701
|
+
generation: row.generation_id,
|
|
702
|
+
createdAt: row.created_at,
|
|
703
|
+
settled,
|
|
704
|
+
};
|
|
705
|
+
spec.active.set(key, activeExecution);
|
|
706
|
+
this.trackActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
707
|
+
let releasePermit = eagerPermit;
|
|
708
|
+
let handlerStarted = false;
|
|
709
|
+
let handlerSettled = false;
|
|
710
|
+
let policySettled = false;
|
|
711
|
+
const releaseIfSettled = () => {
|
|
712
|
+
if (policySettled && (!handlerStarted || handlerSettled)) {
|
|
713
|
+
releasePermit?.();
|
|
714
|
+
settleActive();
|
|
715
|
+
}
|
|
716
|
+
};
|
|
717
|
+
const execution = (async () => {
|
|
718
|
+
releasePermit ?? (releasePermit = await this.acquirePermit());
|
|
719
|
+
const attempt = claimAttempt(this.sql, spec.table, key, row.generation_id, policy.maxAttempts);
|
|
720
|
+
if (attempt === undefined) {
|
|
721
|
+
await this.failExhausted(spec);
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
const startedAt = Date.now();
|
|
725
|
+
spec.emit({
|
|
726
|
+
...spec.entity,
|
|
727
|
+
type: 'attempt_started',
|
|
728
|
+
timestamp: startedAt,
|
|
729
|
+
attempt,
|
|
730
|
+
});
|
|
731
|
+
const controller = new AbortController();
|
|
732
|
+
activeExecution.controller = controller;
|
|
733
|
+
let timeout;
|
|
734
|
+
try {
|
|
735
|
+
handlerStarted = true;
|
|
736
|
+
const handlerResult = Promise.resolve().then(() => spec.invoke(attempt, controller.signal));
|
|
737
|
+
void handlerResult
|
|
738
|
+
.finally(() => {
|
|
739
|
+
handlerSettled = true;
|
|
740
|
+
releaseIfSettled();
|
|
741
|
+
})
|
|
742
|
+
.catch(() => undefined);
|
|
743
|
+
const timeoutResult = new Promise((_resolve, reject) => {
|
|
744
|
+
timeout = setTimeout(() => {
|
|
745
|
+
const error = spec.timeoutError();
|
|
746
|
+
controller.abort(error);
|
|
747
|
+
reject(error);
|
|
748
|
+
}, policy.attemptTimeoutMs);
|
|
749
|
+
});
|
|
750
|
+
const result = await Promise.race([handlerResult, timeoutResult]);
|
|
751
|
+
// One transaction: a failed memo cleanup must undo the completion, not orphan the memos.
|
|
752
|
+
const timestamp = this.transactionSync((sql) => {
|
|
753
|
+
const completedAt = spec.complete(result, attempt);
|
|
754
|
+
if (completedAt !== undefined) {
|
|
755
|
+
deleteMemos(sql, spec.table, key);
|
|
756
|
+
}
|
|
757
|
+
return completedAt;
|
|
758
|
+
});
|
|
759
|
+
if (timestamp !== undefined) {
|
|
760
|
+
this.notify(spec.table, key, row);
|
|
761
|
+
spec.emit({
|
|
762
|
+
...spec.entity,
|
|
763
|
+
type: 'attempt_settled',
|
|
764
|
+
timestamp,
|
|
765
|
+
attempt,
|
|
766
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
767
|
+
outcome: 'completed',
|
|
768
|
+
...(spec.includeResult ? { result } : {}),
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
catch (caught) {
|
|
773
|
+
await this.settleFailure(spec, attempt, startedAt, caught);
|
|
774
|
+
}
|
|
775
|
+
finally {
|
|
776
|
+
if (timeout !== undefined) {
|
|
777
|
+
clearTimeout(timeout);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
})().finally(() => {
|
|
781
|
+
policySettled = true;
|
|
782
|
+
releaseIfSettled();
|
|
783
|
+
});
|
|
784
|
+
void settled.finally(() => {
|
|
785
|
+
if (spec.active.get(key) === activeExecution) {
|
|
786
|
+
spec.active.delete(key);
|
|
787
|
+
}
|
|
788
|
+
this.releaseActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
789
|
+
// Re-arm the in-memory timer now that this key is eligible again, so work
|
|
790
|
+
// the timer skipped while it executed is picked up at its exact time.
|
|
791
|
+
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
792
|
+
});
|
|
793
|
+
return execution;
|
|
794
|
+
}
|
|
795
|
+
async failExhausted(spec) {
|
|
796
|
+
const { key, row, policy } = spec;
|
|
797
|
+
const exhausted = serializeError(spec.exhaustedError());
|
|
798
|
+
const attempt = await this.transaction((sql) => {
|
|
799
|
+
const exhaustedAttempt = exhaustAttempts(sql, spec.table, key, row.generation_id, policy.maxAttempts, exhausted);
|
|
800
|
+
if (exhaustedAttempt !== undefined) {
|
|
801
|
+
deleteMemos(sql, spec.table, key);
|
|
802
|
+
}
|
|
803
|
+
return exhaustedAttempt;
|
|
804
|
+
});
|
|
805
|
+
if (attempt === undefined) {
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
this.notify(spec.table, key);
|
|
809
|
+
spec.emit({
|
|
810
|
+
...spec.entity,
|
|
811
|
+
type: 'terminal',
|
|
812
|
+
timestamp: Date.now(),
|
|
813
|
+
attempt,
|
|
814
|
+
reason: 'attempts_exhausted',
|
|
815
|
+
error: exhausted,
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
async settleFailure(spec, attempt, startedAt, caught) {
|
|
819
|
+
const { key, row } = spec;
|
|
820
|
+
const decision = decideRetry(spec, attempt, caught);
|
|
821
|
+
const serializedError = serializeError(decision.error);
|
|
822
|
+
const updated = await this.transaction((sql) => {
|
|
823
|
+
const owned = updateAttempt(sql, spec.table, { key, generation: row.generation_id, attempt }, 'status = ?, next_attempt_at = ?, last_error = ?, last_error_name = ?', decision.terminal ? 'failed' : 'pending', decision.nextAttemptAt, serializedError.message, serializedError.name);
|
|
824
|
+
if (owned && decision.terminal) {
|
|
825
|
+
deleteMemos(sql, spec.table, key);
|
|
826
|
+
}
|
|
827
|
+
return owned;
|
|
828
|
+
});
|
|
829
|
+
if (!updated) {
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
if (decision.terminal) {
|
|
833
|
+
this.notify(spec.table, key);
|
|
834
|
+
}
|
|
835
|
+
const settled = {
|
|
836
|
+
...spec.entity,
|
|
837
|
+
type: 'attempt_settled',
|
|
838
|
+
timestamp: decision.timestamp,
|
|
839
|
+
attempt,
|
|
840
|
+
durationMs: Math.max(0, decision.timestamp - startedAt),
|
|
841
|
+
error: serializedError,
|
|
842
|
+
};
|
|
843
|
+
spec.emit(decision.terminal
|
|
844
|
+
? { ...settled, outcome: 'failed' }
|
|
845
|
+
: {
|
|
846
|
+
...settled,
|
|
847
|
+
outcome: 'retry_scheduled',
|
|
848
|
+
nextAttemptAt: decision.nextAttemptAt,
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
const engines = new WeakMap();
|
|
853
|
+
// Every copy of this package in an isolate shares the registry, so a second
|
|
854
|
+
// copy reaching the same object is caught instead of fighting over its alarm.
|
|
855
|
+
const installation = Symbol('durability installation');
|
|
856
|
+
const globalRegistry = globalThis;
|
|
857
|
+
const registryKey = Symbol.for('durability.object-schedulers');
|
|
858
|
+
globalRegistry[registryKey] ?? (globalRegistry[registryKey] = new WeakMap());
|
|
859
|
+
const objectSchedulers = globalRegistry[registryKey];
|
|
860
|
+
/** The one engine behind every scheduler and helper of a Durable Object. */
|
|
861
|
+
const engineForObject = (config) => {
|
|
862
|
+
const { context } = config;
|
|
863
|
+
const existing = objectSchedulers.get(context.storage);
|
|
864
|
+
if (existing === undefined) {
|
|
865
|
+
const engine = new Engine(context, config);
|
|
866
|
+
objectSchedulers.set(context.storage, { engine, installation });
|
|
867
|
+
return engine;
|
|
868
|
+
}
|
|
869
|
+
if (existing.installation !== installation) {
|
|
870
|
+
throw new Error('Another copy of durability is already attached to this Durable Object; install durability once, for example as a peer dependency');
|
|
871
|
+
}
|
|
872
|
+
const engine = existing.engine;
|
|
873
|
+
engine.attach(context, config);
|
|
874
|
+
return engine;
|
|
875
|
+
};
|
|
876
|
+
/**
|
|
877
|
+
* Owns the Durable Object's physical alarm, storage, and concurrency pool so
|
|
878
|
+
* every durability helper on the object shares them.
|
|
879
|
+
*
|
|
880
|
+
* A Durable Object has one scheduler. Constructing another for the same
|
|
881
|
+
* object, or a helper from the object's context, joins it; scheduler options
|
|
882
|
+
* merge, and a different value for an option already set throws. Delegate the
|
|
883
|
+
* object's `alarm` method to {@link DurabilityScheduler.alarm} or to any
|
|
884
|
+
* helper's `alarm`: each runs the due work of every helper on the object.
|
|
885
|
+
*
|
|
886
|
+
* @example
|
|
887
|
+
* ```ts
|
|
888
|
+
* class Jobs extends DurableObject<Env> {
|
|
889
|
+
* private readonly scheduler = new DurabilityScheduler({ context: this.ctx });
|
|
890
|
+
* private readonly durability = new Durability({
|
|
891
|
+
* scheduler: this.scheduler,
|
|
892
|
+
* handlers,
|
|
893
|
+
* });
|
|
894
|
+
* private readonly alarms = new DurabilityAlarms({
|
|
895
|
+
* scheduler: this.scheduler,
|
|
896
|
+
* handlers: alarmHandlers,
|
|
897
|
+
* });
|
|
898
|
+
*
|
|
899
|
+
* alarm(info?: AlarmInvocationInfo) {
|
|
900
|
+
* return this.scheduler.alarm(info);
|
|
901
|
+
* }
|
|
902
|
+
* }
|
|
903
|
+
* ```
|
|
904
|
+
*/
|
|
905
|
+
export class DurabilityScheduler {
|
|
906
|
+
constructor(config) {
|
|
907
|
+
engines.set(this, engineForObject(config));
|
|
908
|
+
}
|
|
909
|
+
/** Runs due work for every attached helper and re-arms the physical alarm. */
|
|
910
|
+
alarm(info) {
|
|
911
|
+
return engineFor({ scheduler: this }).alarm(info);
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
traceMethods(DurabilityScheduler.prototype, 'DurabilityScheduler');
|
|
915
|
+
/** Resolves the engine a helper attaches to: the object's one scheduler. */
|
|
916
|
+
export const engineFor = (attachment) => {
|
|
917
|
+
if (attachment.scheduler === undefined) {
|
|
918
|
+
return engineForObject(attachment);
|
|
919
|
+
}
|
|
920
|
+
const engine = engines.get(attachment.scheduler);
|
|
921
|
+
if (!engine) {
|
|
922
|
+
throw new Error('Unknown DurabilityScheduler instance');
|
|
923
|
+
}
|
|
924
|
+
return engine;
|
|
925
|
+
};
|