durability 5.1.0 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -21
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +13 -5
- package/dist/{durability-alarms.js → alarms/index.js} +85 -74
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/experimental.d.ts +10 -10
- package/dist/experimental.js +7 -7
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +91 -83
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +27 -20
- package/dist/{log.js → log/index.js} +200 -166
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +23 -21
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +0 -111
- package/dist/migrations.js +12 -628
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +76 -70
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +35 -77
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/policy.d.ts +19 -19
- package/dist/policy.js +13 -12
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/{schedule.d.ts → schedule/index.d.ts} +16 -8
- package/dist/{schedule.js → schedule/index.js} +97 -83
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +75 -25
- package/dist/{scheduler.js → scheduler/index.js} +308 -70
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +8 -7
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/semaphore/protocol.d.ts +451 -0
- package/dist/semaphore/protocol.js +114 -0
- package/dist/semaphore/queue-store.d.ts +107 -0
- package/dist/semaphore/queue-store.js +258 -0
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +56 -28
- package/dist/semaphore/store.d.ts +53 -0
- package/dist/semaphore/store.js +107 -0
- package/dist/serialization.d.ts +5 -0
- package/dist/serialization.js +41 -16
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +22 -15
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +106 -72
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/topology/snapshot.d.ts +50 -0
- package/dist/topology/snapshot.js +38 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +9 -9
- package/package.json +11 -11
- package/dist/semaphore-protocol.d.ts +0 -231
- package/dist/semaphore-protocol.js +0 -106
- package/dist/semaphore-queue-store.d.ts +0 -95
- package/dist/semaphore-queue-store.js +0 -252
- package/dist/semaphore-store.d.ts +0 -43
- package/dist/semaphore-store.js +0 -108
- package/dist/topology-snapshot.d.ts +0 -70
- package/dist/topology-snapshot.js +0 -48
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -1,21 +1,46 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { trace, traceMethods, traceStorage } from '
|
|
3
|
-
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from '
|
|
4
|
-
import { createEmitter, } from '
|
|
5
|
-
import { isSchemaCurrent, migrateCapability, } from '
|
|
6
|
-
import { assertPositiveInteger } from '
|
|
7
|
-
import { deleteMemos, deleteStaleMemos } from '
|
|
8
|
-
import { claimAttempt, exhaustAttempts, updateAttempt, } from '
|
|
1
|
+
import * as v from 'valibot';
|
|
2
|
+
import { trace, traceMethods, traceStorage } from '../dev.js';
|
|
3
|
+
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from '../errors.js';
|
|
4
|
+
import { createEmitter, } from '../lifecycle.js';
|
|
5
|
+
import { isSchemaCurrent, migrateCapability, } from '../migrations.js';
|
|
6
|
+
import { assertPositiveInteger } from '../policy.js';
|
|
7
|
+
import { deleteMemos, deleteStaleMemos } from '../memoize/index.js';
|
|
8
|
+
import { channelBindings, channelClause, claimAttempt, exhaustAttempts, tableOwner, updateAttempt, } from '../storage.js';
|
|
9
9
|
const minAlarmDelayMs = 1000;
|
|
10
|
+
const failedRunRetryMs = 1000;
|
|
11
|
+
const defaultAlarmConcurrency = 10;
|
|
12
|
+
const schedulerOptionKeys = [
|
|
13
|
+
'alarmConcurrency',
|
|
14
|
+
'alarmMinDelayMs',
|
|
15
|
+
'alarmHandoffMs',
|
|
16
|
+
'liveTimers',
|
|
17
|
+
'onAlarmRetry',
|
|
18
|
+
];
|
|
10
19
|
const maxAlarmDelayMs = 15000;
|
|
11
20
|
// setTimeout treats delays past this as zero, so anything further out can only
|
|
12
21
|
// be served by the physical alarm fallback.
|
|
13
22
|
const maxTimerMs = 2 ** 31 - 1;
|
|
14
|
-
const pendingHeadRowSchema =
|
|
15
|
-
pk:
|
|
16
|
-
|
|
23
|
+
const pendingHeadRowSchema = v.object({
|
|
24
|
+
pk: v.string(),
|
|
25
|
+
channel: v.optional(v.string()),
|
|
26
|
+
next_attempt_at: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
17
27
|
});
|
|
18
|
-
const countRowSchema =
|
|
28
|
+
const countRowSchema = v.object({
|
|
29
|
+
count: v.pipe(v.number(), v.safeInteger(), v.minValue(0)),
|
|
30
|
+
});
|
|
31
|
+
/** The channel every helper uses when it is not given one. */
|
|
32
|
+
export const defaultChannel = 'default';
|
|
33
|
+
/** How long records of a channel no instance attached wait before they are looked at again. */
|
|
34
|
+
const unattachedChannelRetryMs = 60000;
|
|
35
|
+
const channelRowSchema = v.object({ channel: v.string() });
|
|
36
|
+
const activeKey = (channel, key) => JSON.stringify([channel ?? null, key]);
|
|
37
|
+
/** Rejects a channel name a helper would otherwise store rows under. */
|
|
38
|
+
export const assertChannel = (helper, channel) => {
|
|
39
|
+
if (typeof channel !== 'string' || channel.length === 0) {
|
|
40
|
+
throw new TypeError(`${helper} channel must be a non-empty string`);
|
|
41
|
+
}
|
|
42
|
+
return channel;
|
|
43
|
+
};
|
|
19
44
|
const decideRetry = (spec, attempt, error) => {
|
|
20
45
|
const terminal = isNonRetryable(error) ||
|
|
21
46
|
attempt >= spec.policy.maxAttempts ||
|
|
@@ -60,9 +85,21 @@ const decideRetry = (spec, attempt, error) => {
|
|
|
60
85
|
*/
|
|
61
86
|
export class Engine {
|
|
62
87
|
constructor(context, options) {
|
|
88
|
+
this.alarmConcurrency = defaultAlarmConcurrency;
|
|
89
|
+
this.alarmMinDelayMs = 15000;
|
|
90
|
+
this.alarmHandoffMs = 14 * 60000;
|
|
91
|
+
this.liveTimers = false;
|
|
92
|
+
// Only options someone set can conflict; an omitted one takes any value.
|
|
93
|
+
this.explicitOptions = {};
|
|
63
94
|
this.participants = [];
|
|
95
|
+
this.channelsByTable = new Map();
|
|
96
|
+
this.claimedChannels = new Map();
|
|
97
|
+
this.reportedMissingChannels = new Set();
|
|
64
98
|
this.migrated = new Set();
|
|
99
|
+
this.availablePermits = defaultAlarmConcurrency;
|
|
65
100
|
this.permitWaiters = [];
|
|
101
|
+
this.alarmRefreshRequested = false;
|
|
102
|
+
this.failedRuns = 0;
|
|
66
103
|
this.watchers = new Map();
|
|
67
104
|
// Keys whose attempt is currently executing, per table. The in-memory timer
|
|
68
105
|
// skips these so a still-due active row cannot drive a zero-delay timer loop.
|
|
@@ -70,23 +107,60 @@ export class Engine {
|
|
|
70
107
|
this.context = context;
|
|
71
108
|
this.storage = traceStorage(context.storage);
|
|
72
109
|
this.sql = this.storage.sql;
|
|
73
|
-
this.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
110
|
+
this.configure(options);
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Joins one more helper or scheduler handle to this object's scheduler. The
|
|
114
|
+
* first context that can `waitUntil` is kept, so a bare `{ storage }` context
|
|
115
|
+
* attached first does not hide it from the rest.
|
|
116
|
+
*/
|
|
117
|
+
attach(context, options) {
|
|
118
|
+
if (this.context.waitUntil === undefined &&
|
|
119
|
+
context.waitUntil !== undefined) {
|
|
120
|
+
this.context = context;
|
|
121
|
+
}
|
|
122
|
+
this.configure(options);
|
|
123
|
+
}
|
|
124
|
+
configure(options) {
|
|
125
|
+
const merged = { ...this.explicitOptions };
|
|
126
|
+
for (const key of schedulerOptionKeys) {
|
|
127
|
+
const value = options[key];
|
|
128
|
+
if (value === undefined) {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const current = this.explicitOptions[key];
|
|
132
|
+
if (current !== undefined && current !== value) {
|
|
133
|
+
throw new Error(`Scheduler option "${key}" is already set to a different value for this Durable Object`);
|
|
134
|
+
}
|
|
135
|
+
Object.assign(merged, { [key]: value });
|
|
136
|
+
}
|
|
137
|
+
const alarmConcurrency = merged.alarmConcurrency ?? defaultAlarmConcurrency;
|
|
138
|
+
const alarmMinDelayMs = merged.alarmMinDelayMs ?? 15000;
|
|
139
|
+
const alarmHandoffMs = merged.alarmHandoffMs ?? 14 * 60000;
|
|
140
|
+
if (merged.onAlarmRetry !== undefined &&
|
|
141
|
+
typeof merged.onAlarmRetry !== 'function') {
|
|
80
142
|
throw new TypeError('onAlarmRetry must be a function');
|
|
81
143
|
}
|
|
82
|
-
assertPositiveInteger('alarmConcurrency',
|
|
83
|
-
assertPositiveInteger('alarmMinDelayMs',
|
|
84
|
-
if (
|
|
85
|
-
|
|
144
|
+
assertPositiveInteger('alarmConcurrency', alarmConcurrency);
|
|
145
|
+
assertPositiveInteger('alarmMinDelayMs', alarmMinDelayMs);
|
|
146
|
+
if (alarmMinDelayMs < minAlarmDelayMs ||
|
|
147
|
+
alarmMinDelayMs > maxAlarmDelayMs) {
|
|
86
148
|
throw new RangeError(`alarmMinDelayMs must be between ${minAlarmDelayMs} and ${maxAlarmDelayMs}`);
|
|
87
149
|
}
|
|
88
|
-
assertPositiveInteger('alarmHandoffMs',
|
|
89
|
-
|
|
150
|
+
assertPositiveInteger('alarmHandoffMs', alarmHandoffMs);
|
|
151
|
+
// Permits already handed out stay out; raising the limit frees more now.
|
|
152
|
+
this.availablePermits += alarmConcurrency - this.alarmConcurrency;
|
|
153
|
+
this.alarmConcurrency = alarmConcurrency;
|
|
154
|
+
this.alarmMinDelayMs = alarmMinDelayMs;
|
|
155
|
+
this.alarmHandoffMs = alarmHandoffMs;
|
|
156
|
+
this.liveTimers = merged.liveTimers ?? false;
|
|
157
|
+
this.onAlarmRetry = merged.onAlarmRetry;
|
|
158
|
+
this.explicitOptions = merged;
|
|
159
|
+
while (this.availablePermits > 0 && this.permitWaiters.length > 0) {
|
|
160
|
+
const waiter = this.permitWaiters.shift();
|
|
161
|
+
this.availablePermits -= 1;
|
|
162
|
+
waiter(this.createRelease());
|
|
163
|
+
}
|
|
90
164
|
}
|
|
91
165
|
register(participant) {
|
|
92
166
|
const { name } = participant.table;
|
|
@@ -95,6 +169,105 @@ export class Engine {
|
|
|
95
169
|
}
|
|
96
170
|
this.participants.push(participant);
|
|
97
171
|
}
|
|
172
|
+
/** Reserves an instance identity, including for helpers that do not schedule work. */
|
|
173
|
+
claimChannel(kind, channel, helper) {
|
|
174
|
+
const claimed = this.claimedChannels.get(kind) ?? new Set();
|
|
175
|
+
if (claimed.has(channel)) {
|
|
176
|
+
throw new Error(`${helper} for channel "${channel}" is already attached to this scheduler`);
|
|
177
|
+
}
|
|
178
|
+
claimed.add(channel);
|
|
179
|
+
this.claimedChannels.set(kind, claimed);
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Attaches one channel of a table several helper instances share. The first
|
|
183
|
+
* attachment registers the table; each run then runs every attached
|
|
184
|
+
* channel's due rows and retains the rows of channels nothing attached.
|
|
185
|
+
*/
|
|
186
|
+
registerChannel(participant) {
|
|
187
|
+
const { table, channel, helper } = participant;
|
|
188
|
+
let channels = this.channelsByTable.get(table.name);
|
|
189
|
+
if (!channels) {
|
|
190
|
+
const created = new Map();
|
|
191
|
+
this.register({
|
|
192
|
+
table: { name: table.name, key: table.key },
|
|
193
|
+
runDue: async (now, info) => {
|
|
194
|
+
await this.postponeUnattachedChannels(table, created);
|
|
195
|
+
await Promise.all([...created.values()].map((attached) => attached.runDue(now, info)));
|
|
196
|
+
},
|
|
197
|
+
});
|
|
198
|
+
this.channelsByTable.set(table.name, created);
|
|
199
|
+
channels = created;
|
|
200
|
+
}
|
|
201
|
+
this.claimChannel(table.name, channel, helper);
|
|
202
|
+
channels.set(channel, {
|
|
203
|
+
...participant,
|
|
204
|
+
table: { ...table, channel },
|
|
205
|
+
});
|
|
206
|
+
if (participant.nextPendingAt !== undefined) {
|
|
207
|
+
// Missing custom channels are dormant. Defer until construction finishes,
|
|
208
|
+
// then restore their alarm without requiring a new application write.
|
|
209
|
+
const onError = (error) => reportFailure('durability.channel_resume_failed', { table: table.name, channel }, error);
|
|
210
|
+
const resumed = Promise.resolve()
|
|
211
|
+
.then(() => {
|
|
212
|
+
if (participant.nextPendingAt?.(this.sql) !== undefined) {
|
|
213
|
+
return this.scheduleNextAlarm();
|
|
214
|
+
}
|
|
215
|
+
return undefined;
|
|
216
|
+
})
|
|
217
|
+
.catch(async (error) => {
|
|
218
|
+
if (await this.armFallbackAlarm()) {
|
|
219
|
+
this.retryFailedRun();
|
|
220
|
+
}
|
|
221
|
+
onError(error);
|
|
222
|
+
});
|
|
223
|
+
this.waitUntil(resumed, onError);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Due rows of a channel no instance attached, such as one a deploy removed
|
|
228
|
+
* or a library not yet constructed, would keep the alarm awake forever. They
|
|
229
|
+
* are kept and looked at again later, since their owner may return.
|
|
230
|
+
*/
|
|
231
|
+
async postponeUnattachedChannels(table, channels) {
|
|
232
|
+
const now = Date.now();
|
|
233
|
+
const customTiming = [...channels.values()].some((participant) => participant.nextPendingAt !== undefined);
|
|
234
|
+
const channelTables = new Set([
|
|
235
|
+
table.name,
|
|
236
|
+
...[...channels.values()].flatMap((participant) => participant.channelTables ?? []),
|
|
237
|
+
]);
|
|
238
|
+
const query = customTiming
|
|
239
|
+
? [...channelTables]
|
|
240
|
+
.map((name) => `SELECT channel FROM ${name}`)
|
|
241
|
+
.join(' UNION ')
|
|
242
|
+
: `SELECT DISTINCT channel FROM ${table.name}
|
|
243
|
+
WHERE status = 'pending' AND next_attempt_at <= ?`;
|
|
244
|
+
const unattached = v
|
|
245
|
+
.parse(v.array(channelRowSchema), this.sql.exec(query, ...(customTiming ? [] : [now])).toArray())
|
|
246
|
+
.map((row) => row.channel)
|
|
247
|
+
.filter((channel) => !channels.has(channel));
|
|
248
|
+
if (unattached.length === 0) {
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
for (const channel of unattached) {
|
|
252
|
+
const identity = JSON.stringify([table.name, channel]);
|
|
253
|
+
if (customTiming && this.reportedMissingChannels.has(identity)) {
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
this.reportedMissingChannels.add(identity);
|
|
257
|
+
reportFailure('durability.channel_missing', { table: table.name, channel }, new Error(`Channel "${channel}" of ${table.name} has no helper attached; retaining its records`));
|
|
258
|
+
}
|
|
259
|
+
// Custom timing may represent leases or recurring configurations, not pending
|
|
260
|
+
// attempts. Retain those records unchanged until their channel returns.
|
|
261
|
+
if (customTiming) {
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
await this.transaction((sql) => {
|
|
265
|
+
for (const channel of unattached) {
|
|
266
|
+
sql.exec(`UPDATE ${table.name} SET next_attempt_at = ?
|
|
267
|
+
WHERE channel = ? AND status = 'pending' AND next_attempt_at <= ?`, now + unattachedChannelRetryMs, channel, now);
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
}
|
|
98
271
|
waitUntil(promise, onError) {
|
|
99
272
|
if (!this.context.waitUntil) {
|
|
100
273
|
return;
|
|
@@ -168,10 +341,27 @@ export class Engine {
|
|
|
168
341
|
/**
|
|
169
342
|
* The earliest pending time for one participant: `physical` arms the physical
|
|
170
343
|
* alarm (eviction fallback) and includes active work; `timer` drives the
|
|
171
|
-
* in-memory timer and
|
|
344
|
+
* in-memory timer and skips work that is already executing
|
|
172
345
|
* so a still-due active row cannot re-fire a zero-delay timer in a loop.
|
|
173
346
|
*/
|
|
174
347
|
pendingTimes(participant) {
|
|
348
|
+
const channels = this.channelsByTable.get(participant.table.name);
|
|
349
|
+
if (participant.table.channel === undefined &&
|
|
350
|
+
channels !== undefined &&
|
|
351
|
+
[...channels.values()].some((channel) => channel.nextPendingAt !== undefined)) {
|
|
352
|
+
let physical;
|
|
353
|
+
let timer;
|
|
354
|
+
for (const channel of channels.values()) {
|
|
355
|
+
const next = this.pendingTimes(channel);
|
|
356
|
+
if (next.physical !== undefined) {
|
|
357
|
+
physical = Math.min(physical ?? Infinity, next.physical);
|
|
358
|
+
}
|
|
359
|
+
if (next.timer !== undefined) {
|
|
360
|
+
timer = Math.min(timer ?? Infinity, next.timer);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return { physical, timer };
|
|
364
|
+
}
|
|
175
365
|
if (participant.nextPendingAt) {
|
|
176
366
|
const at = participant.nextPendingAt(this.sql);
|
|
177
367
|
if (at === undefined) {
|
|
@@ -179,25 +369,25 @@ export class Engine {
|
|
|
179
369
|
}
|
|
180
370
|
return participant.holding?.()
|
|
181
371
|
? { physical: at }
|
|
182
|
-
: {
|
|
372
|
+
: {
|
|
373
|
+
physical: at,
|
|
374
|
+
timer: participant.nextTimerAt
|
|
375
|
+
? participant.nextTimerAt(this.sql)
|
|
376
|
+
: at,
|
|
377
|
+
};
|
|
183
378
|
}
|
|
184
379
|
const active = this.activeByTable.get(participant.table.name);
|
|
185
|
-
const
|
|
186
|
-
const
|
|
187
|
-
|
|
380
|
+
const partitioned = this.channelsByTable.has(participant.table.name);
|
|
381
|
+
const limit = (active?.size ?? 0) + 1;
|
|
382
|
+
const rows = v.parse(v.array(pendingHeadRowSchema), this.sql
|
|
383
|
+
.exec(`SELECT ${participant.table.key} AS pk, next_attempt_at${partitioned ? ', channel' : ''}
|
|
188
384
|
FROM ${participant.table.name}
|
|
189
385
|
WHERE status = 'pending'
|
|
190
386
|
ORDER BY next_attempt_at ASC
|
|
191
387
|
LIMIT ?`, limit)
|
|
192
388
|
.toArray());
|
|
193
389
|
const physical = rows[0]?.next_attempt_at;
|
|
194
|
-
|
|
195
|
-
if (physical === undefined) {
|
|
196
|
-
return {};
|
|
197
|
-
}
|
|
198
|
-
return { physical, timer: physical };
|
|
199
|
-
}
|
|
200
|
-
const eligible = rows.find((row) => !active?.has(row.pk));
|
|
390
|
+
const eligible = rows.find((row) => !active?.has(activeKey(row.channel, row.pk)));
|
|
201
391
|
return { physical, timer: eligible?.next_attempt_at };
|
|
202
392
|
}
|
|
203
393
|
/**
|
|
@@ -267,14 +457,25 @@ export class Engine {
|
|
|
267
457
|
}, Math.max(delay, 0));
|
|
268
458
|
this.earlyAlarm = { timestamp, timer };
|
|
269
459
|
}
|
|
270
|
-
/** Coalesces
|
|
460
|
+
/** Coalesces refreshes without dropping requests arriving after the pending-work read. */
|
|
271
461
|
scheduleNextAlarm() {
|
|
462
|
+
this.alarmRefreshRequested = true;
|
|
272
463
|
if (this.alarmRefresh) {
|
|
273
464
|
return this.alarmRefresh;
|
|
274
465
|
}
|
|
275
|
-
this.alarmRefresh =
|
|
276
|
-
|
|
277
|
-
|
|
466
|
+
this.alarmRefresh = (async () => {
|
|
467
|
+
try {
|
|
468
|
+
do {
|
|
469
|
+
// eslint-disable-next-line no-await-in-loop
|
|
470
|
+
await this.transaction(() => {
|
|
471
|
+
this.alarmRefreshRequested = false;
|
|
472
|
+
});
|
|
473
|
+
} while (this.alarmRefreshRequested);
|
|
474
|
+
}
|
|
475
|
+
finally {
|
|
476
|
+
this.alarmRefresh = undefined;
|
|
477
|
+
}
|
|
478
|
+
})();
|
|
278
479
|
return this.alarmRefresh;
|
|
279
480
|
}
|
|
280
481
|
/**
|
|
@@ -290,16 +491,18 @@ export class Engine {
|
|
|
290
491
|
running.controller?.abort(new Error(abortReason));
|
|
291
492
|
}
|
|
292
493
|
}
|
|
494
|
+
const where = `created_at < ?${channelClause(table)}`;
|
|
495
|
+
const params = [before, ...channelBindings(table)];
|
|
293
496
|
const removed = await this.transaction((sql) => {
|
|
294
|
-
const { count } =
|
|
295
|
-
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE
|
|
497
|
+
const { count } = v.parse(countRowSchema, sql
|
|
498
|
+
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE ${where}`, ...params)
|
|
296
499
|
.toArray()[0]);
|
|
297
|
-
sql.exec(`DELETE FROM ${table.name} WHERE
|
|
500
|
+
sql.exec(`DELETE FROM ${table.name} WHERE ${where}`, ...params);
|
|
298
501
|
deleteStaleMemos(sql, table);
|
|
299
502
|
return count;
|
|
300
503
|
});
|
|
301
504
|
if (removed > 0) {
|
|
302
|
-
for (const key of this.watchers.get(table
|
|
505
|
+
for (const key of this.watchers.get(tableOwner(table))?.keys() ?? []) {
|
|
303
506
|
this.notify(table, key);
|
|
304
507
|
}
|
|
305
508
|
}
|
|
@@ -307,9 +510,9 @@ export class Engine {
|
|
|
307
510
|
}
|
|
308
511
|
/** In-memory only: listeners never survive eviction. */
|
|
309
512
|
watch(table, key, listener) {
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
this.watchers.set(
|
|
513
|
+
const owner = tableOwner(table);
|
|
514
|
+
const byKey = this.watchers.get(owner) ?? new Map();
|
|
515
|
+
this.watchers.set(owner, byKey);
|
|
313
516
|
const listeners = byKey.get(key) ?? new Set();
|
|
314
517
|
byKey.set(key, listeners);
|
|
315
518
|
listeners.add(listener);
|
|
@@ -322,12 +525,13 @@ export class Engine {
|
|
|
322
525
|
}
|
|
323
526
|
/** Call only after the change has committed. */
|
|
324
527
|
notify(table, key, completed) {
|
|
325
|
-
for (const listener of this.watchers.get(table
|
|
528
|
+
for (const listener of this.watchers.get(tableOwner(table))?.get(key) ??
|
|
529
|
+
[]) {
|
|
326
530
|
listener(completed);
|
|
327
531
|
}
|
|
328
532
|
}
|
|
329
533
|
tryAcquirePermit() {
|
|
330
|
-
if (this.availablePermits
|
|
534
|
+
if (this.availablePermits <= 0) {
|
|
331
535
|
return undefined;
|
|
332
536
|
}
|
|
333
537
|
this.availablePermits -= 1;
|
|
@@ -349,13 +553,15 @@ export class Engine {
|
|
|
349
553
|
return;
|
|
350
554
|
}
|
|
351
555
|
released = true;
|
|
556
|
+
this.availablePermits += 1;
|
|
557
|
+
if (this.availablePermits <= 0) {
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
352
560
|
const waiter = this.permitWaiters.shift();
|
|
353
561
|
if (waiter) {
|
|
562
|
+
this.availablePermits -= 1;
|
|
354
563
|
waiter(this.createRelease());
|
|
355
564
|
}
|
|
356
|
-
else {
|
|
357
|
-
this.availablePermits += 1;
|
|
358
|
-
}
|
|
359
565
|
};
|
|
360
566
|
}
|
|
361
567
|
async runConcurrent(items, worker) {
|
|
@@ -378,6 +584,7 @@ export class Engine {
|
|
|
378
584
|
async alarm(info) {
|
|
379
585
|
try {
|
|
380
586
|
await this.runAlarm(info);
|
|
587
|
+
this.failedRuns = 0;
|
|
381
588
|
}
|
|
382
589
|
catch (error) {
|
|
383
590
|
// Throwing lets the platform retry, but it gives up after a few failures
|
|
@@ -388,8 +595,20 @@ export class Engine {
|
|
|
388
595
|
}
|
|
389
596
|
reportFailure('durability.alarm.failed', { retryInMs: this.alarmMinDelayMs }, error);
|
|
390
597
|
this.notifyAlarmRetry(error, info);
|
|
598
|
+
this.retryFailedRun();
|
|
391
599
|
}
|
|
392
600
|
}
|
|
601
|
+
/**
|
|
602
|
+
* A transient storage fault would otherwise leave all due work to the
|
|
603
|
+
* fallback alarm, up to `alarmMinDelayMs` away: longer than a holder's
|
|
604
|
+
* grant window, for one. The in-memory timer retries sooner, backing off to
|
|
605
|
+
* that fallback while runs keep failing.
|
|
606
|
+
*/
|
|
607
|
+
retryFailedRun() {
|
|
608
|
+
this.failedRuns += 1;
|
|
609
|
+
const delay = Math.min(failedRunRetryMs * 2 ** (this.failedRuns - 1), this.alarmMinDelayMs);
|
|
610
|
+
this.scheduleEarlyAlarm(Date.now() + delay);
|
|
611
|
+
}
|
|
393
612
|
notifyAlarmRetry(failure, invocation) {
|
|
394
613
|
const hook = this.onAlarmRetry;
|
|
395
614
|
if (!hook) {
|
|
@@ -485,7 +704,7 @@ export class Engine {
|
|
|
485
704
|
settled,
|
|
486
705
|
};
|
|
487
706
|
spec.active.set(key, activeExecution);
|
|
488
|
-
this.trackActive(spec.table.name, key);
|
|
707
|
+
this.trackActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
489
708
|
let releasePermit = eagerPermit;
|
|
490
709
|
let handlerStarted = false;
|
|
491
710
|
let handlerSettled = false;
|
|
@@ -567,12 +786,10 @@ export class Engine {
|
|
|
567
786
|
if (spec.active.get(key) === activeExecution) {
|
|
568
787
|
spec.active.delete(key);
|
|
569
788
|
}
|
|
570
|
-
this.releaseActive(spec.table.name, key);
|
|
789
|
+
this.releaseActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
571
790
|
// Re-arm the in-memory timer now that this key is eligible again, so work
|
|
572
791
|
// the timer skipped while it executed is picked up at its exact time.
|
|
573
|
-
|
|
574
|
-
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
575
|
-
}
|
|
792
|
+
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
576
793
|
});
|
|
577
794
|
return execution;
|
|
578
795
|
}
|
|
@@ -634,14 +851,38 @@ export class Engine {
|
|
|
634
851
|
}
|
|
635
852
|
}
|
|
636
853
|
const engines = new WeakMap();
|
|
854
|
+
// Every copy of this package in an isolate shares the registry, so a second
|
|
855
|
+
// copy reaching the same object is caught instead of fighting over its alarm.
|
|
856
|
+
const installation = Symbol('durability installation');
|
|
857
|
+
const globalRegistry = globalThis;
|
|
858
|
+
const registryKey = Symbol.for('durability.object-schedulers');
|
|
859
|
+
globalRegistry[registryKey] ?? (globalRegistry[registryKey] = new WeakMap());
|
|
860
|
+
const objectSchedulers = globalRegistry[registryKey];
|
|
861
|
+
/** The one engine behind every scheduler and helper of a Durable Object. */
|
|
862
|
+
const engineForObject = (config) => {
|
|
863
|
+
const { context } = config;
|
|
864
|
+
const existing = objectSchedulers.get(context.storage);
|
|
865
|
+
if (existing === undefined) {
|
|
866
|
+
const engine = new Engine(context, config);
|
|
867
|
+
objectSchedulers.set(context.storage, { engine, installation });
|
|
868
|
+
return engine;
|
|
869
|
+
}
|
|
870
|
+
if (existing.installation !== installation) {
|
|
871
|
+
throw new Error('Another copy of durability is already attached to this Durable Object; install durability once, for example as a peer dependency');
|
|
872
|
+
}
|
|
873
|
+
const engine = existing.engine;
|
|
874
|
+
engine.attach(context, config);
|
|
875
|
+
return engine;
|
|
876
|
+
};
|
|
637
877
|
/**
|
|
638
|
-
* Owns the Durable Object's physical alarm, storage, and concurrency
|
|
639
|
-
*
|
|
878
|
+
* Owns the Durable Object's physical alarm, storage, and concurrency pool so
|
|
879
|
+
* every durability helper on the object shares them.
|
|
640
880
|
*
|
|
641
|
-
*
|
|
642
|
-
*
|
|
643
|
-
*
|
|
644
|
-
*
|
|
881
|
+
* A Durable Object has one scheduler. Constructing another for the same
|
|
882
|
+
* object, or a helper from the object's context, joins it; scheduler options
|
|
883
|
+
* merge, and a different value for an option already set throws. Delegate the
|
|
884
|
+
* object's `alarm` method to {@link DurabilityScheduler.alarm} or to any
|
|
885
|
+
* helper's `alarm`: each runs the due work of every helper on the object.
|
|
645
886
|
*
|
|
646
887
|
* @example
|
|
647
888
|
* ```ts
|
|
@@ -664,7 +905,7 @@ const engines = new WeakMap();
|
|
|
664
905
|
*/
|
|
665
906
|
export class DurabilityScheduler {
|
|
666
907
|
constructor(config) {
|
|
667
|
-
engines.set(this,
|
|
908
|
+
engines.set(this, engineForObject(config));
|
|
668
909
|
}
|
|
669
910
|
/** Runs due work for every attached helper and re-arms the physical alarm. */
|
|
670
911
|
alarm(info) {
|
|
@@ -672,13 +913,10 @@ export class DurabilityScheduler {
|
|
|
672
913
|
}
|
|
673
914
|
}
|
|
674
915
|
traceMethods(DurabilityScheduler.prototype, 'DurabilityScheduler');
|
|
675
|
-
/**
|
|
676
|
-
* Resolves the engine a helper should attach to: the shared one behind a
|
|
677
|
-
* scheduler, or a private one built from the Durable Object context.
|
|
678
|
-
*/
|
|
916
|
+
/** Resolves the engine a helper attaches to: the object's one scheduler. */
|
|
679
917
|
export const engineFor = (attachment) => {
|
|
680
918
|
if (attachment.scheduler === undefined) {
|
|
681
|
-
return
|
|
919
|
+
return engineForObject(attachment);
|
|
682
920
|
}
|
|
683
921
|
const engine = engines.get(attachment.scheduler);
|
|
684
922
|
if (!engine) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './
|
|
1
|
+
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './protocol.js';
|
|
2
2
|
export type SemaphoreAddress = {
|
|
3
3
|
/** The semaphore key: the name of the Durable Object holding the limit. */
|
|
4
4
|
key: string;
|
|
@@ -10,8 +10,9 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
12
|
var _SemaphoreClient_instances, _SemaphoreClient_config, _SemaphoreClient_nextRequestAt, _SemaphoreClient_address;
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
13
|
+
import * as v from 'valibot';
|
|
14
|
+
import { SemaphoreProtocolError } from '../errors.js';
|
|
15
|
+
import { permitGrantSchema, permitSyncResultSchema, } from './protocol.js';
|
|
15
16
|
/**
|
|
16
17
|
* Transport-independent protocol core. Validates responses and remembers each
|
|
17
18
|
* key's `nextRequestAt` so callers can pace requests the way the holder asked.
|
|
@@ -29,11 +30,11 @@ export class SemaphoreClient {
|
|
|
29
30
|
}
|
|
30
31
|
async request(key, message) {
|
|
31
32
|
const response = await __classPrivateFieldGet(this, _SemaphoreClient_config, "f").invoke(__classPrivateFieldGet(this, _SemaphoreClient_instances, "m", _SemaphoreClient_address).call(this, key), message);
|
|
32
|
-
const parsed =
|
|
33
|
+
const parsed = v.safeParse(permitGrantSchema, response);
|
|
33
34
|
if (!parsed.success) {
|
|
34
35
|
throw new SemaphoreProtocolError(`Unexpected response to a permit request for "${key}"`);
|
|
35
36
|
}
|
|
36
|
-
__classPrivateFieldGet(this, _SemaphoreClient_nextRequestAt, "f").set(key, parsed.
|
|
37
|
+
__classPrivateFieldGet(this, _SemaphoreClient_nextRequestAt, "f").set(key, parsed.output.nextRequestAt);
|
|
37
38
|
// Advisory state must stay bounded for high-cardinality keys.
|
|
38
39
|
if (__classPrivateFieldGet(this, _SemaphoreClient_nextRequestAt, "f").size > 1024) {
|
|
39
40
|
const oldest = __classPrivateFieldGet(this, _SemaphoreClient_nextRequestAt, "f").keys().next().value;
|
|
@@ -41,15 +42,15 @@ export class SemaphoreClient {
|
|
|
41
42
|
__classPrivateFieldGet(this, _SemaphoreClient_nextRequestAt, "f").delete(oldest);
|
|
42
43
|
}
|
|
43
44
|
}
|
|
44
|
-
return parsed.
|
|
45
|
+
return parsed.output;
|
|
45
46
|
}
|
|
46
47
|
async sync(key, message) {
|
|
47
48
|
const response = await __classPrivateFieldGet(this, _SemaphoreClient_config, "f").invoke(__classPrivateFieldGet(this, _SemaphoreClient_instances, "m", _SemaphoreClient_address).call(this, key), message);
|
|
48
|
-
const parsed =
|
|
49
|
+
const parsed = v.safeParse(permitSyncResultSchema, response);
|
|
49
50
|
if (!parsed.success) {
|
|
50
51
|
throw new SemaphoreProtocolError(`Unexpected response to a permit sync for "${key}"`);
|
|
51
52
|
}
|
|
52
|
-
return parsed.
|
|
53
|
+
return parsed.output;
|
|
53
54
|
}
|
|
54
55
|
}
|
|
55
56
|
_SemaphoreClient_config = new WeakMap(), _SemaphoreClient_nextRequestAt = new WeakMap(), _SemaphoreClient_instances = new WeakSet(), _SemaphoreClient_address = function _SemaphoreClient_address(key) {
|
|
@@ -5,17 +5,17 @@
|
|
|
5
5
|
* @packageDocumentation
|
|
6
6
|
*/
|
|
7
7
|
import { type TransformContext, type TransformStub } from '@durability/transforms';
|
|
8
|
-
import { type DurableObjectTarget } from '
|
|
9
|
-
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './
|
|
10
|
-
import { DurabilitySemaphoreServer } from './
|
|
11
|
-
export { SemaphoreProtocolError } from '
|
|
8
|
+
import { type DurableObjectTarget } from '../durable-object-target.js';
|
|
9
|
+
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './protocol.js';
|
|
10
|
+
import { DurabilitySemaphoreServer } from './server.js';
|
|
11
|
+
export { SemaphoreProtocolError } from '../errors.js';
|
|
12
12
|
export { durabilitySemaphoreMigrations, durabilitySemaphoreQueueMigrations, } from './migrations.js';
|
|
13
|
-
export type { GrantReport, GrantResult, PermitGrant, PermitRequest, PermitSync, PermitSyncResult, SemaphoreBlockedReason, SemaphoreRequest, SemaphoreResponse, } from './
|
|
14
|
-
export type { SemaphoreAddress, SemaphoreClientConfig
|
|
15
|
-
export { SemaphoreClient } from './
|
|
16
|
-
export type { DurabilitySemaphoreConfig, DurabilitySemaphoreEvent, SemaphoreUsage, } from './
|
|
17
|
-
export type { SemaphoreLeaseOptions, SemaphoreProvisioning
|
|
18
|
-
export { DurabilitySemaphoreQueue, type AdmittedItem, type DurabilitySemaphoreQueueConfig, type DurabilitySemaphoreQueueEvent, type SemaphoreHolder, type SemaphoreInput, type SemaphoreItemStatus, } from './
|
|
13
|
+
export type { GrantReport, GrantResult, PermitGrant, PermitRequest, PermitSync, PermitSyncResult, SemaphoreBlockedReason, SemaphoreRequest, SemaphoreResponse, } from './protocol.js';
|
|
14
|
+
export type { SemaphoreAddress, SemaphoreClientConfig } from './client.js';
|
|
15
|
+
export { SemaphoreClient } from './client.js';
|
|
16
|
+
export type { DurabilitySemaphoreConfig, DurabilitySemaphoreEvent, SemaphoreUsage, } from './server.js';
|
|
17
|
+
export type { SemaphoreLeaseOptions, SemaphoreProvisioning } from './leases.js';
|
|
18
|
+
export { DurabilitySemaphoreQueue, type AdmittedItem, type DurabilitySemaphoreQueueConfig, type DurabilitySemaphoreQueueEvent, type SemaphoreHolder, type SemaphoreInput, type SemaphoreItemStatus, } from './queue.js';
|
|
19
19
|
export type SemaphoreSession = {
|
|
20
20
|
request(key: string, message: PermitRequest): Promise<PermitGrant>;
|
|
21
21
|
sync(key: string, message: PermitSync): Promise<PermitSyncResult>;
|
|
@@ -5,14 +5,14 @@
|
|
|
5
5
|
* @packageDocumentation
|
|
6
6
|
*/
|
|
7
7
|
import { createTransformStub, } from '@durability/transforms';
|
|
8
|
-
import { resolveNamespace, } from '
|
|
9
|
-
import { SemaphoreProtocolError } from '
|
|
10
|
-
import { SemaphoreClient } from './
|
|
11
|
-
import { DurabilitySemaphoreServer } from './
|
|
12
|
-
export { SemaphoreProtocolError } from '
|
|
8
|
+
import { resolveNamespace, } from '../durable-object-target.js';
|
|
9
|
+
import { SemaphoreProtocolError } from '../errors.js';
|
|
10
|
+
import { SemaphoreClient } from './client.js';
|
|
11
|
+
import { DurabilitySemaphoreServer } from './server.js';
|
|
12
|
+
export { SemaphoreProtocolError } from '../errors.js';
|
|
13
13
|
export { durabilitySemaphoreMigrations, durabilitySemaphoreQueueMigrations, } from './migrations.js';
|
|
14
|
-
export { SemaphoreClient } from './
|
|
15
|
-
export { DurabilitySemaphoreQueue, } from './
|
|
14
|
+
export { SemaphoreClient } from './client.js';
|
|
15
|
+
export { DurabilitySemaphoreQueue, } from './queue.js';
|
|
16
16
|
/**
|
|
17
17
|
* Inside a DO, holds one concurrency limit for many client Durable Objects.
|
|
18
18
|
* Outside the DO, `client()` routes semaphore messages to the application-owned
|