durability 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -12
- package/dist/{durability-alarms.d.ts → alarms/index.d.ts} +13 -5
- package/dist/{durability-alarms.js → alarms/index.js} +69 -55
- package/dist/alarms/migrations.d.ts +9 -0
- package/dist/alarms/migrations.js +133 -0
- package/dist/experimental.d.ts +10 -10
- package/dist/experimental.js +7 -7
- package/dist/{fanout.d.ts → fanout/index.d.ts} +13 -6
- package/dist/{fanout.js → fanout/index.js} +54 -46
- package/dist/fanout/migrations.d.ts +9 -0
- package/dist/fanout/migrations.js +181 -0
- package/dist/{stored-message.d.ts → fanout/stored-message.d.ts} +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.js +13 -8
- package/dist/{log.d.ts → log/index.d.ts} +15 -3
- package/dist/{log.js → log/index.js} +152 -124
- package/dist/log/migrations.d.ts +18 -0
- package/dist/log/migrations.js +297 -0
- package/dist/{memoize.d.ts → memoize/index.d.ts} +1 -1
- package/dist/{memoize.js → memoize/index.js} +14 -11
- package/dist/memoize/migrations.d.ts +10 -0
- package/dist/memoize/migrations.js +21 -0
- package/dist/migrations.d.ts +0 -111
- package/dist/migrations.js +0 -612
- package/dist/{durability.d.ts → operations/index.d.ts} +13 -5
- package/dist/{durability.js → operations/index.js} +59 -47
- package/dist/operations/migrations.d.ts +10 -0
- package/dist/operations/migrations.js +150 -0
- package/dist/{outbox.d.ts → outbox/index.d.ts} +4 -4
- package/dist/{outbox.js → outbox/index.js} +16 -60
- package/dist/outbox/migrations.d.ts +9 -0
- package/dist/outbox/migrations.js +43 -0
- package/dist/{routing-client.js → routing/client.js} +1 -1
- package/dist/{routing.d.ts → routing/index.d.ts} +3 -3
- package/dist/{routing.js → routing/index.js} +3 -3
- package/dist/{load.d.ts → routing/load.d.ts} +1 -1
- package/dist/{schedule.d.ts → schedule/index.d.ts} +16 -8
- package/dist/{schedule.js → schedule/index.js} +65 -49
- package/dist/schedule/migrations.d.ts +9 -0
- package/dist/schedule/migrations.js +118 -0
- package/dist/{scheduler.d.ts → scheduler/index.d.ts} +75 -25
- package/dist/{scheduler.js → scheduler/index.js} +300 -63
- package/dist/{semaphore-client.d.ts → semaphore/client.d.ts} +1 -1
- package/dist/{semaphore-client.js → semaphore/client.js} +2 -2
- package/dist/{semaphore.d.ts → semaphore/index.d.ts} +10 -10
- package/dist/{semaphore.js → semaphore/index.js} +7 -7
- package/dist/{semaphore-leases.d.ts → semaphore/leases.d.ts} +17 -6
- package/dist/{semaphore-leases.js → semaphore/leases.js} +28 -12
- package/dist/semaphore/migrations.d.ts +18 -0
- package/dist/semaphore/migrations.js +444 -0
- package/dist/{semaphore-protocol.d.ts → semaphore/protocol.d.ts} +16 -8
- package/dist/{semaphore-protocol.js → semaphore/protocol.js} +37 -14
- package/dist/{semaphore-queue-store.d.ts → semaphore/queue-store.d.ts} +12 -6
- package/dist/{semaphore-queue-store.js → semaphore/queue-store.js} +69 -61
- package/dist/{semaphore-queue.d.ts → semaphore/queue.d.ts} +15 -5
- package/dist/{semaphore-queue.js → semaphore/queue.js} +105 -46
- package/dist/{semaphore-server.d.ts → semaphore/server.d.ts} +12 -4
- package/dist/{semaphore-server.js → semaphore/server.js} +51 -24
- package/dist/{semaphore-store.d.ts → semaphore/store.d.ts} +8 -3
- package/dist/{semaphore-store.js → semaphore/store.js} +35 -33
- package/dist/{client.d.ts → socket/client.d.ts} +3 -3
- package/dist/{client.js → socket/client.js} +4 -4
- package/dist/{socket.d.ts → socket/index.d.ts} +15 -4
- package/dist/{socket.js → socket/index.js} +66 -20
- package/dist/socket/migrations.d.ts +9 -0
- package/dist/socket/migrations.js +57 -0
- package/dist/{socket-protocol.d.ts → socket/protocol.d.ts} +1 -1
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +13 -6
- package/dist/{topology-client.d.ts → topology/client.d.ts} +2 -2
- package/dist/{topology-client.js → topology/client.js} +3 -3
- package/dist/{topology.d.ts → topology/index.d.ts} +17 -7
- package/dist/{topology.js → topology/index.js} +92 -58
- package/dist/topology/migrations.d.ts +10 -0
- package/dist/topology/migrations.js +138 -0
- package/dist/{tracker.d.ts → tracker/index.d.ts} +3 -3
- package/dist/{tracker.js → tracker/index.js} +2 -2
- package/package.json +11 -11
- /package/dist/{stored-message.js → fanout/stored-message.js} +0 -0
- /package/dist/{routing-client.d.ts → routing/client.d.ts} +0 -0
- /package/dist/{load.js → routing/load.js} +0 -0
- /package/dist/{socket-protocol.js → socket/protocol.js} +0 -0
- /package/dist/{topology-snapshot.d.ts → topology/snapshot.d.ts} +0 -0
- /package/dist/{topology-snapshot.js → topology/snapshot.js} +0 -0
- /package/dist/{tracked-source.d.ts → tracker/source.d.ts} +0 -0
- /package/dist/{tracked-source.js → tracker/source.js} +0 -0
|
@@ -1,21 +1,44 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { trace, traceMethods, traceStorage } from '
|
|
3
|
-
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from '
|
|
4
|
-
import { createEmitter, } from '
|
|
5
|
-
import { isSchemaCurrent, migrateCapability, } from '
|
|
6
|
-
import { assertPositiveInteger } from '
|
|
7
|
-
import { deleteMemos, deleteStaleMemos } from '
|
|
8
|
-
import { claimAttempt, exhaustAttempts, updateAttempt, } from '
|
|
2
|
+
import { trace, traceMethods, traceStorage } from '../dev.js';
|
|
3
|
+
import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from '../errors.js';
|
|
4
|
+
import { createEmitter, } from '../lifecycle.js';
|
|
5
|
+
import { isSchemaCurrent, migrateCapability, } from '../migrations.js';
|
|
6
|
+
import { assertPositiveInteger } from '../policy.js';
|
|
7
|
+
import { deleteMemos, deleteStaleMemos } from '../memoize/index.js';
|
|
8
|
+
import { channelBindings, channelClause, claimAttempt, exhaustAttempts, tableOwner, updateAttempt, } from '../storage.js';
|
|
9
9
|
const minAlarmDelayMs = 1000;
|
|
10
|
+
const failedRunRetryMs = 1000;
|
|
11
|
+
const defaultAlarmConcurrency = 10;
|
|
12
|
+
const schedulerOptionKeys = [
|
|
13
|
+
'alarmConcurrency',
|
|
14
|
+
'alarmMinDelayMs',
|
|
15
|
+
'alarmHandoffMs',
|
|
16
|
+
'liveTimers',
|
|
17
|
+
'onAlarmRetry',
|
|
18
|
+
];
|
|
10
19
|
const maxAlarmDelayMs = 15000;
|
|
11
20
|
// setTimeout treats delays past this as zero, so anything further out can only
|
|
12
21
|
// be served by the physical alarm fallback.
|
|
13
22
|
const maxTimerMs = 2 ** 31 - 1;
|
|
14
23
|
const pendingHeadRowSchema = z.object({
|
|
15
24
|
pk: z.string(),
|
|
25
|
+
channel: z.string().optional(),
|
|
16
26
|
next_attempt_at: z.number().int().nonnegative(),
|
|
17
27
|
});
|
|
18
28
|
const countRowSchema = z.object({ count: z.number().int().nonnegative() });
|
|
29
|
+
/** The channel every helper uses when it is not given one. */
|
|
30
|
+
export const defaultChannel = 'default';
|
|
31
|
+
/** How long records of a channel no instance attached wait before they are looked at again. */
|
|
32
|
+
const unattachedChannelRetryMs = 60000;
|
|
33
|
+
const channelRowSchema = z.object({ channel: z.string() });
|
|
34
|
+
const activeKey = (channel, key) => JSON.stringify([channel ?? null, key]);
|
|
35
|
+
/** Rejects a channel name a helper would otherwise store rows under. */
|
|
36
|
+
export const assertChannel = (helper, channel) => {
|
|
37
|
+
if (typeof channel !== 'string' || channel.length === 0) {
|
|
38
|
+
throw new TypeError(`${helper} channel must be a non-empty string`);
|
|
39
|
+
}
|
|
40
|
+
return channel;
|
|
41
|
+
};
|
|
19
42
|
const decideRetry = (spec, attempt, error) => {
|
|
20
43
|
const terminal = isNonRetryable(error) ||
|
|
21
44
|
attempt >= spec.policy.maxAttempts ||
|
|
@@ -60,9 +83,21 @@ const decideRetry = (spec, attempt, error) => {
|
|
|
60
83
|
*/
|
|
61
84
|
export class Engine {
|
|
62
85
|
constructor(context, options) {
|
|
86
|
+
this.alarmConcurrency = defaultAlarmConcurrency;
|
|
87
|
+
this.alarmMinDelayMs = 15000;
|
|
88
|
+
this.alarmHandoffMs = 14 * 60000;
|
|
89
|
+
this.liveTimers = false;
|
|
90
|
+
// Only options someone set can conflict; an omitted one takes any value.
|
|
91
|
+
this.explicitOptions = {};
|
|
63
92
|
this.participants = [];
|
|
93
|
+
this.channelsByTable = new Map();
|
|
94
|
+
this.claimedChannels = new Map();
|
|
95
|
+
this.reportedMissingChannels = new Set();
|
|
64
96
|
this.migrated = new Set();
|
|
97
|
+
this.availablePermits = defaultAlarmConcurrency;
|
|
65
98
|
this.permitWaiters = [];
|
|
99
|
+
this.alarmRefreshRequested = false;
|
|
100
|
+
this.failedRuns = 0;
|
|
66
101
|
this.watchers = new Map();
|
|
67
102
|
// Keys whose attempt is currently executing, per table. The in-memory timer
|
|
68
103
|
// skips these so a still-due active row cannot drive a zero-delay timer loop.
|
|
@@ -70,23 +105,60 @@ export class Engine {
|
|
|
70
105
|
this.context = context;
|
|
71
106
|
this.storage = traceStorage(context.storage);
|
|
72
107
|
this.sql = this.storage.sql;
|
|
73
|
-
this.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
108
|
+
this.configure(options);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Joins one more helper or scheduler handle to this object's scheduler. The
|
|
112
|
+
* first context that can `waitUntil` is kept, so a bare `{ storage }` context
|
|
113
|
+
* attached first does not hide it from the rest.
|
|
114
|
+
*/
|
|
115
|
+
attach(context, options) {
|
|
116
|
+
if (this.context.waitUntil === undefined &&
|
|
117
|
+
context.waitUntil !== undefined) {
|
|
118
|
+
this.context = context;
|
|
119
|
+
}
|
|
120
|
+
this.configure(options);
|
|
121
|
+
}
|
|
122
|
+
configure(options) {
|
|
123
|
+
const merged = { ...this.explicitOptions };
|
|
124
|
+
for (const key of schedulerOptionKeys) {
|
|
125
|
+
const value = options[key];
|
|
126
|
+
if (value === undefined) {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
const current = this.explicitOptions[key];
|
|
130
|
+
if (current !== undefined && current !== value) {
|
|
131
|
+
throw new Error(`Scheduler option "${key}" is already set to a different value for this Durable Object`);
|
|
132
|
+
}
|
|
133
|
+
Object.assign(merged, { [key]: value });
|
|
134
|
+
}
|
|
135
|
+
const alarmConcurrency = merged.alarmConcurrency ?? defaultAlarmConcurrency;
|
|
136
|
+
const alarmMinDelayMs = merged.alarmMinDelayMs ?? 15000;
|
|
137
|
+
const alarmHandoffMs = merged.alarmHandoffMs ?? 14 * 60000;
|
|
138
|
+
if (merged.onAlarmRetry !== undefined &&
|
|
139
|
+
typeof merged.onAlarmRetry !== 'function') {
|
|
80
140
|
throw new TypeError('onAlarmRetry must be a function');
|
|
81
141
|
}
|
|
82
|
-
assertPositiveInteger('alarmConcurrency',
|
|
83
|
-
assertPositiveInteger('alarmMinDelayMs',
|
|
84
|
-
if (
|
|
85
|
-
|
|
142
|
+
assertPositiveInteger('alarmConcurrency', alarmConcurrency);
|
|
143
|
+
assertPositiveInteger('alarmMinDelayMs', alarmMinDelayMs);
|
|
144
|
+
if (alarmMinDelayMs < minAlarmDelayMs ||
|
|
145
|
+
alarmMinDelayMs > maxAlarmDelayMs) {
|
|
86
146
|
throw new RangeError(`alarmMinDelayMs must be between ${minAlarmDelayMs} and ${maxAlarmDelayMs}`);
|
|
87
147
|
}
|
|
88
|
-
assertPositiveInteger('alarmHandoffMs',
|
|
89
|
-
|
|
148
|
+
assertPositiveInteger('alarmHandoffMs', alarmHandoffMs);
|
|
149
|
+
// Permits already handed out stay out; raising the limit frees more now.
|
|
150
|
+
this.availablePermits += alarmConcurrency - this.alarmConcurrency;
|
|
151
|
+
this.alarmConcurrency = alarmConcurrency;
|
|
152
|
+
this.alarmMinDelayMs = alarmMinDelayMs;
|
|
153
|
+
this.alarmHandoffMs = alarmHandoffMs;
|
|
154
|
+
this.liveTimers = merged.liveTimers ?? false;
|
|
155
|
+
this.onAlarmRetry = merged.onAlarmRetry;
|
|
156
|
+
this.explicitOptions = merged;
|
|
157
|
+
while (this.availablePermits > 0 && this.permitWaiters.length > 0) {
|
|
158
|
+
const waiter = this.permitWaiters.shift();
|
|
159
|
+
this.availablePermits -= 1;
|
|
160
|
+
waiter(this.createRelease());
|
|
161
|
+
}
|
|
90
162
|
}
|
|
91
163
|
register(participant) {
|
|
92
164
|
const { name } = participant.table;
|
|
@@ -95,6 +167,106 @@ export class Engine {
|
|
|
95
167
|
}
|
|
96
168
|
this.participants.push(participant);
|
|
97
169
|
}
|
|
170
|
+
/** Reserves an instance identity, including for helpers that do not schedule work. */
|
|
171
|
+
claimChannel(kind, channel, helper) {
|
|
172
|
+
const claimed = this.claimedChannels.get(kind) ?? new Set();
|
|
173
|
+
if (claimed.has(channel)) {
|
|
174
|
+
throw new Error(`${helper} for channel "${channel}" is already attached to this scheduler`);
|
|
175
|
+
}
|
|
176
|
+
claimed.add(channel);
|
|
177
|
+
this.claimedChannels.set(kind, claimed);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Attaches one channel of a table several helper instances share. The first
|
|
181
|
+
* attachment registers the table; each run then runs every attached
|
|
182
|
+
* channel's due rows and retains the rows of channels nothing attached.
|
|
183
|
+
*/
|
|
184
|
+
registerChannel(participant) {
|
|
185
|
+
const { table, channel, helper } = participant;
|
|
186
|
+
let channels = this.channelsByTable.get(table.name);
|
|
187
|
+
if (!channels) {
|
|
188
|
+
const created = new Map();
|
|
189
|
+
this.register({
|
|
190
|
+
table: { name: table.name, key: table.key },
|
|
191
|
+
runDue: async (now, info) => {
|
|
192
|
+
await this.postponeUnattachedChannels(table, created);
|
|
193
|
+
await Promise.all([...created.values()].map((attached) => attached.runDue(now, info)));
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
this.channelsByTable.set(table.name, created);
|
|
197
|
+
channels = created;
|
|
198
|
+
}
|
|
199
|
+
this.claimChannel(table.name, channel, helper);
|
|
200
|
+
channels.set(channel, {
|
|
201
|
+
...participant,
|
|
202
|
+
table: { ...table, channel },
|
|
203
|
+
});
|
|
204
|
+
if (participant.nextPendingAt !== undefined) {
|
|
205
|
+
// Missing custom channels are dormant. Defer until construction finishes,
|
|
206
|
+
// then restore their alarm without requiring a new application write.
|
|
207
|
+
const onError = (error) => reportFailure('durability.channel_resume_failed', { table: table.name, channel }, error);
|
|
208
|
+
const resumed = Promise.resolve()
|
|
209
|
+
.then(() => {
|
|
210
|
+
if (participant.nextPendingAt?.(this.sql) !== undefined) {
|
|
211
|
+
return this.scheduleNextAlarm();
|
|
212
|
+
}
|
|
213
|
+
return undefined;
|
|
214
|
+
})
|
|
215
|
+
.catch(async (error) => {
|
|
216
|
+
if (await this.armFallbackAlarm()) {
|
|
217
|
+
this.retryFailedRun();
|
|
218
|
+
}
|
|
219
|
+
onError(error);
|
|
220
|
+
});
|
|
221
|
+
this.waitUntil(resumed, onError);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Due rows of a channel no instance attached, such as one a deploy removed
|
|
226
|
+
* or a library not yet constructed, would keep the alarm awake forever. They
|
|
227
|
+
* are kept and looked at again later, since their owner may return.
|
|
228
|
+
*/
|
|
229
|
+
async postponeUnattachedChannels(table, channels) {
|
|
230
|
+
const now = Date.now();
|
|
231
|
+
const customTiming = [...channels.values()].some((participant) => participant.nextPendingAt !== undefined);
|
|
232
|
+
const channelTables = new Set([
|
|
233
|
+
table.name,
|
|
234
|
+
...[...channels.values()].flatMap((participant) => participant.channelTables ?? []),
|
|
235
|
+
]);
|
|
236
|
+
const query = customTiming
|
|
237
|
+
? [...channelTables]
|
|
238
|
+
.map((name) => `SELECT channel FROM ${name}`)
|
|
239
|
+
.join(' UNION ')
|
|
240
|
+
: `SELECT DISTINCT channel FROM ${table.name}
|
|
241
|
+
WHERE status = 'pending' AND next_attempt_at <= ?`;
|
|
242
|
+
const unattached = channelRowSchema
|
|
243
|
+
.array()
|
|
244
|
+
.parse(this.sql.exec(query, ...(customTiming ? [] : [now])).toArray())
|
|
245
|
+
.map((row) => row.channel)
|
|
246
|
+
.filter((channel) => !channels.has(channel));
|
|
247
|
+
if (unattached.length === 0) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
for (const channel of unattached) {
|
|
251
|
+
const identity = JSON.stringify([table.name, channel]);
|
|
252
|
+
if (customTiming && this.reportedMissingChannels.has(identity)) {
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
this.reportedMissingChannels.add(identity);
|
|
256
|
+
reportFailure('durability.channel_missing', { table: table.name, channel }, new Error(`Channel "${channel}" of ${table.name} has no helper attached; retaining its records`));
|
|
257
|
+
}
|
|
258
|
+
// Custom timing may represent leases or recurring configurations, not pending
|
|
259
|
+
// attempts. Retain those records unchanged until their channel returns.
|
|
260
|
+
if (customTiming) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
await this.transaction((sql) => {
|
|
264
|
+
for (const channel of unattached) {
|
|
265
|
+
sql.exec(`UPDATE ${table.name} SET next_attempt_at = ?
|
|
266
|
+
WHERE channel = ? AND status = 'pending' AND next_attempt_at <= ?`, now + unattachedChannelRetryMs, channel, now);
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
}
|
|
98
270
|
waitUntil(promise, onError) {
|
|
99
271
|
if (!this.context.waitUntil) {
|
|
100
272
|
return;
|
|
@@ -168,10 +340,27 @@ export class Engine {
|
|
|
168
340
|
/**
|
|
169
341
|
* The earliest pending time for one participant: `physical` arms the physical
|
|
170
342
|
* alarm (eviction fallback) and includes active work; `timer` drives the
|
|
171
|
-
* in-memory timer and
|
|
343
|
+
* in-memory timer and skips work that is already executing
|
|
172
344
|
* so a still-due active row cannot re-fire a zero-delay timer in a loop.
|
|
173
345
|
*/
|
|
174
346
|
pendingTimes(participant) {
|
|
347
|
+
const channels = this.channelsByTable.get(participant.table.name);
|
|
348
|
+
if (participant.table.channel === undefined &&
|
|
349
|
+
channels !== undefined &&
|
|
350
|
+
[...channels.values()].some((channel) => channel.nextPendingAt !== undefined)) {
|
|
351
|
+
let physical;
|
|
352
|
+
let timer;
|
|
353
|
+
for (const channel of channels.values()) {
|
|
354
|
+
const next = this.pendingTimes(channel);
|
|
355
|
+
if (next.physical !== undefined) {
|
|
356
|
+
physical = Math.min(physical ?? Infinity, next.physical);
|
|
357
|
+
}
|
|
358
|
+
if (next.timer !== undefined) {
|
|
359
|
+
timer = Math.min(timer ?? Infinity, next.timer);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return { physical, timer };
|
|
363
|
+
}
|
|
175
364
|
if (participant.nextPendingAt) {
|
|
176
365
|
const at = participant.nextPendingAt(this.sql);
|
|
177
366
|
if (at === undefined) {
|
|
@@ -179,25 +368,25 @@ export class Engine {
|
|
|
179
368
|
}
|
|
180
369
|
return participant.holding?.()
|
|
181
370
|
? { physical: at }
|
|
182
|
-
: {
|
|
371
|
+
: {
|
|
372
|
+
physical: at,
|
|
373
|
+
timer: participant.nextTimerAt
|
|
374
|
+
? participant.nextTimerAt(this.sql)
|
|
375
|
+
: at,
|
|
376
|
+
};
|
|
183
377
|
}
|
|
184
378
|
const active = this.activeByTable.get(participant.table.name);
|
|
185
|
-
const
|
|
379
|
+
const partitioned = this.channelsByTable.has(participant.table.name);
|
|
380
|
+
const limit = (active?.size ?? 0) + 1;
|
|
186
381
|
const rows = pendingHeadRowSchema.array().parse(this.sql
|
|
187
|
-
.exec(`SELECT ${participant.table.key} AS pk, next_attempt_at
|
|
382
|
+
.exec(`SELECT ${participant.table.key} AS pk, next_attempt_at${partitioned ? ', channel' : ''}
|
|
188
383
|
FROM ${participant.table.name}
|
|
189
384
|
WHERE status = 'pending'
|
|
190
385
|
ORDER BY next_attempt_at ASC
|
|
191
386
|
LIMIT ?`, limit)
|
|
192
387
|
.toArray());
|
|
193
388
|
const physical = rows[0]?.next_attempt_at;
|
|
194
|
-
|
|
195
|
-
if (physical === undefined) {
|
|
196
|
-
return {};
|
|
197
|
-
}
|
|
198
|
-
return { physical, timer: physical };
|
|
199
|
-
}
|
|
200
|
-
const eligible = rows.find((row) => !active?.has(row.pk));
|
|
389
|
+
const eligible = rows.find((row) => !active?.has(activeKey(row.channel, row.pk)));
|
|
201
390
|
return { physical, timer: eligible?.next_attempt_at };
|
|
202
391
|
}
|
|
203
392
|
/**
|
|
@@ -267,14 +456,25 @@ export class Engine {
|
|
|
267
456
|
}, Math.max(delay, 0));
|
|
268
457
|
this.earlyAlarm = { timestamp, timer };
|
|
269
458
|
}
|
|
270
|
-
/** Coalesces
|
|
459
|
+
/** Coalesces refreshes without dropping requests arriving after the pending-work read. */
|
|
271
460
|
scheduleNextAlarm() {
|
|
461
|
+
this.alarmRefreshRequested = true;
|
|
272
462
|
if (this.alarmRefresh) {
|
|
273
463
|
return this.alarmRefresh;
|
|
274
464
|
}
|
|
275
|
-
this.alarmRefresh =
|
|
276
|
-
|
|
277
|
-
|
|
465
|
+
this.alarmRefresh = (async () => {
|
|
466
|
+
try {
|
|
467
|
+
do {
|
|
468
|
+
// eslint-disable-next-line no-await-in-loop
|
|
469
|
+
await this.transaction(() => {
|
|
470
|
+
this.alarmRefreshRequested = false;
|
|
471
|
+
});
|
|
472
|
+
} while (this.alarmRefreshRequested);
|
|
473
|
+
}
|
|
474
|
+
finally {
|
|
475
|
+
this.alarmRefresh = undefined;
|
|
476
|
+
}
|
|
477
|
+
})();
|
|
278
478
|
return this.alarmRefresh;
|
|
279
479
|
}
|
|
280
480
|
/**
|
|
@@ -290,16 +490,18 @@ export class Engine {
|
|
|
290
490
|
running.controller?.abort(new Error(abortReason));
|
|
291
491
|
}
|
|
292
492
|
}
|
|
493
|
+
const where = `created_at < ?${channelClause(table)}`;
|
|
494
|
+
const params = [before, ...channelBindings(table)];
|
|
293
495
|
const removed = await this.transaction((sql) => {
|
|
294
496
|
const { count } = countRowSchema.parse(sql
|
|
295
|
-
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE
|
|
497
|
+
.exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE ${where}`, ...params)
|
|
296
498
|
.toArray()[0]);
|
|
297
|
-
sql.exec(`DELETE FROM ${table.name} WHERE
|
|
499
|
+
sql.exec(`DELETE FROM ${table.name} WHERE ${where}`, ...params);
|
|
298
500
|
deleteStaleMemos(sql, table);
|
|
299
501
|
return count;
|
|
300
502
|
});
|
|
301
503
|
if (removed > 0) {
|
|
302
|
-
for (const key of this.watchers.get(table
|
|
504
|
+
for (const key of this.watchers.get(tableOwner(table))?.keys() ?? []) {
|
|
303
505
|
this.notify(table, key);
|
|
304
506
|
}
|
|
305
507
|
}
|
|
@@ -307,9 +509,9 @@ export class Engine {
|
|
|
307
509
|
}
|
|
308
510
|
/** In-memory only: listeners never survive eviction. */
|
|
309
511
|
watch(table, key, listener) {
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
this.watchers.set(
|
|
512
|
+
const owner = tableOwner(table);
|
|
513
|
+
const byKey = this.watchers.get(owner) ?? new Map();
|
|
514
|
+
this.watchers.set(owner, byKey);
|
|
313
515
|
const listeners = byKey.get(key) ?? new Set();
|
|
314
516
|
byKey.set(key, listeners);
|
|
315
517
|
listeners.add(listener);
|
|
@@ -322,12 +524,13 @@ export class Engine {
|
|
|
322
524
|
}
|
|
323
525
|
/** Call only after the change has committed. */
|
|
324
526
|
notify(table, key, completed) {
|
|
325
|
-
for (const listener of this.watchers.get(table
|
|
527
|
+
for (const listener of this.watchers.get(tableOwner(table))?.get(key) ??
|
|
528
|
+
[]) {
|
|
326
529
|
listener(completed);
|
|
327
530
|
}
|
|
328
531
|
}
|
|
329
532
|
tryAcquirePermit() {
|
|
330
|
-
if (this.availablePermits
|
|
533
|
+
if (this.availablePermits <= 0) {
|
|
331
534
|
return undefined;
|
|
332
535
|
}
|
|
333
536
|
this.availablePermits -= 1;
|
|
@@ -349,13 +552,15 @@ export class Engine {
|
|
|
349
552
|
return;
|
|
350
553
|
}
|
|
351
554
|
released = true;
|
|
555
|
+
this.availablePermits += 1;
|
|
556
|
+
if (this.availablePermits <= 0) {
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
352
559
|
const waiter = this.permitWaiters.shift();
|
|
353
560
|
if (waiter) {
|
|
561
|
+
this.availablePermits -= 1;
|
|
354
562
|
waiter(this.createRelease());
|
|
355
563
|
}
|
|
356
|
-
else {
|
|
357
|
-
this.availablePermits += 1;
|
|
358
|
-
}
|
|
359
564
|
};
|
|
360
565
|
}
|
|
361
566
|
async runConcurrent(items, worker) {
|
|
@@ -378,6 +583,7 @@ export class Engine {
|
|
|
378
583
|
async alarm(info) {
|
|
379
584
|
try {
|
|
380
585
|
await this.runAlarm(info);
|
|
586
|
+
this.failedRuns = 0;
|
|
381
587
|
}
|
|
382
588
|
catch (error) {
|
|
383
589
|
// Throwing lets the platform retry, but it gives up after a few failures
|
|
@@ -388,8 +594,20 @@ export class Engine {
|
|
|
388
594
|
}
|
|
389
595
|
reportFailure('durability.alarm.failed', { retryInMs: this.alarmMinDelayMs }, error);
|
|
390
596
|
this.notifyAlarmRetry(error, info);
|
|
597
|
+
this.retryFailedRun();
|
|
391
598
|
}
|
|
392
599
|
}
|
|
600
|
+
/**
|
|
601
|
+
* A transient storage fault would otherwise leave all due work to the
|
|
602
|
+
* fallback alarm, up to `alarmMinDelayMs` away: longer than a holder's
|
|
603
|
+
* grant window, for one. The in-memory timer retries sooner, backing off to
|
|
604
|
+
* that fallback while runs keep failing.
|
|
605
|
+
*/
|
|
606
|
+
retryFailedRun() {
|
|
607
|
+
this.failedRuns += 1;
|
|
608
|
+
const delay = Math.min(failedRunRetryMs * 2 ** (this.failedRuns - 1), this.alarmMinDelayMs);
|
|
609
|
+
this.scheduleEarlyAlarm(Date.now() + delay);
|
|
610
|
+
}
|
|
393
611
|
notifyAlarmRetry(failure, invocation) {
|
|
394
612
|
const hook = this.onAlarmRetry;
|
|
395
613
|
if (!hook) {
|
|
@@ -485,7 +703,7 @@ export class Engine {
|
|
|
485
703
|
settled,
|
|
486
704
|
};
|
|
487
705
|
spec.active.set(key, activeExecution);
|
|
488
|
-
this.trackActive(spec.table.name, key);
|
|
706
|
+
this.trackActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
489
707
|
let releasePermit = eagerPermit;
|
|
490
708
|
let handlerStarted = false;
|
|
491
709
|
let handlerSettled = false;
|
|
@@ -567,12 +785,10 @@ export class Engine {
|
|
|
567
785
|
if (spec.active.get(key) === activeExecution) {
|
|
568
786
|
spec.active.delete(key);
|
|
569
787
|
}
|
|
570
|
-
this.releaseActive(spec.table.name, key);
|
|
788
|
+
this.releaseActive(spec.table.name, activeKey(spec.table.channel, key));
|
|
571
789
|
// Re-arm the in-memory timer now that this key is eligible again, so work
|
|
572
790
|
// the timer skipped while it executed is picked up at its exact time.
|
|
573
|
-
|
|
574
|
-
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
575
|
-
}
|
|
791
|
+
void this.scheduleNextAlarm().catch((error) => reportFailure('durability.scheduler.refresh_failed', { table: spec.table.name, key }, error));
|
|
576
792
|
});
|
|
577
793
|
return execution;
|
|
578
794
|
}
|
|
@@ -634,14 +850,38 @@ export class Engine {
|
|
|
634
850
|
}
|
|
635
851
|
}
|
|
636
852
|
const engines = new WeakMap();
|
|
853
|
+
// Every copy of this package in an isolate shares the registry, so a second
|
|
854
|
+
// copy reaching the same object is caught instead of fighting over its alarm.
|
|
855
|
+
const installation = Symbol('durability installation');
|
|
856
|
+
const globalRegistry = globalThis;
|
|
857
|
+
const registryKey = Symbol.for('durability.object-schedulers');
|
|
858
|
+
globalRegistry[registryKey] ?? (globalRegistry[registryKey] = new WeakMap());
|
|
859
|
+
const objectSchedulers = globalRegistry[registryKey];
|
|
860
|
+
/** The one engine behind every scheduler and helper of a Durable Object. */
|
|
861
|
+
const engineForObject = (config) => {
|
|
862
|
+
const { context } = config;
|
|
863
|
+
const existing = objectSchedulers.get(context.storage);
|
|
864
|
+
if (existing === undefined) {
|
|
865
|
+
const engine = new Engine(context, config);
|
|
866
|
+
objectSchedulers.set(context.storage, { engine, installation });
|
|
867
|
+
return engine;
|
|
868
|
+
}
|
|
869
|
+
if (existing.installation !== installation) {
|
|
870
|
+
throw new Error('Another copy of durability is already attached to this Durable Object; install durability once, for example as a peer dependency');
|
|
871
|
+
}
|
|
872
|
+
const engine = existing.engine;
|
|
873
|
+
engine.attach(context, config);
|
|
874
|
+
return engine;
|
|
875
|
+
};
|
|
637
876
|
/**
|
|
638
|
-
* Owns the Durable Object's physical alarm, storage, and concurrency
|
|
639
|
-
*
|
|
877
|
+
* Owns the Durable Object's physical alarm, storage, and concurrency pool so
|
|
878
|
+
* every durability helper on the object shares them.
|
|
640
879
|
*
|
|
641
|
-
*
|
|
642
|
-
*
|
|
643
|
-
*
|
|
644
|
-
*
|
|
880
|
+
* A Durable Object has one scheduler. Constructing another for the same
|
|
881
|
+
* object, or a helper from the object's context, joins it; scheduler options
|
|
882
|
+
* merge, and a different value for an option already set throws. Delegate the
|
|
883
|
+
* object's `alarm` method to {@link DurabilityScheduler.alarm} or to any
|
|
884
|
+
* helper's `alarm`: each runs the due work of every helper on the object.
|
|
645
885
|
*
|
|
646
886
|
* @example
|
|
647
887
|
* ```ts
|
|
@@ -664,7 +904,7 @@ const engines = new WeakMap();
|
|
|
664
904
|
*/
|
|
665
905
|
export class DurabilityScheduler {
|
|
666
906
|
constructor(config) {
|
|
667
|
-
engines.set(this,
|
|
907
|
+
engines.set(this, engineForObject(config));
|
|
668
908
|
}
|
|
669
909
|
/** Runs due work for every attached helper and re-arms the physical alarm. */
|
|
670
910
|
alarm(info) {
|
|
@@ -672,13 +912,10 @@ export class DurabilityScheduler {
|
|
|
672
912
|
}
|
|
673
913
|
}
|
|
674
914
|
traceMethods(DurabilityScheduler.prototype, 'DurabilityScheduler');
|
|
675
|
-
/**
|
|
676
|
-
* Resolves the engine a helper should attach to: the shared one behind a
|
|
677
|
-
* scheduler, or a private one built from the Durable Object context.
|
|
678
|
-
*/
|
|
915
|
+
/** Resolves the engine a helper attaches to: the object's one scheduler. */
|
|
679
916
|
export const engineFor = (attachment) => {
|
|
680
917
|
if (attachment.scheduler === undefined) {
|
|
681
|
-
return
|
|
918
|
+
return engineForObject(attachment);
|
|
682
919
|
}
|
|
683
920
|
const engine = engines.get(attachment.scheduler);
|
|
684
921
|
if (!engine) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './
|
|
1
|
+
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './protocol.js';
|
|
2
2
|
export type SemaphoreAddress = {
|
|
3
3
|
/** The semaphore key: the name of the Durable Object holding the limit. */
|
|
4
4
|
key: string;
|
|
@@ -10,8 +10,8 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
12
|
var _SemaphoreClient_instances, _SemaphoreClient_config, _SemaphoreClient_nextRequestAt, _SemaphoreClient_address;
|
|
13
|
-
import { SemaphoreProtocolError } from '
|
|
14
|
-
import { permitGrantSchema, permitSyncResultSchema, } from './
|
|
13
|
+
import { SemaphoreProtocolError } from '../errors.js';
|
|
14
|
+
import { permitGrantSchema, permitSyncResultSchema, } from './protocol.js';
|
|
15
15
|
/**
|
|
16
16
|
* Transport-independent protocol core. Validates responses and remembers each
|
|
17
17
|
* key's `nextRequestAt` so callers can pace requests the way the holder asked.
|
|
@@ -5,17 +5,17 @@
|
|
|
5
5
|
* @packageDocumentation
|
|
6
6
|
*/
|
|
7
7
|
import { type TransformContext, type TransformStub } from '@durability/transforms';
|
|
8
|
-
import { type DurableObjectTarget } from '
|
|
9
|
-
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './
|
|
10
|
-
import { DurabilitySemaphoreServer } from './
|
|
11
|
-
export { SemaphoreProtocolError } from '
|
|
8
|
+
import { type DurableObjectTarget } from '../durable-object-target.js';
|
|
9
|
+
import { type PermitGrant, type PermitRequest, type PermitSync, type PermitSyncResult, type SemaphoreRequest, type SemaphoreResponse } from './protocol.js';
|
|
10
|
+
import { DurabilitySemaphoreServer } from './server.js';
|
|
11
|
+
export { SemaphoreProtocolError } from '../errors.js';
|
|
12
12
|
export { durabilitySemaphoreMigrations, durabilitySemaphoreQueueMigrations, } from './migrations.js';
|
|
13
|
-
export type { GrantReport, GrantResult, PermitGrant, PermitRequest, PermitSync, PermitSyncResult, SemaphoreBlockedReason, SemaphoreRequest, SemaphoreResponse, } from './
|
|
14
|
-
export type { SemaphoreAddress, SemaphoreClientConfig
|
|
15
|
-
export { SemaphoreClient } from './
|
|
16
|
-
export type { DurabilitySemaphoreConfig, DurabilitySemaphoreEvent, SemaphoreUsage, } from './
|
|
17
|
-
export type { SemaphoreLeaseOptions, SemaphoreProvisioning
|
|
18
|
-
export { DurabilitySemaphoreQueue, type AdmittedItem, type DurabilitySemaphoreQueueConfig, type DurabilitySemaphoreQueueEvent, type SemaphoreHolder, type SemaphoreInput, type SemaphoreItemStatus, } from './
|
|
13
|
+
export type { GrantReport, GrantResult, PermitGrant, PermitRequest, PermitSync, PermitSyncResult, SemaphoreBlockedReason, SemaphoreRequest, SemaphoreResponse, } from './protocol.js';
|
|
14
|
+
export type { SemaphoreAddress, SemaphoreClientConfig } from './client.js';
|
|
15
|
+
export { SemaphoreClient } from './client.js';
|
|
16
|
+
export type { DurabilitySemaphoreConfig, DurabilitySemaphoreEvent, SemaphoreUsage, } from './server.js';
|
|
17
|
+
export type { SemaphoreLeaseOptions, SemaphoreProvisioning } from './leases.js';
|
|
18
|
+
export { DurabilitySemaphoreQueue, type AdmittedItem, type DurabilitySemaphoreQueueConfig, type DurabilitySemaphoreQueueEvent, type SemaphoreHolder, type SemaphoreInput, type SemaphoreItemStatus, } from './queue.js';
|
|
19
19
|
export type SemaphoreSession = {
|
|
20
20
|
request(key: string, message: PermitRequest): Promise<PermitGrant>;
|
|
21
21
|
sync(key: string, message: PermitSync): Promise<PermitSyncResult>;
|
|
@@ -5,14 +5,14 @@
|
|
|
5
5
|
* @packageDocumentation
|
|
6
6
|
*/
|
|
7
7
|
import { createTransformStub, } from '@durability/transforms';
|
|
8
|
-
import { resolveNamespace, } from '
|
|
9
|
-
import { SemaphoreProtocolError } from '
|
|
10
|
-
import { SemaphoreClient } from './
|
|
11
|
-
import { DurabilitySemaphoreServer } from './
|
|
12
|
-
export { SemaphoreProtocolError } from '
|
|
8
|
+
import { resolveNamespace, } from '../durable-object-target.js';
|
|
9
|
+
import { SemaphoreProtocolError } from '../errors.js';
|
|
10
|
+
import { SemaphoreClient } from './client.js';
|
|
11
|
+
import { DurabilitySemaphoreServer } from './server.js';
|
|
12
|
+
export { SemaphoreProtocolError } from '../errors.js';
|
|
13
13
|
export { durabilitySemaphoreMigrations, durabilitySemaphoreQueueMigrations, } from './migrations.js';
|
|
14
|
-
export { SemaphoreClient } from './
|
|
15
|
-
export { DurabilitySemaphoreQueue, } from './
|
|
14
|
+
export { SemaphoreClient } from './client.js';
|
|
15
|
+
export { DurabilitySemaphoreQueue, } from './queue.js';
|
|
16
16
|
/**
|
|
17
17
|
* Inside a DO, holds one concurrency limit for many client Durable Objects.
|
|
18
18
|
* Outside the DO, `client()` routes semaphore messages to the application-owned
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { LeaseGrant, LeaseReport, SemaphoreBlockedReason } from './
|
|
2
|
-
import type { HolderStore, LeaseRow } from './
|
|
1
|
+
import type { LeaseGrant, LeaseReport, SemaphoreBlockedReason } from './protocol.js';
|
|
2
|
+
import type { HolderStore, LeaseRow } from './store.js';
|
|
3
3
|
/**
|
|
4
4
|
* Where lease capacity comes from.
|
|
5
5
|
*
|
|
@@ -47,9 +47,21 @@ export type LeaseEvent = {
|
|
|
47
47
|
timestamp: number;
|
|
48
48
|
leaseId: string;
|
|
49
49
|
clientId: string;
|
|
50
|
+
clientChannel?: string;
|
|
50
51
|
group: string;
|
|
51
52
|
permits: number;
|
|
52
53
|
};
|
|
54
|
+
/** A client is its object's `clientId` together with the queue channel on that object. */
|
|
55
|
+
export type SemaphoreClientIdentity = {
|
|
56
|
+
clientId: string;
|
|
57
|
+
clientChannel: string;
|
|
58
|
+
};
|
|
59
|
+
/** Map key for a client identity; JSON keeps raw IDs and channels unambiguous. */
|
|
60
|
+
export declare const clientKey: (clientId: string, clientChannel: string) => string;
|
|
61
|
+
export declare const ownedBy: (row: {
|
|
62
|
+
client_id: string;
|
|
63
|
+
client_channel: string;
|
|
64
|
+
}, client: SemaphoreClientIdentity) => boolean;
|
|
53
65
|
type GroupPolicy = {
|
|
54
66
|
limit?: number;
|
|
55
67
|
blocked?: SemaphoreBlockedReason;
|
|
@@ -77,14 +89,13 @@ export type LeaseReportOutcome = {
|
|
|
77
89
|
* and charges their reported live footprint so a forgotten lease cannot
|
|
78
90
|
* over-admit for the rest of this round.
|
|
79
91
|
*/
|
|
80
|
-
export declare const applyLeaseReports: (store: HolderStore, policy: LeasePolicy,
|
|
81
|
-
export type LeaseCandidate = {
|
|
82
|
-
clientId: string;
|
|
92
|
+
export declare const applyLeaseReports: (store: HolderStore, policy: LeasePolicy, client: SemaphoreClientIdentity, reports: LeaseReport[], now: number) => LeaseReportOutcome;
|
|
93
|
+
export type LeaseCandidate = SemaphoreClientIdentity & {
|
|
83
94
|
group: string;
|
|
84
95
|
oldestAt: number;
|
|
85
96
|
};
|
|
86
97
|
export type LeaseAllocation = {
|
|
87
|
-
/** New and retired leases per client. */
|
|
98
|
+
/** New and retired leases per client, keyed by {@link clientKey}. */
|
|
88
99
|
grants: Map<string, LeaseGrant[]>;
|
|
89
100
|
footprint: number;
|
|
90
101
|
events: LeaseEvent[];
|