durability 2.1.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +303 -116
- package/dist/durability-alarms.d.ts +90 -0
- package/dist/durability-alarms.js +165 -0
- package/dist/durability.d.ts +158 -0
- package/dist/durability.js +260 -0
- package/dist/errors.d.ts +83 -0
- package/dist/errors.js +184 -0
- package/dist/fanout.d.ts +61 -0
- package/dist/fanout.js +528 -0
- package/dist/index.d.ts +10 -435
- package/dist/index.js +7 -1420
- package/dist/kv-storage.d.ts +2 -0
- package/dist/kv-storage.js +421 -0
- package/dist/lifecycle.d.ts +70 -0
- package/dist/lifecycle.js +36 -0
- package/dist/load.d.ts +40 -0
- package/dist/load.js +68 -0
- package/dist/migrations.d.ts +50 -0
- package/dist/migrations.js +379 -0
- package/dist/policy.d.ts +74 -0
- package/dist/policy.js +49 -0
- package/dist/routing-client.d.ts +28 -0
- package/dist/routing-client.js +109 -0
- package/dist/routing.d.ts +33 -0
- package/dist/routing.js +58 -0
- package/dist/scheduler.d.ts +157 -0
- package/dist/scheduler.js +478 -0
- package/dist/serialization.d.ts +2 -0
- package/dist/serialization.js +13 -0
- package/dist/sqlite-storage.d.ts +2 -0
- package/dist/sqlite-storage.js +251 -0
- package/dist/storage.d.ts +134 -0
- package/dist/storage.js +73 -0
- package/dist/stored-message.d.ts +29 -0
- package/dist/stored-message.js +1 -0
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -1,1420 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
.optional(),
|
|
9
|
-
maxAttempts: z.number().optional(),
|
|
10
|
-
});
|
|
11
|
-
const durabilityMethodOptionsSchema = z.object({
|
|
12
|
-
attemptTimeoutMs: z.number().optional(),
|
|
13
|
-
retries: retryOptionsSchema.optional(),
|
|
14
|
-
});
|
|
15
|
-
const durabilityAlarmMethodOptionsSchema = durabilityMethodOptionsSchema.extend({
|
|
16
|
-
retryTimeouts: z.boolean().optional(),
|
|
17
|
-
});
|
|
18
|
-
const callRowSchema = z.object({
|
|
19
|
-
id: z.string(),
|
|
20
|
-
operation: z.string(),
|
|
21
|
-
payload: z.string(),
|
|
22
|
-
status: z.enum(['pending', 'completed', 'failed']),
|
|
23
|
-
result: z.string().nullable(),
|
|
24
|
-
attempt: z.number().int().nonnegative(),
|
|
25
|
-
next_attempt_at: z.number().int().nonnegative(),
|
|
26
|
-
last_error: z.string().nullable(),
|
|
27
|
-
last_error_name: z.string().nullable(),
|
|
28
|
-
completed_at: z.number().int().nonnegative().nullable(),
|
|
29
|
-
created_at: z.number().int().nonnegative(),
|
|
30
|
-
generation_id: z.string(),
|
|
31
|
-
});
|
|
32
|
-
const callAttemptSchema = callRowSchema.pick({ attempt: true });
|
|
33
|
-
const callIdSchema = callRowSchema.pick({ id: true });
|
|
34
|
-
const callNextAttemptSchema = callRowSchema.pick({ next_attempt_at: true });
|
|
35
|
-
const alarmRowSchema = z.object({
|
|
36
|
-
name: z.string(),
|
|
37
|
-
generation_id: z.string(),
|
|
38
|
-
status: z.enum(['pending', 'failed']),
|
|
39
|
-
scheduled_at: z.number().int().nonnegative(),
|
|
40
|
-
next_attempt_at: z.number().int().nonnegative(),
|
|
41
|
-
attempt: z.number().int().nonnegative(),
|
|
42
|
-
last_error: z.string().nullable(),
|
|
43
|
-
last_error_name: z.string().nullable(),
|
|
44
|
-
created_at: z.number().int().nonnegative(),
|
|
45
|
-
});
|
|
46
|
-
const alarmAttemptSchema = alarmRowSchema.pick({ attempt: true });
|
|
47
|
-
const countSchema = z.object({ count: z.number().int().nonnegative() });
|
|
48
|
-
const tableName = 'durability_calls';
|
|
49
|
-
const alarmTableName = 'durability_alarms';
|
|
50
|
-
const migrationTableName = 'durability_migrations';
|
|
51
|
-
const storedValueSchema = z.discriminatedUnion('kind', [
|
|
52
|
-
z.object({ kind: z.literal('value'), value: z.json() }),
|
|
53
|
-
z.object({ kind: z.literal('undefined') }),
|
|
54
|
-
]);
|
|
55
|
-
/**
|
|
56
|
-
* Ordered SQLite migrations for the tables owned by durability.
|
|
57
|
-
*
|
|
58
|
-
* Every SQLite-backed Durable Object instance has an independent database, so
|
|
59
|
-
* the durability schema must exist in each instance before calls can be stored.
|
|
60
|
-
* {@link createDurability} applies these migrations automatically on instance
|
|
61
|
-
* initialization. They are tracked in the namespaced `durability_migrations`
|
|
62
|
-
* table so they can coexist with application-owned migrations in the same
|
|
63
|
-
* database.
|
|
64
|
-
*
|
|
65
|
-
* The list is exported so migration names can be inspected before an explicit
|
|
66
|
-
* rollback with {@link migrateDurability}. Most consumers should not execute
|
|
67
|
-
* the SQL directly.
|
|
68
|
-
*
|
|
69
|
-
* @example
|
|
70
|
-
* ```ts
|
|
71
|
-
* const currentTarget = durabilityMigrations.at(-1)?.name ?? null;
|
|
72
|
-
* migrateDurability(this.ctx, currentTarget);
|
|
73
|
-
* ```
|
|
74
|
-
*/
|
|
75
|
-
export const durabilityMigrations = [
|
|
76
|
-
{
|
|
77
|
-
name: 'durability_0001_create_calls',
|
|
78
|
-
up: `
|
|
79
|
-
CREATE TABLE IF NOT EXISTS durability_calls (
|
|
80
|
-
id TEXT PRIMARY KEY,
|
|
81
|
-
operation TEXT NOT NULL,
|
|
82
|
-
payload TEXT NOT NULL,
|
|
83
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
|
|
84
|
-
result TEXT,
|
|
85
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
86
|
-
next_attempt_at INTEGER NOT NULL,
|
|
87
|
-
last_error TEXT,
|
|
88
|
-
last_error_name TEXT,
|
|
89
|
-
completed_at INTEGER
|
|
90
|
-
);
|
|
91
|
-
`,
|
|
92
|
-
down: 'DROP TABLE IF EXISTS durability_calls;',
|
|
93
|
-
},
|
|
94
|
-
{
|
|
95
|
-
name: 'durability_0002_pending_index',
|
|
96
|
-
up: `
|
|
97
|
-
CREATE INDEX IF NOT EXISTS durability_calls_pending_idx
|
|
98
|
-
ON durability_calls (status, next_attempt_at);
|
|
99
|
-
`,
|
|
100
|
-
down: 'DROP INDEX IF EXISTS durability_calls_pending_idx;',
|
|
101
|
-
},
|
|
102
|
-
{
|
|
103
|
-
name: 'durability_0003_create_alarms',
|
|
104
|
-
up: `
|
|
105
|
-
CREATE TABLE IF NOT EXISTS durability_alarms (
|
|
106
|
-
name TEXT PRIMARY KEY,
|
|
107
|
-
generation_id TEXT NOT NULL,
|
|
108
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
109
|
-
scheduled_at INTEGER NOT NULL,
|
|
110
|
-
next_attempt_at INTEGER NOT NULL,
|
|
111
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
112
|
-
last_error TEXT,
|
|
113
|
-
last_error_name TEXT
|
|
114
|
-
);
|
|
115
|
-
CREATE INDEX IF NOT EXISTS durability_alarms_pending_idx
|
|
116
|
-
ON durability_alarms (status, next_attempt_at);
|
|
117
|
-
`,
|
|
118
|
-
down: `
|
|
119
|
-
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
120
|
-
DROP TABLE IF EXISTS durability_alarms;
|
|
121
|
-
`,
|
|
122
|
-
},
|
|
123
|
-
{
|
|
124
|
-
name: 'durability_0004_generation_and_created_at',
|
|
125
|
-
up: `
|
|
126
|
-
DROP INDEX IF EXISTS durability_calls_pending_idx;
|
|
127
|
-
ALTER TABLE durability_calls RENAME TO durability_calls_v3;
|
|
128
|
-
CREATE TABLE durability_calls (
|
|
129
|
-
id TEXT PRIMARY KEY,
|
|
130
|
-
operation TEXT NOT NULL,
|
|
131
|
-
payload TEXT NOT NULL,
|
|
132
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
|
|
133
|
-
result TEXT,
|
|
134
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
135
|
-
next_attempt_at INTEGER NOT NULL,
|
|
136
|
-
last_error TEXT,
|
|
137
|
-
last_error_name TEXT,
|
|
138
|
-
completed_at INTEGER,
|
|
139
|
-
generation_id TEXT NOT NULL DEFAULT 'legacy',
|
|
140
|
-
created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))
|
|
141
|
-
);
|
|
142
|
-
INSERT INTO durability_calls
|
|
143
|
-
SELECT id, operation, payload, status, result, attempt, next_attempt_at,
|
|
144
|
-
last_error, last_error_name, completed_at, 'legacy:' || id,
|
|
145
|
-
CAST(unixepoch('subsec') * 1000 AS INTEGER)
|
|
146
|
-
FROM durability_calls_v3;
|
|
147
|
-
DROP TABLE durability_calls_v3;
|
|
148
|
-
CREATE INDEX durability_calls_pending_idx
|
|
149
|
-
ON durability_calls (status, next_attempt_at);
|
|
150
|
-
CREATE INDEX durability_calls_created_idx
|
|
151
|
-
ON durability_calls (created_at);
|
|
152
|
-
|
|
153
|
-
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
154
|
-
ALTER TABLE durability_alarms RENAME TO durability_alarms_v3;
|
|
155
|
-
CREATE TABLE durability_alarms (
|
|
156
|
-
name TEXT PRIMARY KEY,
|
|
157
|
-
generation_id TEXT NOT NULL,
|
|
158
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
159
|
-
scheduled_at INTEGER NOT NULL,
|
|
160
|
-
next_attempt_at INTEGER NOT NULL,
|
|
161
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
162
|
-
last_error TEXT,
|
|
163
|
-
last_error_name TEXT,
|
|
164
|
-
created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))
|
|
165
|
-
);
|
|
166
|
-
INSERT INTO durability_alarms
|
|
167
|
-
SELECT name, generation_id, status, scheduled_at, next_attempt_at,
|
|
168
|
-
attempt, last_error, last_error_name,
|
|
169
|
-
CAST(unixepoch('subsec') * 1000 AS INTEGER)
|
|
170
|
-
FROM durability_alarms_v3;
|
|
171
|
-
DROP TABLE durability_alarms_v3;
|
|
172
|
-
CREATE INDEX durability_alarms_pending_idx
|
|
173
|
-
ON durability_alarms (status, next_attempt_at);
|
|
174
|
-
CREATE INDEX durability_alarms_created_idx
|
|
175
|
-
ON durability_alarms (created_at);
|
|
176
|
-
`,
|
|
177
|
-
down: `
|
|
178
|
-
DROP INDEX IF EXISTS durability_calls_pending_idx;
|
|
179
|
-
DROP INDEX IF EXISTS durability_calls_created_idx;
|
|
180
|
-
ALTER TABLE durability_calls RENAME TO durability_calls_v4;
|
|
181
|
-
CREATE TABLE durability_calls (
|
|
182
|
-
id TEXT PRIMARY KEY,
|
|
183
|
-
operation TEXT NOT NULL,
|
|
184
|
-
payload TEXT NOT NULL,
|
|
185
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
|
|
186
|
-
result TEXT,
|
|
187
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
188
|
-
next_attempt_at INTEGER NOT NULL,
|
|
189
|
-
last_error TEXT,
|
|
190
|
-
last_error_name TEXT,
|
|
191
|
-
completed_at INTEGER
|
|
192
|
-
);
|
|
193
|
-
INSERT INTO durability_calls
|
|
194
|
-
SELECT id, operation, payload, status, result, attempt, next_attempt_at,
|
|
195
|
-
last_error, last_error_name, completed_at
|
|
196
|
-
FROM durability_calls_v4;
|
|
197
|
-
DROP TABLE durability_calls_v4;
|
|
198
|
-
CREATE INDEX durability_calls_pending_idx
|
|
199
|
-
ON durability_calls (status, next_attempt_at);
|
|
200
|
-
|
|
201
|
-
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
202
|
-
DROP INDEX IF EXISTS durability_alarms_created_idx;
|
|
203
|
-
ALTER TABLE durability_alarms RENAME TO durability_alarms_v4;
|
|
204
|
-
CREATE TABLE durability_alarms (
|
|
205
|
-
name TEXT PRIMARY KEY,
|
|
206
|
-
generation_id TEXT NOT NULL,
|
|
207
|
-
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
208
|
-
scheduled_at INTEGER NOT NULL,
|
|
209
|
-
next_attempt_at INTEGER NOT NULL,
|
|
210
|
-
attempt INTEGER NOT NULL DEFAULT 0,
|
|
211
|
-
last_error TEXT,
|
|
212
|
-
last_error_name TEXT
|
|
213
|
-
);
|
|
214
|
-
INSERT INTO durability_alarms
|
|
215
|
-
SELECT name, generation_id, status, scheduled_at, next_attempt_at,
|
|
216
|
-
attempt, last_error, last_error_name
|
|
217
|
-
FROM durability_alarms_v4;
|
|
218
|
-
DROP TABLE durability_alarms_v4;
|
|
219
|
-
CREATE INDEX durability_alarms_pending_idx
|
|
220
|
-
ON durability_alarms (status, next_attempt_at);
|
|
221
|
-
`,
|
|
222
|
-
},
|
|
223
|
-
];
|
|
224
|
-
/**
|
|
225
|
-
* Moves the durability schema in one Durable Object instance to a target.
|
|
226
|
-
*
|
|
227
|
-
* {@link createDurability} calls this with the latest target automatically. Call
|
|
228
|
-
* it directly only for explicit schema management, especially rollbacks. Passing
|
|
229
|
-
* `null` removes every durability table and permanently deletes stored calls.
|
|
230
|
-
*
|
|
231
|
-
* @example Apply all pending durability migrations.
|
|
232
|
-
* ```ts
|
|
233
|
-
* migrateDurability(this.ctx);
|
|
234
|
-
* ```
|
|
235
|
-
*
|
|
236
|
-
* @example Roll back to a specific version, then remove the schema entirely.
|
|
237
|
-
* ```ts
|
|
238
|
-
* migrateDurability(this.ctx, 'durability_0001_create_calls');
|
|
239
|
-
* migrateDurability(this.ctx, null);
|
|
240
|
-
* ```
|
|
241
|
-
*
|
|
242
|
-
* @param context Durable Object context containing SQLite-backed storage.
|
|
243
|
-
* @param target Migration name to migrate to, or null to roll back all migrations.
|
|
244
|
-
*/
|
|
245
|
-
export const migrateDurability = (context, target = durabilityMigrations[durabilityMigrations.length - 1]
|
|
246
|
-
?.name ?? null) => {
|
|
247
|
-
const targetIndex = target === null
|
|
248
|
-
? -1
|
|
249
|
-
: durabilityMigrations.findIndex((migration) => migration.name === target);
|
|
250
|
-
if (target !== null && targetIndex === -1) {
|
|
251
|
-
throw new Error(`Unknown durability migration target "${target}"`);
|
|
252
|
-
}
|
|
253
|
-
const qb = new DOQB(context.storage.sql);
|
|
254
|
-
const workersQbMigrations = durabilityMigrations.map(({ name, up }) => ({ name, sql: up }));
|
|
255
|
-
const migrationBuilder = qb.migrations({
|
|
256
|
-
migrations: workersQbMigrations,
|
|
257
|
-
tableName: 'durability_migrations',
|
|
258
|
-
});
|
|
259
|
-
const appliedNames = new Set(migrationBuilder.getApplied().map((migration) => migration.name));
|
|
260
|
-
const rolledBack = [];
|
|
261
|
-
for (let index = durabilityMigrations.length - 1; index > targetIndex; index--) {
|
|
262
|
-
const migration = durabilityMigrations[index];
|
|
263
|
-
if (!migration || !appliedNames.has(migration.name)) {
|
|
264
|
-
continue;
|
|
265
|
-
}
|
|
266
|
-
qb.raw({
|
|
267
|
-
query: `${migration.down}\nDELETE FROM ${migrationTableName} WHERE name = '${migration.name}';`,
|
|
268
|
-
}).execute();
|
|
269
|
-
rolledBack.push(migration.name);
|
|
270
|
-
}
|
|
271
|
-
const applied = qb
|
|
272
|
-
.migrations({
|
|
273
|
-
migrations: workersQbMigrations.slice(0, targetIndex + 1),
|
|
274
|
-
tableName: 'durability_migrations',
|
|
275
|
-
})
|
|
276
|
-
.apply()
|
|
277
|
-
.map((migration) => migration.name);
|
|
278
|
-
if (target === null) {
|
|
279
|
-
context.storage.sql.exec(`DROP TABLE IF EXISTS ${migrationTableName}`);
|
|
280
|
-
}
|
|
281
|
-
return { applied, rolledBack };
|
|
282
|
-
};
|
|
283
|
-
/**
|
|
284
|
-
* Marks a handler failure as terminal so it is persisted without another retry.
|
|
285
|
-
*
|
|
286
|
-
* Use this for permanent failures such as invalid input. Transient errors should
|
|
287
|
-
* be thrown normally so the configured retry policy can handle them.
|
|
288
|
-
*
|
|
289
|
-
* @example
|
|
290
|
-
* ```ts
|
|
291
|
-
* if (!recipient.isValid) {
|
|
292
|
-
* throw new NonRetryableError('Recipient is invalid');
|
|
293
|
-
* }
|
|
294
|
-
* ```
|
|
295
|
-
*/
|
|
296
|
-
export class NonRetryableError extends Error {
|
|
297
|
-
constructor(message) {
|
|
298
|
-
super(message);
|
|
299
|
-
this.name = 'NonRetryableError';
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
/**
|
|
303
|
-
* Error recorded when a handler attempt exceeds its configured timeout.
|
|
304
|
-
*
|
|
305
|
-
* Durability creates this error and aborts the attempt's signal; consumers do
|
|
306
|
-
* not need to throw it themselves. Timeout failures follow the normal retry
|
|
307
|
-
* policy and are exposed by `getResult` if they become terminal.
|
|
308
|
-
*
|
|
309
|
-
* @example
|
|
310
|
-
* ```ts
|
|
311
|
-
* const state = await durability.resizeImage.getResult('resize:image-1');
|
|
312
|
-
* if (
|
|
313
|
-
* state.status === 'failed' &&
|
|
314
|
-
* state.error.name === 'DurableAttemptTimeoutError'
|
|
315
|
-
* ) {
|
|
316
|
-
* console.error('Resize timed out');
|
|
317
|
-
* }
|
|
318
|
-
* ```
|
|
319
|
-
*/
|
|
320
|
-
export class DurableAttemptTimeoutError extends Error {
|
|
321
|
-
constructor(operation, timeoutMs) {
|
|
322
|
-
super(`Durable operation "${operation}" timed out after ${timeoutMs}ms`);
|
|
323
|
-
this.name = 'DurableAttemptTimeoutError';
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
/** Error recorded when a named alarm attempt exceeds its configured timeout. */
|
|
327
|
-
export class DurableAlarmTimeoutError extends Error {
|
|
328
|
-
constructor(name, timeoutMs) {
|
|
329
|
-
super(`Durable alarm "${name}" timed out after ${timeoutMs}ms`);
|
|
330
|
-
this.name = 'DurableAlarmTimeoutError';
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
/** Error persisted when retry-delay evaluation fails or is unsafe. */
|
|
334
|
-
export class DurableRetryPolicyError extends Error {
|
|
335
|
-
constructor(entity, reason) {
|
|
336
|
-
super(`Retry policy for ${entity} must produce a non-negative safe-integer timestamp`);
|
|
337
|
-
this.name = 'DurableRetryPolicyError';
|
|
338
|
-
if (reason !== undefined) {
|
|
339
|
-
Object.defineProperty(this, 'cause', { value: reason });
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
/** Error persisted when a successful result cannot be JSON-serialized. */
|
|
344
|
-
export class DurableResultSerializationError extends Error {
|
|
345
|
-
constructor(operation, reason) {
|
|
346
|
-
super(`Result for durable operation "${operation}" is not JSON-serializable`);
|
|
347
|
-
this.name = 'DurableResultSerializationError';
|
|
348
|
-
if (reason !== undefined) {
|
|
349
|
-
Object.defineProperty(this, 'cause', { value: reason });
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
/** Error persisted when pending work has already reached its attempt limit. */
|
|
354
|
-
export class DurableAttemptsExhaustedError extends Error {
|
|
355
|
-
constructor(entity, name, max) {
|
|
356
|
-
super(`Durable ${entity} "${name}" exhausted its ${max} attempts`);
|
|
357
|
-
this.name = 'DurableAttemptsExhaustedError';
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
/**
|
|
361
|
-
* Error thrown when an idempotency key is reused for another operation.
|
|
362
|
-
*
|
|
363
|
-
* A call ID permanently identifies its original operation so `getResult` cannot
|
|
364
|
-
* return a value with the wrong inferred type.
|
|
365
|
-
*
|
|
366
|
-
* @example
|
|
367
|
-
* ```ts
|
|
368
|
-
* await durability.resizeImage({ id: 'job:1', payload: resizeInput });
|
|
369
|
-
* await durability.sendEmail({ id: 'job:1', payload: emailInput });
|
|
370
|
-
* // Throws DuplicateDurableCallError because job:1 belongs to resizeImage.
|
|
371
|
-
* ```
|
|
372
|
-
*/
|
|
373
|
-
export class DuplicateDurableCallError extends Error {
|
|
374
|
-
constructor(id, existingOperation, requestedOperation) {
|
|
375
|
-
super(`Durable call "${id}" already belongs to operation "${existingOperation}", not "${requestedOperation}"`);
|
|
376
|
-
this.name = 'DuplicateDurableCallError';
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
export const createDurability = (context, handlers, options = {}) => {
|
|
380
|
-
migrateDurability(context);
|
|
381
|
-
const qb = new DOQB(context.storage.sql);
|
|
382
|
-
const active = new Map();
|
|
383
|
-
const activeAlarms = new Map();
|
|
384
|
-
const alarmHandlers = new Map(Object.entries(options.alarms ?? {}).map(([name, handler]) => [
|
|
385
|
-
name,
|
|
386
|
-
alarmHandlerSchema.parse(handler),
|
|
387
|
-
]));
|
|
388
|
-
const alarmConcurrency = options.alarmConcurrency ?? 10;
|
|
389
|
-
const alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
|
|
390
|
-
const defaultAttemptTimeoutMs = options.attemptTimeoutMs ?? 5 * 60000;
|
|
391
|
-
const retryOptions = retryOptionsSchema.parse(options.retries ?? {});
|
|
392
|
-
const defaultRetryDelay = retryOptions.delay ?? ((attempt) => jitter(exponential(attempt)));
|
|
393
|
-
const defaultMaxAttempts = retryOptions.maxAttempts ?? 5;
|
|
394
|
-
if (!Number.isInteger(alarmConcurrency) || alarmConcurrency < 1) {
|
|
395
|
-
throw new RangeError('alarmConcurrency must be a positive integer');
|
|
396
|
-
}
|
|
397
|
-
if (!Number.isInteger(defaultAttemptTimeoutMs) ||
|
|
398
|
-
defaultAttemptTimeoutMs < 1) {
|
|
399
|
-
throw new RangeError('attemptTimeoutMs must be a positive integer');
|
|
400
|
-
}
|
|
401
|
-
if (!Number.isInteger(defaultMaxAttempts) || defaultMaxAttempts < 1) {
|
|
402
|
-
throw new RangeError('retries.maxAttempts must be a positive integer');
|
|
403
|
-
}
|
|
404
|
-
const methodOptions = new Map(Object.entries(options.methods ?? {}).map(([operation, method]) => [
|
|
405
|
-
operation,
|
|
406
|
-
durabilityMethodOptionsSchema.parse(method),
|
|
407
|
-
]));
|
|
408
|
-
for (const [operation, method] of methodOptions) {
|
|
409
|
-
if (method?.attemptTimeoutMs !== undefined &&
|
|
410
|
-
(!Number.isInteger(method.attemptTimeoutMs) ||
|
|
411
|
-
method.attemptTimeoutMs < 1)) {
|
|
412
|
-
throw new RangeError(`methods.${operation}.attemptTimeoutMs must be a positive integer`);
|
|
413
|
-
}
|
|
414
|
-
if (method?.retries?.maxAttempts !== undefined &&
|
|
415
|
-
(!Number.isInteger(method.retries.maxAttempts) ||
|
|
416
|
-
method.retries.maxAttempts < 1)) {
|
|
417
|
-
throw new RangeError(`methods.${operation}.retries.maxAttempts must be a positive integer`);
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
const alarmMethodOptions = new Map(Object.entries(options.alarmMethods ?? {}).map(([name, method]) => [
|
|
421
|
-
name,
|
|
422
|
-
durabilityAlarmMethodOptionsSchema.parse(method),
|
|
423
|
-
]));
|
|
424
|
-
for (const [name, method] of alarmMethodOptions) {
|
|
425
|
-
if (method?.attemptTimeoutMs !== undefined &&
|
|
426
|
-
(!Number.isInteger(method.attemptTimeoutMs) ||
|
|
427
|
-
method.attemptTimeoutMs < 1)) {
|
|
428
|
-
throw new RangeError(`alarmMethods.${name}.attemptTimeoutMs must be a positive integer`);
|
|
429
|
-
}
|
|
430
|
-
if (method?.retries?.maxAttempts !== undefined &&
|
|
431
|
-
(!Number.isInteger(method.retries.maxAttempts) ||
|
|
432
|
-
method.retries.maxAttempts < 1)) {
|
|
433
|
-
throw new RangeError(`alarmMethods.${name}.retries.maxAttempts must be a positive integer`);
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
const executionPolicy = (operation) => {
|
|
437
|
-
const method = methodOptions.get(operation);
|
|
438
|
-
return {
|
|
439
|
-
attemptTimeoutMs: method?.attemptTimeoutMs ?? defaultAttemptTimeoutMs,
|
|
440
|
-
delay: method?.retries?.delay ?? defaultRetryDelay,
|
|
441
|
-
maxAttempts: method?.retries?.maxAttempts ?? defaultMaxAttempts,
|
|
442
|
-
};
|
|
443
|
-
};
|
|
444
|
-
const alarmExecutionPolicy = (name) => {
|
|
445
|
-
const method = alarmMethodOptions.get(name);
|
|
446
|
-
return {
|
|
447
|
-
attemptTimeoutMs: method?.attemptTimeoutMs ?? defaultAttemptTimeoutMs,
|
|
448
|
-
delay: method?.retries?.delay ?? defaultRetryDelay,
|
|
449
|
-
maxAttempts: method?.retries?.maxAttempts ?? defaultMaxAttempts,
|
|
450
|
-
retryTimeouts: method?.retryTimeouts ?? false,
|
|
451
|
-
};
|
|
452
|
-
};
|
|
453
|
-
const serializeError = (error) => {
|
|
454
|
-
let name = 'Error';
|
|
455
|
-
let message = 'Unknown thrown value';
|
|
456
|
-
try {
|
|
457
|
-
if (error === null) {
|
|
458
|
-
return { name, message: 'null' };
|
|
459
|
-
}
|
|
460
|
-
if (typeof error !== 'object' && typeof error !== 'function') {
|
|
461
|
-
return { name, message: String(error) };
|
|
462
|
-
}
|
|
463
|
-
try {
|
|
464
|
-
const candidate = Reflect.get(error, 'name');
|
|
465
|
-
if (typeof candidate === 'string' && candidate.length > 0) {
|
|
466
|
-
name = candidate;
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
catch {
|
|
470
|
-
name = 'Error';
|
|
471
|
-
}
|
|
472
|
-
try {
|
|
473
|
-
const candidate = Reflect.get(error, 'message');
|
|
474
|
-
if (typeof candidate === 'string' && candidate.length > 0) {
|
|
475
|
-
message = candidate;
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
catch {
|
|
479
|
-
message = 'Unknown thrown value';
|
|
480
|
-
}
|
|
481
|
-
return { name, message };
|
|
482
|
-
}
|
|
483
|
-
catch {
|
|
484
|
-
return { name: 'Error', message: 'Unknown thrown value' };
|
|
485
|
-
}
|
|
486
|
-
};
|
|
487
|
-
const isErrorInstance = (error, constructor) => {
|
|
488
|
-
try {
|
|
489
|
-
return error instanceof constructor;
|
|
490
|
-
}
|
|
491
|
-
catch {
|
|
492
|
-
return false;
|
|
493
|
-
}
|
|
494
|
-
};
|
|
495
|
-
const isNonRetryable = (error) => {
|
|
496
|
-
if (isErrorInstance(error, NonRetryableError)) {
|
|
497
|
-
return true;
|
|
498
|
-
}
|
|
499
|
-
if ((typeof error !== 'object' && typeof error !== 'function') ||
|
|
500
|
-
error === null) {
|
|
501
|
-
return false;
|
|
502
|
-
}
|
|
503
|
-
try {
|
|
504
|
-
if (Reflect.get(error, 'name') === 'NonRetryableError') {
|
|
505
|
-
return true;
|
|
506
|
-
}
|
|
507
|
-
const constructor = Reflect.get(error, 'constructor');
|
|
508
|
-
return (constructor !== null &&
|
|
509
|
-
(typeof constructor === 'object' ||
|
|
510
|
-
typeof constructor === 'function') &&
|
|
511
|
-
Reflect.get(constructor, 'name') === 'NonRetryableError');
|
|
512
|
-
}
|
|
513
|
-
catch {
|
|
514
|
-
return false;
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
const reportFailure = (event, details, error) => {
|
|
518
|
-
try {
|
|
519
|
-
const normalized = serializeError(error);
|
|
520
|
-
let stack;
|
|
521
|
-
try {
|
|
522
|
-
const candidate = error !== null &&
|
|
523
|
-
(typeof error === 'object' || typeof error === 'function')
|
|
524
|
-
? Reflect.get(error, 'stack')
|
|
525
|
-
: undefined;
|
|
526
|
-
stack = typeof candidate === 'string' ? candidate : undefined;
|
|
527
|
-
}
|
|
528
|
-
catch {
|
|
529
|
-
stack = undefined;
|
|
530
|
-
}
|
|
531
|
-
console.error({
|
|
532
|
-
event,
|
|
533
|
-
...details,
|
|
534
|
-
error: { ...normalized, ...(stack ? { stack } : {}) },
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
catch {
|
|
538
|
-
return;
|
|
539
|
-
}
|
|
540
|
-
};
|
|
541
|
-
const emit = (event) => {
|
|
542
|
-
if (!options.onLifecycleEvent) {
|
|
543
|
-
return;
|
|
544
|
-
}
|
|
545
|
-
const report = (error) => reportFailure('durability.lifecycle_hook.failed', {
|
|
546
|
-
entityKind: event.entityKind,
|
|
547
|
-
lifecycleType: event.type,
|
|
548
|
-
entityId: 'id' in event ? event.id : 'durability',
|
|
549
|
-
}, error);
|
|
550
|
-
try {
|
|
551
|
-
const result = options.onLifecycleEvent(event);
|
|
552
|
-
if (result === undefined) {
|
|
553
|
-
return;
|
|
554
|
-
}
|
|
555
|
-
const caught = Promise.resolve(result).catch(report);
|
|
556
|
-
if (context.waitUntil) {
|
|
557
|
-
try {
|
|
558
|
-
context.waitUntil(caught);
|
|
559
|
-
return;
|
|
560
|
-
}
|
|
561
|
-
catch (error) {
|
|
562
|
-
report(error);
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
void caught;
|
|
566
|
-
}
|
|
567
|
-
catch (error) {
|
|
568
|
-
report(error);
|
|
569
|
-
}
|
|
570
|
-
};
|
|
571
|
-
const serialize = (value) => JSON.stringify(value === undefined
|
|
572
|
-
? { kind: 'undefined' }
|
|
573
|
-
: { kind: 'value', value: z.json().parse(value) });
|
|
574
|
-
const deserialize = (value) => {
|
|
575
|
-
const parsed = JSON.parse(value);
|
|
576
|
-
const stored = storedValueSchema.parse(parsed);
|
|
577
|
-
return stored.kind === 'undefined' ? undefined : stored.value;
|
|
578
|
-
};
|
|
579
|
-
const getCall = (id) => callRowSchema.optional().parse(qb
|
|
580
|
-
.fetchOne({
|
|
581
|
-
tableName,
|
|
582
|
-
where: { conditions: 'id = ?', params: [id] },
|
|
583
|
-
})
|
|
584
|
-
.execute().results);
|
|
585
|
-
const getResult = (operation, idempotencyKey) => {
|
|
586
|
-
const call = getCall(idempotencyKey);
|
|
587
|
-
if (!call) {
|
|
588
|
-
return { status: 'not_found' };
|
|
589
|
-
}
|
|
590
|
-
assertOperation(idempotencyKey, call.operation, operation);
|
|
591
|
-
if (call.status === 'pending') {
|
|
592
|
-
return {
|
|
593
|
-
status: 'pending',
|
|
594
|
-
attempt: call.attempt,
|
|
595
|
-
nextAttemptAt: call.next_attempt_at,
|
|
596
|
-
lastError: call.last_error,
|
|
597
|
-
};
|
|
598
|
-
}
|
|
599
|
-
if (call.status === 'failed') {
|
|
600
|
-
return {
|
|
601
|
-
status: 'failed',
|
|
602
|
-
attempt: call.attempt,
|
|
603
|
-
error: {
|
|
604
|
-
name: call.last_error_name ?? 'Error',
|
|
605
|
-
message: call.last_error ?? 'Durable operation failed',
|
|
606
|
-
},
|
|
607
|
-
};
|
|
608
|
-
}
|
|
609
|
-
if (call.result === null) {
|
|
610
|
-
throw new Error(`Completed durable call "${idempotencyKey}" has no result`);
|
|
611
|
-
}
|
|
612
|
-
return { status: 'completed', result: deserialize(call.result) };
|
|
613
|
-
};
|
|
614
|
-
const listDueIds = (now, limit) => callIdSchema
|
|
615
|
-
.array()
|
|
616
|
-
.parse(qb
|
|
617
|
-
.fetchAll({
|
|
618
|
-
tableName,
|
|
619
|
-
fields: 'id',
|
|
620
|
-
where: {
|
|
621
|
-
conditions: "status = 'pending' AND next_attempt_at <= ?",
|
|
622
|
-
params: [now],
|
|
623
|
-
},
|
|
624
|
-
orderBy: 'next_attempt_at ASC',
|
|
625
|
-
limit,
|
|
626
|
-
})
|
|
627
|
-
.execute().results ?? [])
|
|
628
|
-
.map((call) => call.id);
|
|
629
|
-
const getNextPendingAt = () => callNextAttemptSchema.optional().parse(qb
|
|
630
|
-
.fetchOne({
|
|
631
|
-
tableName,
|
|
632
|
-
fields: 'next_attempt_at',
|
|
633
|
-
where: { conditions: "status = 'pending'" },
|
|
634
|
-
orderBy: 'next_attempt_at ASC',
|
|
635
|
-
})
|
|
636
|
-
.execute().results)?.next_attempt_at;
|
|
637
|
-
const listDueAlarms = (now, limit) => alarmRowSchema.array().parse(qb
|
|
638
|
-
.fetchAll({
|
|
639
|
-
tableName: alarmTableName,
|
|
640
|
-
where: {
|
|
641
|
-
conditions: "status = 'pending' AND next_attempt_at <= ?",
|
|
642
|
-
params: [now],
|
|
643
|
-
},
|
|
644
|
-
orderBy: 'next_attempt_at ASC',
|
|
645
|
-
limit,
|
|
646
|
-
})
|
|
647
|
-
.execute().results ?? []);
|
|
648
|
-
const getNextNamedAlarmAt = () => alarmRowSchema
|
|
649
|
-
.pick({ next_attempt_at: true })
|
|
650
|
-
.optional()
|
|
651
|
-
.parse(qb
|
|
652
|
-
.fetchOne({
|
|
653
|
-
tableName: alarmTableName,
|
|
654
|
-
fields: 'next_attempt_at',
|
|
655
|
-
where: { conditions: "status = 'pending'" },
|
|
656
|
-
orderBy: 'next_attempt_at ASC',
|
|
657
|
-
})
|
|
658
|
-
.execute().results)?.next_attempt_at;
|
|
659
|
-
const getNextAlarmAt = () => {
|
|
660
|
-
const candidates = [getNextPendingAt(), getNextNamedAlarmAt()].filter((value) => value !== undefined);
|
|
661
|
-
return candidates.length === 0 ? undefined : Math.min(...candidates);
|
|
662
|
-
};
|
|
663
|
-
const runConcurrent = async (items, concurrency, worker) => {
|
|
664
|
-
let nextIndex = 0;
|
|
665
|
-
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
666
|
-
while (nextIndex < items.length) {
|
|
667
|
-
const item = items[nextIndex++];
|
|
668
|
-
if (item !== undefined) {
|
|
669
|
-
// eslint-disable-next-line no-await-in-loop
|
|
670
|
-
await worker(item);
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
}));
|
|
674
|
-
};
|
|
675
|
-
const assertOperation = (id, existingOperation, requestedOperation) => {
|
|
676
|
-
if (existingOperation !== requestedOperation) {
|
|
677
|
-
throw new DuplicateDurableCallError(id, existingOperation, requestedOperation);
|
|
678
|
-
}
|
|
679
|
-
};
|
|
680
|
-
const reconcileAlarm = async (transaction) => {
|
|
681
|
-
const nextAlarmAt = getNextAlarmAt();
|
|
682
|
-
const currentAlarm = await transaction.getAlarm();
|
|
683
|
-
if (nextAlarmAt === undefined) {
|
|
684
|
-
if (currentAlarm !== null) {
|
|
685
|
-
await transaction.deleteAlarm();
|
|
686
|
-
}
|
|
687
|
-
return;
|
|
688
|
-
}
|
|
689
|
-
const target = Math.max(nextAlarmAt, Date.now());
|
|
690
|
-
if (currentAlarm !== target) {
|
|
691
|
-
await transaction.setAlarm(target);
|
|
692
|
-
}
|
|
693
|
-
};
|
|
694
|
-
const armForHandoff = async (timestamp) => {
|
|
695
|
-
await context.storage.transaction(async (transaction) => {
|
|
696
|
-
const currentAlarm = await transaction.getAlarm();
|
|
697
|
-
if (currentAlarm === null || currentAlarm > timestamp) {
|
|
698
|
-
await transaction.setAlarm(timestamp);
|
|
699
|
-
}
|
|
700
|
-
});
|
|
701
|
-
};
|
|
702
|
-
let alarmRefresh;
|
|
703
|
-
const scheduleNextAlarm = () => {
|
|
704
|
-
if (alarmRefresh) {
|
|
705
|
-
return alarmRefresh;
|
|
706
|
-
}
|
|
707
|
-
alarmRefresh = context.storage.transaction(reconcileAlarm).finally(() => {
|
|
708
|
-
alarmRefresh = undefined;
|
|
709
|
-
});
|
|
710
|
-
return alarmRefresh;
|
|
711
|
-
};
|
|
712
|
-
let availablePermits = alarmConcurrency;
|
|
713
|
-
const permitWaiters = [];
|
|
714
|
-
const createRelease = () => {
|
|
715
|
-
let released = false;
|
|
716
|
-
return () => {
|
|
717
|
-
if (released) {
|
|
718
|
-
return;
|
|
719
|
-
}
|
|
720
|
-
released = true;
|
|
721
|
-
const waiter = permitWaiters.shift();
|
|
722
|
-
if (waiter) {
|
|
723
|
-
waiter(createRelease());
|
|
724
|
-
}
|
|
725
|
-
else {
|
|
726
|
-
availablePermits += 1;
|
|
727
|
-
}
|
|
728
|
-
};
|
|
729
|
-
};
|
|
730
|
-
const tryAcquirePermit = () => {
|
|
731
|
-
if (availablePermits === 0) {
|
|
732
|
-
return undefined;
|
|
733
|
-
}
|
|
734
|
-
availablePermits -= 1;
|
|
735
|
-
return createRelease();
|
|
736
|
-
};
|
|
737
|
-
const acquirePermit = () => {
|
|
738
|
-
const release = tryAcquirePermit();
|
|
739
|
-
if (release) {
|
|
740
|
-
return Promise.resolve(release);
|
|
741
|
-
}
|
|
742
|
-
return new Promise((resolve) => {
|
|
743
|
-
permitWaiters.push(resolve);
|
|
744
|
-
});
|
|
745
|
-
};
|
|
746
|
-
const operationEntity = (call) => ({
|
|
747
|
-
entityKind: 'operation',
|
|
748
|
-
operation: call.operation,
|
|
749
|
-
id: call.id,
|
|
750
|
-
generation: call.generation_id,
|
|
751
|
-
});
|
|
752
|
-
const alarmEntity = (row) => ({
|
|
753
|
-
entityKind: 'named_alarm',
|
|
754
|
-
alarm: row.name,
|
|
755
|
-
id: row.name,
|
|
756
|
-
generation: row.generation_id,
|
|
757
|
-
});
|
|
758
|
-
const scheduleNamedAlarm = async (name, scheduledTime) => {
|
|
759
|
-
if (!Number.isSafeInteger(scheduledTime) || scheduledTime < 0) {
|
|
760
|
-
throw new RangeError('scheduledTime must be a non-negative safe integer');
|
|
761
|
-
}
|
|
762
|
-
const generation = crypto.randomUUID();
|
|
763
|
-
const createdAt = Date.now();
|
|
764
|
-
await context.storage.transaction(async (transaction) => {
|
|
765
|
-
context.storage.sql.exec(`INSERT INTO durability_alarms
|
|
766
|
-
(name, generation_id, status, scheduled_at, next_attempt_at, attempt,
|
|
767
|
-
created_at)
|
|
768
|
-
VALUES (?, ?, 'pending', ?, ?, 0, ?)
|
|
769
|
-
ON CONFLICT(name) DO UPDATE SET
|
|
770
|
-
generation_id = excluded.generation_id,
|
|
771
|
-
status = 'pending',
|
|
772
|
-
scheduled_at = excluded.scheduled_at,
|
|
773
|
-
next_attempt_at = excluded.next_attempt_at,
|
|
774
|
-
attempt = 0,
|
|
775
|
-
last_error = NULL,
|
|
776
|
-
last_error_name = NULL,
|
|
777
|
-
created_at = excluded.created_at`, name, generation, scheduledTime, scheduledTime, createdAt);
|
|
778
|
-
await reconcileAlarm(transaction);
|
|
779
|
-
});
|
|
780
|
-
emit({
|
|
781
|
-
type: 'scheduled',
|
|
782
|
-
entityKind: 'named_alarm',
|
|
783
|
-
alarm: name,
|
|
784
|
-
id: name,
|
|
785
|
-
generation,
|
|
786
|
-
timestamp: createdAt,
|
|
787
|
-
attempt: 0,
|
|
788
|
-
scheduledTime,
|
|
789
|
-
});
|
|
790
|
-
};
|
|
791
|
-
const executeNamedAlarm = (row, platform) => {
|
|
792
|
-
const running = activeAlarms.get(row.name);
|
|
793
|
-
if (running) {
|
|
794
|
-
return running.settled;
|
|
795
|
-
}
|
|
796
|
-
const policy = alarmExecutionPolicy(row.name);
|
|
797
|
-
let settleActive;
|
|
798
|
-
const settled = new Promise((resolve) => {
|
|
799
|
-
settleActive = resolve;
|
|
800
|
-
});
|
|
801
|
-
const activeExecution = {
|
|
802
|
-
generation: row.generation_id,
|
|
803
|
-
createdAt: row.created_at,
|
|
804
|
-
settled,
|
|
805
|
-
};
|
|
806
|
-
activeAlarms.set(row.name, activeExecution);
|
|
807
|
-
let handlerStarted = false;
|
|
808
|
-
let handlerSettled = false;
|
|
809
|
-
let policySettled = false;
|
|
810
|
-
let releasePermit;
|
|
811
|
-
const releaseIfSettled = () => {
|
|
812
|
-
if (policySettled && (!handlerStarted || handlerSettled)) {
|
|
813
|
-
releasePermit?.();
|
|
814
|
-
settleActive?.();
|
|
815
|
-
}
|
|
816
|
-
};
|
|
817
|
-
const execution = (async () => {
|
|
818
|
-
releasePermit = await acquirePermit();
|
|
819
|
-
const claimed = alarmAttemptSchema.optional().parse(context.storage.sql
|
|
820
|
-
.exec(`UPDATE durability_alarms
|
|
821
|
-
SET attempt = attempt + 1
|
|
822
|
-
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
823
|
-
AND attempt < ?
|
|
824
|
-
RETURNING attempt`, row.name, row.generation_id, policy.maxAttempts)
|
|
825
|
-
.toArray()[0]);
|
|
826
|
-
if (!claimed) {
|
|
827
|
-
const current = alarmRowSchema.optional().parse(qb
|
|
828
|
-
.fetchOne({
|
|
829
|
-
tableName: alarmTableName,
|
|
830
|
-
where: { conditions: 'name = ?', params: [row.name] },
|
|
831
|
-
})
|
|
832
|
-
.execute().results);
|
|
833
|
-
if (current?.generation_id === row.generation_id &&
|
|
834
|
-
current.attempt >= policy.maxAttempts) {
|
|
835
|
-
const exhausted = serializeError(new DurableAttemptsExhaustedError('named alarm', row.name, policy.maxAttempts));
|
|
836
|
-
let updated = false;
|
|
837
|
-
await context.storage.transaction(async (transaction) => {
|
|
838
|
-
updated =
|
|
839
|
-
context.storage.sql
|
|
840
|
-
.exec(`UPDATE durability_alarms
|
|
841
|
-
SET status = 'failed', last_error = ?, last_error_name = ?
|
|
842
|
-
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
843
|
-
AND attempt >= ?
|
|
844
|
-
RETURNING name`, exhausted.message, exhausted.name, row.name, row.generation_id, policy.maxAttempts)
|
|
845
|
-
.toArray().length > 0;
|
|
846
|
-
await reconcileAlarm(transaction);
|
|
847
|
-
});
|
|
848
|
-
if (updated) {
|
|
849
|
-
emit({
|
|
850
|
-
...alarmEntity(current),
|
|
851
|
-
type: 'terminal',
|
|
852
|
-
timestamp: Date.now(),
|
|
853
|
-
attempt: current.attempt,
|
|
854
|
-
reason: 'attempts_exhausted',
|
|
855
|
-
error: exhausted,
|
|
856
|
-
});
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
return;
|
|
860
|
-
}
|
|
861
|
-
const attempt = claimed.attempt;
|
|
862
|
-
const startedAt = Date.now();
|
|
863
|
-
const entity = alarmEntity(row);
|
|
864
|
-
emit({
|
|
865
|
-
...entity,
|
|
866
|
-
type: 'attempt_started',
|
|
867
|
-
timestamp: startedAt,
|
|
868
|
-
attempt,
|
|
869
|
-
});
|
|
870
|
-
const controller = new AbortController();
|
|
871
|
-
activeExecution.controller = controller;
|
|
872
|
-
let timeout;
|
|
873
|
-
try {
|
|
874
|
-
const handler = alarmHandlers.get(row.name);
|
|
875
|
-
if (!handler) {
|
|
876
|
-
throw new NonRetryableError(`No named alarm handler registered for "${row.name}"`);
|
|
877
|
-
}
|
|
878
|
-
handlerStarted = true;
|
|
879
|
-
const handlerResult = Promise.resolve().then(() => handler({
|
|
880
|
-
name: row.name,
|
|
881
|
-
scheduledTime: row.scheduled_at,
|
|
882
|
-
attempt,
|
|
883
|
-
isRetry: attempt > 1,
|
|
884
|
-
retryCount: attempt - 1,
|
|
885
|
-
idempotencyKey: `durability-alarm:v1:${row.generation_id}`,
|
|
886
|
-
signal: controller.signal,
|
|
887
|
-
platform,
|
|
888
|
-
}));
|
|
889
|
-
void handlerResult
|
|
890
|
-
.finally(() => {
|
|
891
|
-
handlerSettled = true;
|
|
892
|
-
releaseIfSettled();
|
|
893
|
-
})
|
|
894
|
-
.catch(() => undefined);
|
|
895
|
-
const timeoutResult = new Promise((_resolve, reject) => {
|
|
896
|
-
timeout = setTimeout(() => {
|
|
897
|
-
const error = new DurableAlarmTimeoutError(row.name, policy.attemptTimeoutMs);
|
|
898
|
-
controller.abort(error);
|
|
899
|
-
reject(error);
|
|
900
|
-
}, policy.attemptTimeoutMs);
|
|
901
|
-
});
|
|
902
|
-
await Promise.race([handlerResult, timeoutResult]);
|
|
903
|
-
const timestamp = Date.now();
|
|
904
|
-
const deleted = context.storage.sql
|
|
905
|
-
.exec(`DELETE FROM durability_alarms
|
|
906
|
-
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
907
|
-
AND attempt = ?
|
|
908
|
-
RETURNING name`, row.name, row.generation_id, attempt)
|
|
909
|
-
.toArray().length > 0;
|
|
910
|
-
if (deleted) {
|
|
911
|
-
emit({
|
|
912
|
-
...entity,
|
|
913
|
-
type: 'attempt_settled',
|
|
914
|
-
timestamp,
|
|
915
|
-
attempt,
|
|
916
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
917
|
-
outcome: 'completed',
|
|
918
|
-
});
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
catch (caught) {
|
|
922
|
-
let error = caught;
|
|
923
|
-
let terminal = isNonRetryable(error) ||
|
|
924
|
-
attempt >= policy.maxAttempts ||
|
|
925
|
-
(isErrorInstance(error, DurableAlarmTimeoutError) &&
|
|
926
|
-
!policy.retryTimeouts);
|
|
927
|
-
let timestamp = Date.now();
|
|
928
|
-
let delay = 0;
|
|
929
|
-
if (!terminal) {
|
|
930
|
-
try {
|
|
931
|
-
const rawDelay = policy.delay(attempt);
|
|
932
|
-
if (!Number.isFinite(rawDelay) || rawDelay < 0) {
|
|
933
|
-
throw new DurableRetryPolicyError(`named alarm "${row.name}"`);
|
|
934
|
-
}
|
|
935
|
-
delay = Math.round(rawDelay);
|
|
936
|
-
timestamp = Date.now();
|
|
937
|
-
if (!Number.isSafeInteger(delay) ||
|
|
938
|
-
!Number.isSafeInteger(timestamp + delay)) {
|
|
939
|
-
throw new DurableRetryPolicyError(`named alarm "${row.name}"`);
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
catch (policyError) {
|
|
943
|
-
error = isErrorInstance(policyError, DurableRetryPolicyError)
|
|
944
|
-
? policyError
|
|
945
|
-
: new DurableRetryPolicyError(`named alarm "${row.name}"`, policyError);
|
|
946
|
-
delay = 0;
|
|
947
|
-
terminal = true;
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
const nextAttemptAt = timestamp + delay;
|
|
951
|
-
const serializedError = serializeError(error);
|
|
952
|
-
let updated = false;
|
|
953
|
-
await context.storage.transaction(async (transaction) => {
|
|
954
|
-
updated =
|
|
955
|
-
context.storage.sql
|
|
956
|
-
.exec(`UPDATE durability_alarms
|
|
957
|
-
SET status = ?, next_attempt_at = ?, last_error = ?,
|
|
958
|
-
last_error_name = ?
|
|
959
|
-
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
960
|
-
AND attempt = ?
|
|
961
|
-
RETURNING name`, terminal ? 'failed' : 'pending', nextAttemptAt, serializedError.message, serializedError.name, row.name, row.generation_id, attempt)
|
|
962
|
-
.toArray().length > 0;
|
|
963
|
-
await reconcileAlarm(transaction);
|
|
964
|
-
});
|
|
965
|
-
if (updated) {
|
|
966
|
-
emit(terminal
|
|
967
|
-
? {
|
|
968
|
-
...entity,
|
|
969
|
-
type: 'attempt_settled',
|
|
970
|
-
timestamp,
|
|
971
|
-
attempt,
|
|
972
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
973
|
-
outcome: 'failed',
|
|
974
|
-
error: serializedError,
|
|
975
|
-
}
|
|
976
|
-
: {
|
|
977
|
-
...entity,
|
|
978
|
-
type: 'attempt_settled',
|
|
979
|
-
timestamp,
|
|
980
|
-
attempt,
|
|
981
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
982
|
-
outcome: 'retry_scheduled',
|
|
983
|
-
error: serializedError,
|
|
984
|
-
nextAttemptAt,
|
|
985
|
-
});
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
finally {
|
|
989
|
-
if (timeout !== undefined) {
|
|
990
|
-
clearTimeout(timeout);
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
})().finally(() => {
|
|
994
|
-
policySettled = true;
|
|
995
|
-
releaseIfSettled();
|
|
996
|
-
});
|
|
997
|
-
void settled.finally(() => {
|
|
998
|
-
if (activeAlarms.get(row.name) === activeExecution) {
|
|
999
|
-
activeAlarms.delete(row.name);
|
|
1000
|
-
}
|
|
1001
|
-
});
|
|
1002
|
-
return execution;
|
|
1003
|
-
};
|
|
1004
|
-
const execute = (id, permitMode = 'wait') => {
|
|
1005
|
-
const call = getCall(id);
|
|
1006
|
-
if (!call || call.status !== 'pending') {
|
|
1007
|
-
return permitMode === 'wait' ? Promise.resolve() : undefined;
|
|
1008
|
-
}
|
|
1009
|
-
const running = active.get(id);
|
|
1010
|
-
if (running?.generation === call.generation_id) {
|
|
1011
|
-
return permitMode === 'wait' ? running.settled : undefined;
|
|
1012
|
-
}
|
|
1013
|
-
let releasePermit;
|
|
1014
|
-
if (permitMode === 'immediate') {
|
|
1015
|
-
releasePermit = tryAcquirePermit();
|
|
1016
|
-
if (!releasePermit) {
|
|
1017
|
-
return undefined;
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
const policy = executionPolicy(call.operation);
|
|
1021
|
-
let settleActive;
|
|
1022
|
-
const settled = new Promise((resolve) => {
|
|
1023
|
-
settleActive = resolve;
|
|
1024
|
-
});
|
|
1025
|
-
const activeExecution = {
|
|
1026
|
-
generation: call.generation_id,
|
|
1027
|
-
createdAt: call.created_at,
|
|
1028
|
-
settled,
|
|
1029
|
-
};
|
|
1030
|
-
active.set(id, activeExecution);
|
|
1031
|
-
let handlerStarted = false;
|
|
1032
|
-
let handlerSettled = false;
|
|
1033
|
-
let policySettled = false;
|
|
1034
|
-
const releaseIfSettled = () => {
|
|
1035
|
-
if (policySettled && (!handlerStarted || handlerSettled)) {
|
|
1036
|
-
releasePermit?.();
|
|
1037
|
-
settleActive?.();
|
|
1038
|
-
}
|
|
1039
|
-
};
|
|
1040
|
-
const execution = (async () => {
|
|
1041
|
-
releasePermit ?? (releasePermit = await acquirePermit());
|
|
1042
|
-
const claimed = callAttemptSchema.optional().parse(context.storage.sql
|
|
1043
|
-
.exec(`UPDATE durability_calls
|
|
1044
|
-
SET attempt = attempt + 1
|
|
1045
|
-
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1046
|
-
AND attempt < ?
|
|
1047
|
-
RETURNING attempt`, id, call.generation_id, policy.maxAttempts)
|
|
1048
|
-
.toArray()[0]);
|
|
1049
|
-
if (!claimed) {
|
|
1050
|
-
const current = getCall(id);
|
|
1051
|
-
if (current?.generation_id === call.generation_id &&
|
|
1052
|
-
current.status === 'pending' &&
|
|
1053
|
-
current.attempt >= policy.maxAttempts) {
|
|
1054
|
-
const exhausted = serializeError(new DurableAttemptsExhaustedError('operation', call.operation, policy.maxAttempts));
|
|
1055
|
-
let updated = false;
|
|
1056
|
-
await context.storage.transaction(async (transaction) => {
|
|
1057
|
-
updated =
|
|
1058
|
-
context.storage.sql
|
|
1059
|
-
.exec(`UPDATE durability_calls
|
|
1060
|
-
SET status = 'failed', last_error = ?, last_error_name = ?
|
|
1061
|
-
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1062
|
-
AND attempt >= ?
|
|
1063
|
-
RETURNING id`, exhausted.message, exhausted.name, id, call.generation_id, policy.maxAttempts)
|
|
1064
|
-
.toArray().length > 0;
|
|
1065
|
-
await reconcileAlarm(transaction);
|
|
1066
|
-
});
|
|
1067
|
-
if (updated) {
|
|
1068
|
-
emit({
|
|
1069
|
-
...operationEntity(current),
|
|
1070
|
-
type: 'terminal',
|
|
1071
|
-
timestamp: Date.now(),
|
|
1072
|
-
attempt: current.attempt,
|
|
1073
|
-
reason: 'attempts_exhausted',
|
|
1074
|
-
error: exhausted,
|
|
1075
|
-
});
|
|
1076
|
-
}
|
|
1077
|
-
}
|
|
1078
|
-
return;
|
|
1079
|
-
}
|
|
1080
|
-
const attempt = claimed.attempt;
|
|
1081
|
-
const startedAt = Date.now();
|
|
1082
|
-
const entity = operationEntity(call);
|
|
1083
|
-
emit({
|
|
1084
|
-
...entity,
|
|
1085
|
-
type: 'attempt_started',
|
|
1086
|
-
timestamp: startedAt,
|
|
1087
|
-
attempt,
|
|
1088
|
-
});
|
|
1089
|
-
const controller = new AbortController();
|
|
1090
|
-
activeExecution.controller = controller;
|
|
1091
|
-
let timeout;
|
|
1092
|
-
try {
|
|
1093
|
-
const handler = handlers[call.operation];
|
|
1094
|
-
if (!handler) {
|
|
1095
|
-
throw new NonRetryableError(`No handler registered for operation "${call.operation}"`);
|
|
1096
|
-
}
|
|
1097
|
-
handlerStarted = true;
|
|
1098
|
-
const handlerResult = Promise.resolve().then(() => handler({
|
|
1099
|
-
id,
|
|
1100
|
-
operation: call.operation,
|
|
1101
|
-
payload: deserialize(call.payload),
|
|
1102
|
-
attempt,
|
|
1103
|
-
signal: controller.signal,
|
|
1104
|
-
}));
|
|
1105
|
-
void handlerResult
|
|
1106
|
-
.finally(() => {
|
|
1107
|
-
handlerSettled = true;
|
|
1108
|
-
releaseIfSettled();
|
|
1109
|
-
})
|
|
1110
|
-
.catch(() => undefined);
|
|
1111
|
-
const timeoutResult = new Promise((_resolve, reject) => {
|
|
1112
|
-
timeout = setTimeout(() => {
|
|
1113
|
-
const error = new DurableAttemptTimeoutError(call.operation, policy.attemptTimeoutMs);
|
|
1114
|
-
controller.abort(error);
|
|
1115
|
-
reject(error);
|
|
1116
|
-
}, policy.attemptTimeoutMs);
|
|
1117
|
-
});
|
|
1118
|
-
const result = await Promise.race([handlerResult, timeoutResult]);
|
|
1119
|
-
let serialized;
|
|
1120
|
-
try {
|
|
1121
|
-
serialized = serialize(result);
|
|
1122
|
-
}
|
|
1123
|
-
catch (error) {
|
|
1124
|
-
throw new DurableResultSerializationError(call.operation, error);
|
|
1125
|
-
}
|
|
1126
|
-
const timestamp = Date.now();
|
|
1127
|
-
const updated = context.storage.sql
|
|
1128
|
-
.exec(`UPDATE durability_calls
|
|
1129
|
-
SET status = 'completed', result = ?, last_error = NULL,
|
|
1130
|
-
last_error_name = NULL, completed_at = ?
|
|
1131
|
-
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1132
|
-
AND attempt = ?
|
|
1133
|
-
RETURNING id`, serialized, timestamp, id, call.generation_id, attempt)
|
|
1134
|
-
.toArray().length > 0;
|
|
1135
|
-
if (updated) {
|
|
1136
|
-
emit({
|
|
1137
|
-
...entity,
|
|
1138
|
-
type: 'attempt_settled',
|
|
1139
|
-
timestamp,
|
|
1140
|
-
attempt,
|
|
1141
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
1142
|
-
outcome: 'completed',
|
|
1143
|
-
});
|
|
1144
|
-
}
|
|
1145
|
-
}
|
|
1146
|
-
catch (caught) {
|
|
1147
|
-
let error = caught;
|
|
1148
|
-
let terminal = isNonRetryable(error) ||
|
|
1149
|
-
isErrorInstance(error, DurableResultSerializationError) ||
|
|
1150
|
-
attempt >= policy.maxAttempts;
|
|
1151
|
-
let timestamp = Date.now();
|
|
1152
|
-
let delay = 0;
|
|
1153
|
-
if (!terminal) {
|
|
1154
|
-
try {
|
|
1155
|
-
const rawDelay = policy.delay(attempt);
|
|
1156
|
-
if (!Number.isFinite(rawDelay) || rawDelay < 0) {
|
|
1157
|
-
throw new DurableRetryPolicyError(`operation "${call.operation}"`);
|
|
1158
|
-
}
|
|
1159
|
-
delay = Math.round(rawDelay);
|
|
1160
|
-
timestamp = Date.now();
|
|
1161
|
-
if (!Number.isSafeInteger(delay) ||
|
|
1162
|
-
!Number.isSafeInteger(timestamp + delay)) {
|
|
1163
|
-
throw new DurableRetryPolicyError(`operation "${call.operation}"`);
|
|
1164
|
-
}
|
|
1165
|
-
}
|
|
1166
|
-
catch (policyError) {
|
|
1167
|
-
error = isErrorInstance(policyError, DurableRetryPolicyError)
|
|
1168
|
-
? policyError
|
|
1169
|
-
: new DurableRetryPolicyError(`operation "${call.operation}"`, policyError);
|
|
1170
|
-
delay = 0;
|
|
1171
|
-
terminal = true;
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1174
|
-
const nextAttemptAt = timestamp + delay;
|
|
1175
|
-
const serializedError = serializeError(error);
|
|
1176
|
-
let updated = false;
|
|
1177
|
-
await context.storage.transaction(async (transaction) => {
|
|
1178
|
-
updated =
|
|
1179
|
-
context.storage.sql
|
|
1180
|
-
.exec(`UPDATE durability_calls
|
|
1181
|
-
SET status = ?, next_attempt_at = ?, last_error = ?,
|
|
1182
|
-
last_error_name = ?
|
|
1183
|
-
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1184
|
-
AND attempt = ?
|
|
1185
|
-
RETURNING id`, terminal ? 'failed' : 'pending', nextAttemptAt, serializedError.message, serializedError.name, id, call.generation_id, attempt)
|
|
1186
|
-
.toArray().length > 0;
|
|
1187
|
-
await reconcileAlarm(transaction);
|
|
1188
|
-
});
|
|
1189
|
-
if (updated) {
|
|
1190
|
-
emit(terminal
|
|
1191
|
-
? {
|
|
1192
|
-
...entity,
|
|
1193
|
-
type: 'attempt_settled',
|
|
1194
|
-
timestamp,
|
|
1195
|
-
attempt,
|
|
1196
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
1197
|
-
outcome: 'failed',
|
|
1198
|
-
error: serializedError,
|
|
1199
|
-
}
|
|
1200
|
-
: {
|
|
1201
|
-
...entity,
|
|
1202
|
-
type: 'attempt_settled',
|
|
1203
|
-
timestamp,
|
|
1204
|
-
attempt,
|
|
1205
|
-
durationMs: Math.max(0, timestamp - startedAt),
|
|
1206
|
-
outcome: 'retry_scheduled',
|
|
1207
|
-
error: serializedError,
|
|
1208
|
-
nextAttemptAt,
|
|
1209
|
-
});
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
|
-
finally {
|
|
1213
|
-
if (timeout !== undefined) {
|
|
1214
|
-
clearTimeout(timeout);
|
|
1215
|
-
}
|
|
1216
|
-
}
|
|
1217
|
-
})().finally(() => {
|
|
1218
|
-
policySettled = true;
|
|
1219
|
-
releaseIfSettled();
|
|
1220
|
-
});
|
|
1221
|
-
void settled.finally(() => {
|
|
1222
|
-
if (active.get(id) === activeExecution) {
|
|
1223
|
-
active.delete(id);
|
|
1224
|
-
}
|
|
1225
|
-
});
|
|
1226
|
-
return execution;
|
|
1227
|
-
};
|
|
1228
|
-
const run = async (input) => {
|
|
1229
|
-
const payload = serialize(input.payload);
|
|
1230
|
-
const now = Date.now();
|
|
1231
|
-
const generation = crypto.randomUUID();
|
|
1232
|
-
let inserted = false;
|
|
1233
|
-
let winner;
|
|
1234
|
-
await context.storage.transaction(async (transaction) => {
|
|
1235
|
-
inserted =
|
|
1236
|
-
context.storage.sql
|
|
1237
|
-
.exec(`INSERT INTO durability_calls
|
|
1238
|
-
(id, operation, payload, status, attempt, next_attempt_at,
|
|
1239
|
-
generation_id, created_at)
|
|
1240
|
-
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?)
|
|
1241
|
-
ON CONFLICT(id) DO NOTHING
|
|
1242
|
-
RETURNING id`, input.id, input.operation, payload, now, generation, now)
|
|
1243
|
-
.toArray().length > 0;
|
|
1244
|
-
winner = getCall(input.id);
|
|
1245
|
-
if (!winner) {
|
|
1246
|
-
throw new Error(`Durable call "${input.id}" was not persisted`);
|
|
1247
|
-
}
|
|
1248
|
-
assertOperation(input.id, winner.operation, input.operation);
|
|
1249
|
-
await reconcileAlarm(transaction);
|
|
1250
|
-
});
|
|
1251
|
-
if (inserted && winner) {
|
|
1252
|
-
emit({
|
|
1253
|
-
...operationEntity(winner),
|
|
1254
|
-
type: 'registered',
|
|
1255
|
-
timestamp: now,
|
|
1256
|
-
attempt: 0,
|
|
1257
|
-
});
|
|
1258
|
-
}
|
|
1259
|
-
if (inserted && winner?.status === 'pending') {
|
|
1260
|
-
const eagerExecution = execute(input.id, 'immediate');
|
|
1261
|
-
if (!eagerExecution) {
|
|
1262
|
-
return;
|
|
1263
|
-
}
|
|
1264
|
-
const background = eagerExecution
|
|
1265
|
-
.then(scheduleNextAlarm)
|
|
1266
|
-
.catch((error) => {
|
|
1267
|
-
reportFailure('durability.background_execution.failed', { operation: input.operation, id: input.id }, error);
|
|
1268
|
-
});
|
|
1269
|
-
if (context.waitUntil) {
|
|
1270
|
-
try {
|
|
1271
|
-
context.waitUntil(background);
|
|
1272
|
-
}
|
|
1273
|
-
catch (error) {
|
|
1274
|
-
reportFailure('durability.background_execution.failed', { operation: input.operation, id: input.id }, error);
|
|
1275
|
-
}
|
|
1276
|
-
}
|
|
1277
|
-
void background;
|
|
1278
|
-
}
|
|
1279
|
-
};
|
|
1280
|
-
const purgeBefore = async (before) => {
|
|
1281
|
-
if (!Number.isSafeInteger(before) || before < 0) {
|
|
1282
|
-
throw new RangeError('timestamp must be a non-negative safe integer');
|
|
1283
|
-
}
|
|
1284
|
-
let operations = 0;
|
|
1285
|
-
let namedAlarms = 0;
|
|
1286
|
-
await context.storage.transaction(async (transaction) => {
|
|
1287
|
-
operations = countSchema.parse(context.storage.sql
|
|
1288
|
-
.exec('SELECT COUNT(*) AS count FROM durability_calls WHERE created_at < ?', before)
|
|
1289
|
-
.toArray()[0]).count;
|
|
1290
|
-
namedAlarms = countSchema.parse(context.storage.sql
|
|
1291
|
-
.exec('SELECT COUNT(*) AS count FROM durability_alarms WHERE created_at < ?', before)
|
|
1292
|
-
.toArray()[0]).count;
|
|
1293
|
-
for (const running of active.values()) {
|
|
1294
|
-
if (running.createdAt < before) {
|
|
1295
|
-
running.controller?.abort(new Error('Durable operation was purged'));
|
|
1296
|
-
}
|
|
1297
|
-
}
|
|
1298
|
-
for (const running of activeAlarms.values()) {
|
|
1299
|
-
if (running.createdAt < before) {
|
|
1300
|
-
running.controller?.abort(new Error('Durable alarm was purged'));
|
|
1301
|
-
}
|
|
1302
|
-
}
|
|
1303
|
-
context.storage.sql.exec('DELETE FROM durability_calls WHERE created_at < ?', before);
|
|
1304
|
-
context.storage.sql.exec('DELETE FROM durability_alarms WHERE created_at < ?', before);
|
|
1305
|
-
await reconcileAlarm(transaction);
|
|
1306
|
-
});
|
|
1307
|
-
const result = {
|
|
1308
|
-
operations,
|
|
1309
|
-
namedAlarms,
|
|
1310
|
-
total: operations + namedAlarms,
|
|
1311
|
-
};
|
|
1312
|
-
emit({
|
|
1313
|
-
type: 'purged',
|
|
1314
|
-
entityKind: 'durability',
|
|
1315
|
-
timestamp: Date.now(),
|
|
1316
|
-
before,
|
|
1317
|
-
...result,
|
|
1318
|
-
});
|
|
1319
|
-
return result;
|
|
1320
|
-
};
|
|
1321
|
-
const alarm = async (alarmInfo) => {
|
|
1322
|
-
const startedAt = Date.now();
|
|
1323
|
-
const due = listDueIds(startedAt, 100);
|
|
1324
|
-
const dueAlarms = listDueAlarms(startedAt, 100);
|
|
1325
|
-
if (due.length === 0 && dueAlarms.length === 0) {
|
|
1326
|
-
await scheduleNextAlarm();
|
|
1327
|
-
return;
|
|
1328
|
-
}
|
|
1329
|
-
const operationExecution = runConcurrent(due, alarmConcurrency, async (id) => {
|
|
1330
|
-
try {
|
|
1331
|
-
await execute(id);
|
|
1332
|
-
}
|
|
1333
|
-
catch {
|
|
1334
|
-
return;
|
|
1335
|
-
}
|
|
1336
|
-
});
|
|
1337
|
-
const namedAlarmFailures = [];
|
|
1338
|
-
const namedAlarmExecution = runConcurrent(dueAlarms, alarmConcurrency, async (row) => {
|
|
1339
|
-
try {
|
|
1340
|
-
await executeNamedAlarm(row, alarmInfo);
|
|
1341
|
-
}
|
|
1342
|
-
catch (error) {
|
|
1343
|
-
namedAlarmFailures.push(error);
|
|
1344
|
-
}
|
|
1345
|
-
}).then(() => {
|
|
1346
|
-
if (namedAlarmFailures.length > 0) {
|
|
1347
|
-
throw namedAlarmFailures[0];
|
|
1348
|
-
}
|
|
1349
|
-
return undefined;
|
|
1350
|
-
});
|
|
1351
|
-
const execution = Promise.all([
|
|
1352
|
-
operationExecution,
|
|
1353
|
-
namedAlarmExecution,
|
|
1354
|
-
]).then(() => undefined);
|
|
1355
|
-
void execution.catch(() => undefined);
|
|
1356
|
-
let timeout;
|
|
1357
|
-
const handoff = Symbol('alarm handoff');
|
|
1358
|
-
const remaining = alarmHandoffMs - (Date.now() - startedAt);
|
|
1359
|
-
let result;
|
|
1360
|
-
try {
|
|
1361
|
-
result = await Promise.race([
|
|
1362
|
-
execution,
|
|
1363
|
-
new Promise((resolve) => {
|
|
1364
|
-
timeout = setTimeout(() => resolve(handoff), Math.max(remaining, 0));
|
|
1365
|
-
}),
|
|
1366
|
-
]);
|
|
1367
|
-
}
|
|
1368
|
-
finally {
|
|
1369
|
-
if (timeout !== undefined) {
|
|
1370
|
-
clearTimeout(timeout);
|
|
1371
|
-
}
|
|
1372
|
-
}
|
|
1373
|
-
if (result === handoff) {
|
|
1374
|
-
await armForHandoff(Date.now());
|
|
1375
|
-
return;
|
|
1376
|
-
}
|
|
1377
|
-
await scheduleNextAlarm();
|
|
1378
|
-
};
|
|
1379
|
-
for (const name of alarmHandlers.keys()) {
|
|
1380
|
-
if (name in alarm) {
|
|
1381
|
-
throw new Error(`"${name}" is reserved by durability alarms`);
|
|
1382
|
-
}
|
|
1383
|
-
Object.assign(alarm, {
|
|
1384
|
-
[name]: (scheduledTime) => scheduleNamedAlarm(name, scheduledTime),
|
|
1385
|
-
});
|
|
1386
|
-
}
|
|
1387
|
-
const durability = { alarm, purgeBefore };
|
|
1388
|
-
for (const operation of Object.keys(handlers)) {
|
|
1389
|
-
if (operation === 'alarm' || operation === 'purgeBefore') {
|
|
1390
|
-
throw new Error(`"${operation}" is reserved by durability`);
|
|
1391
|
-
}
|
|
1392
|
-
const durableOperation = (input) => run({ ...input, operation });
|
|
1393
|
-
durableOperation.getResult = async (idempotencyKey) => getResult(operation, idempotencyKey);
|
|
1394
|
-
durability[operation] = durableOperation;
|
|
1395
|
-
}
|
|
1396
|
-
const isCompleteDurability = (value) => {
|
|
1397
|
-
const alarmValue = value['alarm'];
|
|
1398
|
-
if (typeof alarmValue !== 'function' ||
|
|
1399
|
-
typeof value['purgeBefore'] !== 'function') {
|
|
1400
|
-
return false;
|
|
1401
|
-
}
|
|
1402
|
-
for (const name of alarmHandlers.keys()) {
|
|
1403
|
-
if (typeof Reflect.get(alarmValue, name) !== 'function') {
|
|
1404
|
-
return false;
|
|
1405
|
-
}
|
|
1406
|
-
}
|
|
1407
|
-
for (const operation of Object.keys(handlers)) {
|
|
1408
|
-
const method = value[operation];
|
|
1409
|
-
if (typeof method !== 'function' ||
|
|
1410
|
-
typeof Reflect.get(method, 'getResult') !== 'function') {
|
|
1411
|
-
return false;
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
return true;
|
|
1415
|
-
};
|
|
1416
|
-
if (!isCompleteDurability(durability)) {
|
|
1417
|
-
throw new Error('Durability methods were not initialized');
|
|
1418
|
-
}
|
|
1419
|
-
return durability;
|
|
1420
|
-
};
|
|
1
|
+
export { Durability, } from './durability.js';
|
|
2
|
+
export { DurabilityAlarms, } from './durability-alarms.js';
|
|
3
|
+
export { DurabilityFanout, } from './fanout.js';
|
|
4
|
+
export { RoutingLoad, } from './load.js';
|
|
5
|
+
export { DurabilityScheduler, } from './scheduler.js';
|
|
6
|
+
export { DuplicateDurableCallError, DurableAlarmTimeoutError, DurableAttemptsExhaustedError, DurableAttemptTimeoutError, DurableResultSerializationError, DurableRetryPolicyError, NonRetryableError, FanoutTimeoutError, FanoutSettlementError, FanoutEnqueueError, RoutingError, } from './errors.js';
|
|
7
|
+
export { durabilityNamedAlarmMigrations, durabilityOperationMigrations, durabilityFanoutMigrations, } from './migrations.js';
|