durability 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -9
- package/dist/index.d.ts +133 -48
- package/dist/index.js +1042 -173
- package/package.json +18 -6
package/dist/index.js
CHANGED
|
@@ -1,7 +1,52 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { DOQB } from 'workers-qb';
|
|
3
3
|
import { exponential, jitter } from './utils.js';
|
|
4
|
+
const alarmHandlerSchema = z.custom((value) => typeof value === 'function');
|
|
5
|
+
const retryOptionsSchema = z.object({
|
|
6
|
+
delay: z
|
|
7
|
+
.custom((value) => typeof value === 'function')
|
|
8
|
+
.optional(),
|
|
9
|
+
maxAttempts: z.number().optional(),
|
|
10
|
+
});
|
|
11
|
+
const durabilityMethodOptionsSchema = z.object({
|
|
12
|
+
attemptTimeoutMs: z.number().optional(),
|
|
13
|
+
retries: retryOptionsSchema.optional(),
|
|
14
|
+
});
|
|
15
|
+
const durabilityAlarmMethodOptionsSchema = durabilityMethodOptionsSchema.extend({
|
|
16
|
+
retryTimeouts: z.boolean().optional(),
|
|
17
|
+
});
|
|
18
|
+
const callRowSchema = z.object({
|
|
19
|
+
id: z.string(),
|
|
20
|
+
operation: z.string(),
|
|
21
|
+
payload: z.string(),
|
|
22
|
+
status: z.enum(['pending', 'completed', 'failed']),
|
|
23
|
+
result: z.string().nullable(),
|
|
24
|
+
attempt: z.number().int().nonnegative(),
|
|
25
|
+
next_attempt_at: z.number().int().nonnegative(),
|
|
26
|
+
last_error: z.string().nullable(),
|
|
27
|
+
last_error_name: z.string().nullable(),
|
|
28
|
+
completed_at: z.number().int().nonnegative().nullable(),
|
|
29
|
+
created_at: z.number().int().nonnegative(),
|
|
30
|
+
generation_id: z.string(),
|
|
31
|
+
});
|
|
32
|
+
const callAttemptSchema = callRowSchema.pick({ attempt: true });
|
|
33
|
+
const callIdSchema = callRowSchema.pick({ id: true });
|
|
34
|
+
const callNextAttemptSchema = callRowSchema.pick({ next_attempt_at: true });
|
|
35
|
+
const alarmRowSchema = z.object({
|
|
36
|
+
name: z.string(),
|
|
37
|
+
generation_id: z.string(),
|
|
38
|
+
status: z.enum(['pending', 'failed']),
|
|
39
|
+
scheduled_at: z.number().int().nonnegative(),
|
|
40
|
+
next_attempt_at: z.number().int().nonnegative(),
|
|
41
|
+
attempt: z.number().int().nonnegative(),
|
|
42
|
+
last_error: z.string().nullable(),
|
|
43
|
+
last_error_name: z.string().nullable(),
|
|
44
|
+
created_at: z.number().int().nonnegative(),
|
|
45
|
+
});
|
|
46
|
+
const alarmAttemptSchema = alarmRowSchema.pick({ attempt: true });
|
|
47
|
+
const countSchema = z.object({ count: z.number().int().nonnegative() });
|
|
4
48
|
const tableName = 'durability_calls';
|
|
49
|
+
const alarmTableName = 'durability_alarms';
|
|
5
50
|
const migrationTableName = 'durability_migrations';
|
|
6
51
|
const storedValueSchema = z.discriminatedUnion('kind', [
|
|
7
52
|
z.object({ kind: z.literal('value'), value: z.json() }),
|
|
@@ -54,6 +99,127 @@ export const durabilityMigrations = [
|
|
|
54
99
|
`,
|
|
55
100
|
down: 'DROP INDEX IF EXISTS durability_calls_pending_idx;',
|
|
56
101
|
},
|
|
102
|
+
{
|
|
103
|
+
name: 'durability_0003_create_alarms',
|
|
104
|
+
up: `
|
|
105
|
+
CREATE TABLE IF NOT EXISTS durability_alarms (
|
|
106
|
+
name TEXT PRIMARY KEY,
|
|
107
|
+
generation_id TEXT NOT NULL,
|
|
108
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
109
|
+
scheduled_at INTEGER NOT NULL,
|
|
110
|
+
next_attempt_at INTEGER NOT NULL,
|
|
111
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
112
|
+
last_error TEXT,
|
|
113
|
+
last_error_name TEXT
|
|
114
|
+
);
|
|
115
|
+
CREATE INDEX IF NOT EXISTS durability_alarms_pending_idx
|
|
116
|
+
ON durability_alarms (status, next_attempt_at);
|
|
117
|
+
`,
|
|
118
|
+
down: `
|
|
119
|
+
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
120
|
+
DROP TABLE IF EXISTS durability_alarms;
|
|
121
|
+
`,
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
name: 'durability_0004_generation_and_created_at',
|
|
125
|
+
up: `
|
|
126
|
+
DROP INDEX IF EXISTS durability_calls_pending_idx;
|
|
127
|
+
ALTER TABLE durability_calls RENAME TO durability_calls_v3;
|
|
128
|
+
CREATE TABLE durability_calls (
|
|
129
|
+
id TEXT PRIMARY KEY,
|
|
130
|
+
operation TEXT NOT NULL,
|
|
131
|
+
payload TEXT NOT NULL,
|
|
132
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
|
|
133
|
+
result TEXT,
|
|
134
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
135
|
+
next_attempt_at INTEGER NOT NULL,
|
|
136
|
+
last_error TEXT,
|
|
137
|
+
last_error_name TEXT,
|
|
138
|
+
completed_at INTEGER,
|
|
139
|
+
generation_id TEXT NOT NULL DEFAULT 'legacy',
|
|
140
|
+
created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))
|
|
141
|
+
);
|
|
142
|
+
INSERT INTO durability_calls
|
|
143
|
+
SELECT id, operation, payload, status, result, attempt, next_attempt_at,
|
|
144
|
+
last_error, last_error_name, completed_at, 'legacy:' || id,
|
|
145
|
+
CAST(unixepoch('subsec') * 1000 AS INTEGER)
|
|
146
|
+
FROM durability_calls_v3;
|
|
147
|
+
DROP TABLE durability_calls_v3;
|
|
148
|
+
CREATE INDEX durability_calls_pending_idx
|
|
149
|
+
ON durability_calls (status, next_attempt_at);
|
|
150
|
+
CREATE INDEX durability_calls_created_idx
|
|
151
|
+
ON durability_calls (created_at);
|
|
152
|
+
|
|
153
|
+
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
154
|
+
ALTER TABLE durability_alarms RENAME TO durability_alarms_v3;
|
|
155
|
+
CREATE TABLE durability_alarms (
|
|
156
|
+
name TEXT PRIMARY KEY,
|
|
157
|
+
generation_id TEXT NOT NULL,
|
|
158
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
159
|
+
scheduled_at INTEGER NOT NULL,
|
|
160
|
+
next_attempt_at INTEGER NOT NULL,
|
|
161
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
162
|
+
last_error TEXT,
|
|
163
|
+
last_error_name TEXT,
|
|
164
|
+
created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))
|
|
165
|
+
);
|
|
166
|
+
INSERT INTO durability_alarms
|
|
167
|
+
SELECT name, generation_id, status, scheduled_at, next_attempt_at,
|
|
168
|
+
attempt, last_error, last_error_name,
|
|
169
|
+
CAST(unixepoch('subsec') * 1000 AS INTEGER)
|
|
170
|
+
FROM durability_alarms_v3;
|
|
171
|
+
DROP TABLE durability_alarms_v3;
|
|
172
|
+
CREATE INDEX durability_alarms_pending_idx
|
|
173
|
+
ON durability_alarms (status, next_attempt_at);
|
|
174
|
+
CREATE INDEX durability_alarms_created_idx
|
|
175
|
+
ON durability_alarms (created_at);
|
|
176
|
+
`,
|
|
177
|
+
down: `
|
|
178
|
+
DROP INDEX IF EXISTS durability_calls_pending_idx;
|
|
179
|
+
DROP INDEX IF EXISTS durability_calls_created_idx;
|
|
180
|
+
ALTER TABLE durability_calls RENAME TO durability_calls_v4;
|
|
181
|
+
CREATE TABLE durability_calls (
|
|
182
|
+
id TEXT PRIMARY KEY,
|
|
183
|
+
operation TEXT NOT NULL,
|
|
184
|
+
payload TEXT NOT NULL,
|
|
185
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'failed')),
|
|
186
|
+
result TEXT,
|
|
187
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
188
|
+
next_attempt_at INTEGER NOT NULL,
|
|
189
|
+
last_error TEXT,
|
|
190
|
+
last_error_name TEXT,
|
|
191
|
+
completed_at INTEGER
|
|
192
|
+
);
|
|
193
|
+
INSERT INTO durability_calls
|
|
194
|
+
SELECT id, operation, payload, status, result, attempt, next_attempt_at,
|
|
195
|
+
last_error, last_error_name, completed_at
|
|
196
|
+
FROM durability_calls_v4;
|
|
197
|
+
DROP TABLE durability_calls_v4;
|
|
198
|
+
CREATE INDEX durability_calls_pending_idx
|
|
199
|
+
ON durability_calls (status, next_attempt_at);
|
|
200
|
+
|
|
201
|
+
DROP INDEX IF EXISTS durability_alarms_pending_idx;
|
|
202
|
+
DROP INDEX IF EXISTS durability_alarms_created_idx;
|
|
203
|
+
ALTER TABLE durability_alarms RENAME TO durability_alarms_v4;
|
|
204
|
+
CREATE TABLE durability_alarms (
|
|
205
|
+
name TEXT PRIMARY KEY,
|
|
206
|
+
generation_id TEXT NOT NULL,
|
|
207
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'failed')),
|
|
208
|
+
scheduled_at INTEGER NOT NULL,
|
|
209
|
+
next_attempt_at INTEGER NOT NULL,
|
|
210
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
211
|
+
last_error TEXT,
|
|
212
|
+
last_error_name TEXT
|
|
213
|
+
);
|
|
214
|
+
INSERT INTO durability_alarms
|
|
215
|
+
SELECT name, generation_id, status, scheduled_at, next_attempt_at,
|
|
216
|
+
attempt, last_error, last_error_name
|
|
217
|
+
FROM durability_alarms_v4;
|
|
218
|
+
DROP TABLE durability_alarms_v4;
|
|
219
|
+
CREATE INDEX durability_alarms_pending_idx
|
|
220
|
+
ON durability_alarms (status, next_attempt_at);
|
|
221
|
+
`,
|
|
222
|
+
},
|
|
57
223
|
];
|
|
58
224
|
/**
|
|
59
225
|
* Moves the durability schema in one Durable Object instance to a target.
|
|
@@ -109,6 +275,9 @@ export const migrateDurability = (context, target = durabilityMigrations[durabil
|
|
|
109
275
|
})
|
|
110
276
|
.apply()
|
|
111
277
|
.map((migration) => migration.name);
|
|
278
|
+
if (target === null) {
|
|
279
|
+
context.storage.sql.exec(`DROP TABLE IF EXISTS ${migrationTableName}`);
|
|
280
|
+
}
|
|
112
281
|
return { applied, rolledBack };
|
|
113
282
|
};
|
|
114
283
|
/**
|
|
@@ -154,6 +323,40 @@ export class DurableAttemptTimeoutError extends Error {
|
|
|
154
323
|
this.name = 'DurableAttemptTimeoutError';
|
|
155
324
|
}
|
|
156
325
|
}
|
|
326
|
+
/** Error recorded when a named alarm attempt exceeds its configured timeout. */
|
|
327
|
+
export class DurableAlarmTimeoutError extends Error {
|
|
328
|
+
constructor(name, timeoutMs) {
|
|
329
|
+
super(`Durable alarm "${name}" timed out after ${timeoutMs}ms`);
|
|
330
|
+
this.name = 'DurableAlarmTimeoutError';
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
/** Error persisted when retry-delay evaluation fails or is unsafe. */
|
|
334
|
+
export class DurableRetryPolicyError extends Error {
|
|
335
|
+
constructor(entity, reason) {
|
|
336
|
+
super(`Retry policy for ${entity} must produce a non-negative safe-integer timestamp`);
|
|
337
|
+
this.name = 'DurableRetryPolicyError';
|
|
338
|
+
if (reason !== undefined) {
|
|
339
|
+
Object.defineProperty(this, 'cause', { value: reason });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/** Error persisted when a successful result cannot be JSON-serialized. */
|
|
344
|
+
export class DurableResultSerializationError extends Error {
|
|
345
|
+
constructor(operation, reason) {
|
|
346
|
+
super(`Result for durable operation "${operation}" is not JSON-serializable`);
|
|
347
|
+
this.name = 'DurableResultSerializationError';
|
|
348
|
+
if (reason !== undefined) {
|
|
349
|
+
Object.defineProperty(this, 'cause', { value: reason });
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
/** Error persisted when pending work has already reached its attempt limit. */
|
|
354
|
+
export class DurableAttemptsExhaustedError extends Error {
|
|
355
|
+
constructor(entity, name, max) {
|
|
356
|
+
super(`Durable ${entity} "${name}" exhausted its ${max} attempts`);
|
|
357
|
+
this.name = 'DurableAttemptsExhaustedError';
|
|
358
|
+
}
|
|
359
|
+
}
|
|
157
360
|
/**
|
|
158
361
|
* Error thrown when an idempotency key is reused for another operation.
|
|
159
362
|
*
|
|
@@ -173,56 +376,21 @@ export class DuplicateDurableCallError extends Error {
|
|
|
173
376
|
this.name = 'DuplicateDurableCallError';
|
|
174
377
|
}
|
|
175
378
|
}
|
|
176
|
-
/**
|
|
177
|
-
* Creates typed, alarm-backed durable methods from an operation handler map.
|
|
178
|
-
*
|
|
179
|
-
* Registered calls persist before execution and are deduplicated by their IDs.
|
|
180
|
-
* Generated methods resolve after registration; read handler results with each
|
|
181
|
-
* method's `getResult` function.
|
|
182
|
-
*
|
|
183
|
-
* The Durable Object class must use SQLite storage, and its `alarm` method must
|
|
184
|
-
* delegate to the returned alarm handler. External side effects should use the
|
|
185
|
-
* call ID as an idempotency key because a crash can occur after a side effect but
|
|
186
|
-
* before the completion record commits.
|
|
187
|
-
*
|
|
188
|
-
* @example
|
|
189
|
-
* ```ts
|
|
190
|
-
* class ImageJobs extends DurableObject<Env> {
|
|
191
|
-
* private readonly durability = createDurability(this.ctx, {
|
|
192
|
-
* resizeImage: async ({ id, payload, signal }) =>
|
|
193
|
-
* this.env.IMAGES.resize(payload.imageId, {
|
|
194
|
-
* idempotencyKey: id,
|
|
195
|
-
* signal,
|
|
196
|
-
* }),
|
|
197
|
-
* });
|
|
198
|
-
*
|
|
199
|
-
* resize(imageId: string) {
|
|
200
|
-
* return this.durability.resizeImage({
|
|
201
|
-
* id: `resize:${imageId}`,
|
|
202
|
-
* payload: { imageId },
|
|
203
|
-
* });
|
|
204
|
-
* }
|
|
205
|
-
*
|
|
206
|
-
* alarm(info: AlarmInvocationInfo) {
|
|
207
|
-
* return this.durability.alarm(info);
|
|
208
|
-
* }
|
|
209
|
-
* }
|
|
210
|
-
* ```
|
|
211
|
-
*
|
|
212
|
-
* @param context Durable Object context containing SQLite-backed storage.
|
|
213
|
-
* @param handlers Operation handlers keyed by their public method names.
|
|
214
|
-
* @param options Concurrency, timeout, and retry policy configuration.
|
|
215
|
-
*/
|
|
216
379
|
export const createDurability = (context, handlers, options = {}) => {
|
|
217
380
|
migrateDurability(context);
|
|
218
381
|
const qb = new DOQB(context.storage.sql);
|
|
219
382
|
const active = new Map();
|
|
383
|
+
const activeAlarms = new Map();
|
|
384
|
+
const alarmHandlers = new Map(Object.entries(options.alarms ?? {}).map(([name, handler]) => [
|
|
385
|
+
name,
|
|
386
|
+
alarmHandlerSchema.parse(handler),
|
|
387
|
+
]));
|
|
220
388
|
const alarmConcurrency = options.alarmConcurrency ?? 10;
|
|
221
389
|
const alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
|
|
222
390
|
const defaultAttemptTimeoutMs = options.attemptTimeoutMs ?? 5 * 60000;
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
const defaultMaxAttempts =
|
|
391
|
+
const retryOptions = retryOptionsSchema.parse(options.retries ?? {});
|
|
392
|
+
const defaultRetryDelay = retryOptions.delay ?? ((attempt) => jitter(exponential(attempt)));
|
|
393
|
+
const defaultMaxAttempts = retryOptions.maxAttempts ?? 5;
|
|
226
394
|
if (!Number.isInteger(alarmConcurrency) || alarmConcurrency < 1) {
|
|
227
395
|
throw new RangeError('alarmConcurrency must be a positive integer');
|
|
228
396
|
}
|
|
@@ -233,8 +401,11 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
233
401
|
if (!Number.isInteger(defaultMaxAttempts) || defaultMaxAttempts < 1) {
|
|
234
402
|
throw new RangeError('retries.maxAttempts must be a positive integer');
|
|
235
403
|
}
|
|
236
|
-
const methodOptions = (options.methods ?? {})
|
|
237
|
-
|
|
404
|
+
const methodOptions = new Map(Object.entries(options.methods ?? {}).map(([operation, method]) => [
|
|
405
|
+
operation,
|
|
406
|
+
durabilityMethodOptionsSchema.parse(method),
|
|
407
|
+
]));
|
|
408
|
+
for (const [operation, method] of methodOptions) {
|
|
238
409
|
if (method?.attemptTimeoutMs !== undefined &&
|
|
239
410
|
(!Number.isInteger(method.attemptTimeoutMs) ||
|
|
240
411
|
method.attemptTimeoutMs < 1)) {
|
|
@@ -246,24 +417,156 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
246
417
|
throw new RangeError(`methods.${operation}.retries.maxAttempts must be a positive integer`);
|
|
247
418
|
}
|
|
248
419
|
}
|
|
420
|
+
const alarmMethodOptions = new Map(Object.entries(options.alarmMethods ?? {}).map(([name, method]) => [
|
|
421
|
+
name,
|
|
422
|
+
durabilityAlarmMethodOptionsSchema.parse(method),
|
|
423
|
+
]));
|
|
424
|
+
for (const [name, method] of alarmMethodOptions) {
|
|
425
|
+
if (method?.attemptTimeoutMs !== undefined &&
|
|
426
|
+
(!Number.isInteger(method.attemptTimeoutMs) ||
|
|
427
|
+
method.attemptTimeoutMs < 1)) {
|
|
428
|
+
throw new RangeError(`alarmMethods.${name}.attemptTimeoutMs must be a positive integer`);
|
|
429
|
+
}
|
|
430
|
+
if (method?.retries?.maxAttempts !== undefined &&
|
|
431
|
+
(!Number.isInteger(method.retries.maxAttempts) ||
|
|
432
|
+
method.retries.maxAttempts < 1)) {
|
|
433
|
+
throw new RangeError(`alarmMethods.${name}.retries.maxAttempts must be a positive integer`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
249
436
|
const executionPolicy = (operation) => {
|
|
250
|
-
const method = methodOptions
|
|
437
|
+
const method = methodOptions.get(operation);
|
|
251
438
|
return {
|
|
252
439
|
attemptTimeoutMs: method?.attemptTimeoutMs ?? defaultAttemptTimeoutMs,
|
|
253
440
|
delay: method?.retries?.delay ?? defaultRetryDelay,
|
|
254
441
|
maxAttempts: method?.retries?.maxAttempts ?? defaultMaxAttempts,
|
|
255
442
|
};
|
|
256
443
|
};
|
|
444
|
+
const alarmExecutionPolicy = (name) => {
|
|
445
|
+
const method = alarmMethodOptions.get(name);
|
|
446
|
+
return {
|
|
447
|
+
attemptTimeoutMs: method?.attemptTimeoutMs ?? defaultAttemptTimeoutMs,
|
|
448
|
+
delay: method?.retries?.delay ?? defaultRetryDelay,
|
|
449
|
+
maxAttempts: method?.retries?.maxAttempts ?? defaultMaxAttempts,
|
|
450
|
+
retryTimeouts: method?.retryTimeouts ?? false,
|
|
451
|
+
};
|
|
452
|
+
};
|
|
453
|
+
const serializeError = (error) => {
|
|
454
|
+
let name = 'Error';
|
|
455
|
+
let message = 'Unknown thrown value';
|
|
456
|
+
try {
|
|
457
|
+
if (error === null) {
|
|
458
|
+
return { name, message: 'null' };
|
|
459
|
+
}
|
|
460
|
+
if (typeof error !== 'object' && typeof error !== 'function') {
|
|
461
|
+
return { name, message: String(error) };
|
|
462
|
+
}
|
|
463
|
+
try {
|
|
464
|
+
const candidate = Reflect.get(error, 'name');
|
|
465
|
+
if (typeof candidate === 'string' && candidate.length > 0) {
|
|
466
|
+
name = candidate;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
name = 'Error';
|
|
471
|
+
}
|
|
472
|
+
try {
|
|
473
|
+
const candidate = Reflect.get(error, 'message');
|
|
474
|
+
if (typeof candidate === 'string' && candidate.length > 0) {
|
|
475
|
+
message = candidate;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
catch {
|
|
479
|
+
message = 'Unknown thrown value';
|
|
480
|
+
}
|
|
481
|
+
return { name, message };
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
return { name: 'Error', message: 'Unknown thrown value' };
|
|
485
|
+
}
|
|
486
|
+
};
|
|
487
|
+
const isErrorInstance = (error, constructor) => {
|
|
488
|
+
try {
|
|
489
|
+
return error instanceof constructor;
|
|
490
|
+
}
|
|
491
|
+
catch {
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
};
|
|
257
495
|
const isNonRetryable = (error) => {
|
|
258
|
-
if (error
|
|
496
|
+
if (isErrorInstance(error, NonRetryableError)) {
|
|
259
497
|
return true;
|
|
260
498
|
}
|
|
261
|
-
if (typeof error !== 'object'
|
|
499
|
+
if ((typeof error !== 'object' && typeof error !== 'function') ||
|
|
500
|
+
error === null) {
|
|
262
501
|
return false;
|
|
263
502
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
503
|
+
try {
|
|
504
|
+
if (Reflect.get(error, 'name') === 'NonRetryableError') {
|
|
505
|
+
return true;
|
|
506
|
+
}
|
|
507
|
+
const constructor = Reflect.get(error, 'constructor');
|
|
508
|
+
return (constructor !== null &&
|
|
509
|
+
(typeof constructor === 'object' ||
|
|
510
|
+
typeof constructor === 'function') &&
|
|
511
|
+
Reflect.get(constructor, 'name') === 'NonRetryableError');
|
|
512
|
+
}
|
|
513
|
+
catch {
|
|
514
|
+
return false;
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
const reportFailure = (event, details, error) => {
|
|
518
|
+
try {
|
|
519
|
+
const normalized = serializeError(error);
|
|
520
|
+
let stack;
|
|
521
|
+
try {
|
|
522
|
+
const candidate = error !== null &&
|
|
523
|
+
(typeof error === 'object' || typeof error === 'function')
|
|
524
|
+
? Reflect.get(error, 'stack')
|
|
525
|
+
: undefined;
|
|
526
|
+
stack = typeof candidate === 'string' ? candidate : undefined;
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
stack = undefined;
|
|
530
|
+
}
|
|
531
|
+
console.error({
|
|
532
|
+
event,
|
|
533
|
+
...details,
|
|
534
|
+
error: { ...normalized, ...(stack ? { stack } : {}) },
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
catch {
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
const emit = (event) => {
|
|
542
|
+
if (!options.onLifecycleEvent) {
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
const report = (error) => reportFailure('durability.lifecycle_hook.failed', {
|
|
546
|
+
entityKind: event.entityKind,
|
|
547
|
+
lifecycleType: event.type,
|
|
548
|
+
entityId: 'id' in event ? event.id : 'durability',
|
|
549
|
+
}, error);
|
|
550
|
+
try {
|
|
551
|
+
const result = options.onLifecycleEvent(event);
|
|
552
|
+
if (result === undefined) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
const caught = Promise.resolve(result).catch(report);
|
|
556
|
+
if (context.waitUntil) {
|
|
557
|
+
try {
|
|
558
|
+
context.waitUntil(caught);
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
catch (error) {
|
|
562
|
+
report(error);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
void caught;
|
|
566
|
+
}
|
|
567
|
+
catch (error) {
|
|
568
|
+
report(error);
|
|
569
|
+
}
|
|
267
570
|
};
|
|
268
571
|
const serialize = (value) => JSON.stringify(value === undefined
|
|
269
572
|
? { kind: 'undefined' }
|
|
@@ -273,12 +576,12 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
273
576
|
const stored = storedValueSchema.parse(parsed);
|
|
274
577
|
return stored.kind === 'undefined' ? undefined : stored.value;
|
|
275
578
|
};
|
|
276
|
-
const getCall = (id) => qb
|
|
579
|
+
const getCall = (id) => callRowSchema.optional().parse(qb
|
|
277
580
|
.fetchOne({
|
|
278
581
|
tableName,
|
|
279
582
|
where: { conditions: 'id = ?', params: [id] },
|
|
280
583
|
})
|
|
281
|
-
.execute().results;
|
|
584
|
+
.execute().results);
|
|
282
585
|
const getResult = (operation, idempotencyKey) => {
|
|
283
586
|
const call = getCall(idempotencyKey);
|
|
284
587
|
if (!call) {
|
|
@@ -308,7 +611,9 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
308
611
|
}
|
|
309
612
|
return { status: 'completed', result: deserialize(call.result) };
|
|
310
613
|
};
|
|
311
|
-
const listDueIds = (now, limit) =>
|
|
614
|
+
const listDueIds = (now, limit) => callIdSchema
|
|
615
|
+
.array()
|
|
616
|
+
.parse(qb
|
|
312
617
|
.fetchAll({
|
|
313
618
|
tableName,
|
|
314
619
|
fields: 'id',
|
|
@@ -319,15 +624,42 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
319
624
|
orderBy: 'next_attempt_at ASC',
|
|
320
625
|
limit,
|
|
321
626
|
})
|
|
322
|
-
.execute().results ?? [])
|
|
323
|
-
|
|
627
|
+
.execute().results ?? [])
|
|
628
|
+
.map((call) => call.id);
|
|
629
|
+
const getNextPendingAt = () => callNextAttemptSchema.optional().parse(qb
|
|
324
630
|
.fetchOne({
|
|
325
631
|
tableName,
|
|
326
632
|
fields: 'next_attempt_at',
|
|
327
633
|
where: { conditions: "status = 'pending'" },
|
|
328
634
|
orderBy: 'next_attempt_at ASC',
|
|
329
635
|
})
|
|
330
|
-
.execute().results?.next_attempt_at;
|
|
636
|
+
.execute().results)?.next_attempt_at;
|
|
637
|
+
const listDueAlarms = (now, limit) => alarmRowSchema.array().parse(qb
|
|
638
|
+
.fetchAll({
|
|
639
|
+
tableName: alarmTableName,
|
|
640
|
+
where: {
|
|
641
|
+
conditions: "status = 'pending' AND next_attempt_at <= ?",
|
|
642
|
+
params: [now],
|
|
643
|
+
},
|
|
644
|
+
orderBy: 'next_attempt_at ASC',
|
|
645
|
+
limit,
|
|
646
|
+
})
|
|
647
|
+
.execute().results ?? []);
|
|
648
|
+
const getNextNamedAlarmAt = () => alarmRowSchema
|
|
649
|
+
.pick({ next_attempt_at: true })
|
|
650
|
+
.optional()
|
|
651
|
+
.parse(qb
|
|
652
|
+
.fetchOne({
|
|
653
|
+
tableName: alarmTableName,
|
|
654
|
+
fields: 'next_attempt_at',
|
|
655
|
+
where: { conditions: "status = 'pending'" },
|
|
656
|
+
orderBy: 'next_attempt_at ASC',
|
|
657
|
+
})
|
|
658
|
+
.execute().results)?.next_attempt_at;
|
|
659
|
+
const getNextAlarmAt = () => {
|
|
660
|
+
const candidates = [getNextPendingAt(), getNextNamedAlarmAt()].filter((value) => value !== undefined);
|
|
661
|
+
return candidates.length === 0 ? undefined : Math.min(...candidates);
|
|
662
|
+
};
|
|
331
663
|
const runConcurrent = async (items, concurrency, worker) => {
|
|
332
664
|
let nextIndex = 0;
|
|
333
665
|
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
@@ -345,68 +677,424 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
345
677
|
throw new DuplicateDurableCallError(id, existingOperation, requestedOperation);
|
|
346
678
|
}
|
|
347
679
|
};
|
|
348
|
-
const
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
680
|
+
const reconcileAlarm = async (transaction) => {
|
|
681
|
+
const nextAlarmAt = getNextAlarmAt();
|
|
682
|
+
const currentAlarm = await transaction.getAlarm();
|
|
683
|
+
if (nextAlarmAt === undefined) {
|
|
684
|
+
if (currentAlarm !== null) {
|
|
685
|
+
await transaction.deleteAlarm();
|
|
686
|
+
}
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
const target = Math.max(nextAlarmAt, Date.now());
|
|
690
|
+
if (currentAlarm !== target) {
|
|
691
|
+
await transaction.setAlarm(target);
|
|
352
692
|
}
|
|
353
693
|
};
|
|
694
|
+
const armForHandoff = async (timestamp) => {
|
|
695
|
+
await context.storage.transaction(async (transaction) => {
|
|
696
|
+
const currentAlarm = await transaction.getAlarm();
|
|
697
|
+
if (currentAlarm === null || currentAlarm > timestamp) {
|
|
698
|
+
await transaction.setAlarm(timestamp);
|
|
699
|
+
}
|
|
700
|
+
});
|
|
701
|
+
};
|
|
354
702
|
let alarmRefresh;
|
|
355
703
|
const scheduleNextAlarm = () => {
|
|
356
704
|
if (alarmRefresh) {
|
|
357
705
|
return alarmRefresh;
|
|
358
706
|
}
|
|
359
|
-
alarmRefresh = context.storage
|
|
360
|
-
.transaction(async (transaction) => {
|
|
361
|
-
const nextAttemptAt = getNextPendingAt();
|
|
362
|
-
const currentAlarm = await transaction.getAlarm();
|
|
363
|
-
if (nextAttemptAt === undefined) {
|
|
364
|
-
if (currentAlarm !== null) {
|
|
365
|
-
await transaction.deleteAlarm();
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
else if (currentAlarm === null) {
|
|
369
|
-
await transaction.setAlarm(nextAttemptAt);
|
|
370
|
-
}
|
|
371
|
-
})
|
|
372
|
-
.finally(() => {
|
|
707
|
+
alarmRefresh = context.storage.transaction(reconcileAlarm).finally(() => {
|
|
373
708
|
alarmRefresh = undefined;
|
|
374
709
|
});
|
|
375
710
|
return alarmRefresh;
|
|
376
711
|
};
|
|
377
|
-
|
|
378
|
-
|
|
712
|
+
let availablePermits = alarmConcurrency;
|
|
713
|
+
const permitWaiters = [];
|
|
714
|
+
const createRelease = () => {
|
|
715
|
+
let released = false;
|
|
716
|
+
return () => {
|
|
717
|
+
if (released) {
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
released = true;
|
|
721
|
+
const waiter = permitWaiters.shift();
|
|
722
|
+
if (waiter) {
|
|
723
|
+
waiter(createRelease());
|
|
724
|
+
}
|
|
725
|
+
else {
|
|
726
|
+
availablePermits += 1;
|
|
727
|
+
}
|
|
728
|
+
};
|
|
729
|
+
};
|
|
730
|
+
const tryAcquirePermit = () => {
|
|
731
|
+
if (availablePermits === 0) {
|
|
732
|
+
return undefined;
|
|
733
|
+
}
|
|
734
|
+
availablePermits -= 1;
|
|
735
|
+
return createRelease();
|
|
736
|
+
};
|
|
737
|
+
const acquirePermit = () => {
|
|
738
|
+
const release = tryAcquirePermit();
|
|
739
|
+
if (release) {
|
|
740
|
+
return Promise.resolve(release);
|
|
741
|
+
}
|
|
742
|
+
return new Promise((resolve) => {
|
|
743
|
+
permitWaiters.push(resolve);
|
|
744
|
+
});
|
|
745
|
+
};
|
|
746
|
+
const operationEntity = (call) => ({
|
|
747
|
+
entityKind: 'operation',
|
|
748
|
+
operation: call.operation,
|
|
749
|
+
id: call.id,
|
|
750
|
+
generation: call.generation_id,
|
|
751
|
+
});
|
|
752
|
+
const alarmEntity = (row) => ({
|
|
753
|
+
entityKind: 'named_alarm',
|
|
754
|
+
alarm: row.name,
|
|
755
|
+
id: row.name,
|
|
756
|
+
generation: row.generation_id,
|
|
757
|
+
});
|
|
758
|
+
const scheduleNamedAlarm = async (name, scheduledTime) => {
|
|
759
|
+
if (!Number.isSafeInteger(scheduledTime) || scheduledTime < 0) {
|
|
760
|
+
throw new RangeError('scheduledTime must be a non-negative safe integer');
|
|
761
|
+
}
|
|
762
|
+
const generation = crypto.randomUUID();
|
|
763
|
+
const createdAt = Date.now();
|
|
764
|
+
await context.storage.transaction(async (transaction) => {
|
|
765
|
+
context.storage.sql.exec(`INSERT INTO durability_alarms
|
|
766
|
+
(name, generation_id, status, scheduled_at, next_attempt_at, attempt,
|
|
767
|
+
created_at)
|
|
768
|
+
VALUES (?, ?, 'pending', ?, ?, 0, ?)
|
|
769
|
+
ON CONFLICT(name) DO UPDATE SET
|
|
770
|
+
generation_id = excluded.generation_id,
|
|
771
|
+
status = 'pending',
|
|
772
|
+
scheduled_at = excluded.scheduled_at,
|
|
773
|
+
next_attempt_at = excluded.next_attempt_at,
|
|
774
|
+
attempt = 0,
|
|
775
|
+
last_error = NULL,
|
|
776
|
+
last_error_name = NULL,
|
|
777
|
+
created_at = excluded.created_at`, name, generation, scheduledTime, scheduledTime, createdAt);
|
|
778
|
+
await reconcileAlarm(transaction);
|
|
779
|
+
});
|
|
780
|
+
emit({
|
|
781
|
+
type: 'scheduled',
|
|
782
|
+
entityKind: 'named_alarm',
|
|
783
|
+
alarm: name,
|
|
784
|
+
id: name,
|
|
785
|
+
generation,
|
|
786
|
+
timestamp: createdAt,
|
|
787
|
+
attempt: 0,
|
|
788
|
+
scheduledTime,
|
|
789
|
+
});
|
|
790
|
+
};
|
|
791
|
+
const executeNamedAlarm = (row, platform) => {
|
|
792
|
+
const running = activeAlarms.get(row.name);
|
|
379
793
|
if (running) {
|
|
380
|
-
return running;
|
|
794
|
+
return running.settled;
|
|
381
795
|
}
|
|
796
|
+
const policy = alarmExecutionPolicy(row.name);
|
|
797
|
+
let settleActive;
|
|
798
|
+
const settled = new Promise((resolve) => {
|
|
799
|
+
settleActive = resolve;
|
|
800
|
+
});
|
|
801
|
+
const activeExecution = {
|
|
802
|
+
generation: row.generation_id,
|
|
803
|
+
createdAt: row.created_at,
|
|
804
|
+
settled,
|
|
805
|
+
};
|
|
806
|
+
activeAlarms.set(row.name, activeExecution);
|
|
807
|
+
let handlerStarted = false;
|
|
808
|
+
let handlerSettled = false;
|
|
809
|
+
let policySettled = false;
|
|
810
|
+
let releasePermit;
|
|
811
|
+
const releaseIfSettled = () => {
|
|
812
|
+
if (policySettled && (!handlerStarted || handlerSettled)) {
|
|
813
|
+
releasePermit?.();
|
|
814
|
+
settleActive?.();
|
|
815
|
+
}
|
|
816
|
+
};
|
|
382
817
|
const execution = (async () => {
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
818
|
+
releasePermit = await acquirePermit();
|
|
819
|
+
const claimed = alarmAttemptSchema.optional().parse(context.storage.sql
|
|
820
|
+
.exec(`UPDATE durability_alarms
|
|
821
|
+
SET attempt = attempt + 1
|
|
822
|
+
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
823
|
+
AND attempt < ?
|
|
824
|
+
RETURNING attempt`, row.name, row.generation_id, policy.maxAttempts)
|
|
825
|
+
.toArray()[0]);
|
|
826
|
+
if (!claimed) {
|
|
827
|
+
const current = alarmRowSchema.optional().parse(qb
|
|
828
|
+
.fetchOne({
|
|
829
|
+
tableName: alarmTableName,
|
|
830
|
+
where: { conditions: 'name = ?', params: [row.name] },
|
|
831
|
+
})
|
|
832
|
+
.execute().results);
|
|
833
|
+
if (current?.generation_id === row.generation_id &&
|
|
834
|
+
current.attempt >= policy.maxAttempts) {
|
|
835
|
+
const exhausted = serializeError(new DurableAttemptsExhaustedError('named alarm', row.name, policy.maxAttempts));
|
|
836
|
+
let updated = false;
|
|
837
|
+
await context.storage.transaction(async (transaction) => {
|
|
838
|
+
updated =
|
|
839
|
+
context.storage.sql
|
|
840
|
+
.exec(`UPDATE durability_alarms
|
|
841
|
+
SET status = 'failed', last_error = ?, last_error_name = ?
|
|
842
|
+
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
843
|
+
AND attempt >= ?
|
|
844
|
+
RETURNING name`, exhausted.message, exhausted.name, row.name, row.generation_id, policy.maxAttempts)
|
|
845
|
+
.toArray().length > 0;
|
|
846
|
+
await reconcileAlarm(transaction);
|
|
847
|
+
});
|
|
848
|
+
if (updated) {
|
|
849
|
+
emit({
|
|
850
|
+
...alarmEntity(current),
|
|
851
|
+
type: 'terminal',
|
|
852
|
+
timestamp: Date.now(),
|
|
853
|
+
attempt: current.attempt,
|
|
854
|
+
reason: 'attempts_exhausted',
|
|
855
|
+
error: exhausted,
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
const attempt = claimed.attempt;
|
|
862
|
+
const startedAt = Date.now();
|
|
863
|
+
const entity = alarmEntity(row);
|
|
864
|
+
emit({
|
|
865
|
+
...entity,
|
|
866
|
+
type: 'attempt_started',
|
|
867
|
+
timestamp: startedAt,
|
|
868
|
+
attempt,
|
|
869
|
+
});
|
|
870
|
+
const controller = new AbortController();
|
|
871
|
+
activeExecution.controller = controller;
|
|
872
|
+
let timeout;
|
|
873
|
+
try {
|
|
874
|
+
const handler = alarmHandlers.get(row.name);
|
|
875
|
+
if (!handler) {
|
|
876
|
+
throw new NonRetryableError(`No named alarm handler registered for "${row.name}"`);
|
|
877
|
+
}
|
|
878
|
+
handlerStarted = true;
|
|
879
|
+
const handlerResult = Promise.resolve().then(() => handler({
|
|
880
|
+
name: row.name,
|
|
881
|
+
scheduledTime: row.scheduled_at,
|
|
882
|
+
attempt,
|
|
883
|
+
isRetry: attempt > 1,
|
|
884
|
+
retryCount: attempt - 1,
|
|
885
|
+
idempotencyKey: `durability-alarm:v1:${row.generation_id}`,
|
|
886
|
+
signal: controller.signal,
|
|
887
|
+
platform,
|
|
888
|
+
}));
|
|
889
|
+
void handlerResult
|
|
890
|
+
.finally(() => {
|
|
891
|
+
handlerSettled = true;
|
|
892
|
+
releaseIfSettled();
|
|
893
|
+
})
|
|
894
|
+
.catch(() => undefined);
|
|
895
|
+
const timeoutResult = new Promise((_resolve, reject) => {
|
|
896
|
+
timeout = setTimeout(() => {
|
|
897
|
+
const error = new DurableAlarmTimeoutError(row.name, policy.attemptTimeoutMs);
|
|
898
|
+
controller.abort(error);
|
|
899
|
+
reject(error);
|
|
900
|
+
}, policy.attemptTimeoutMs);
|
|
901
|
+
});
|
|
902
|
+
await Promise.race([handlerResult, timeoutResult]);
|
|
903
|
+
const timestamp = Date.now();
|
|
904
|
+
const deleted = context.storage.sql
|
|
905
|
+
.exec(`DELETE FROM durability_alarms
|
|
906
|
+
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
907
|
+
AND attempt = ?
|
|
908
|
+
RETURNING name`, row.name, row.generation_id, attempt)
|
|
909
|
+
.toArray().length > 0;
|
|
910
|
+
if (deleted) {
|
|
911
|
+
emit({
|
|
912
|
+
...entity,
|
|
913
|
+
type: 'attempt_settled',
|
|
914
|
+
timestamp,
|
|
915
|
+
attempt,
|
|
916
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
917
|
+
outcome: 'completed',
|
|
918
|
+
});
|
|
919
|
+
}
|
|
386
920
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
921
|
+
catch (caught) {
|
|
922
|
+
let error = caught;
|
|
923
|
+
let terminal = isNonRetryable(error) ||
|
|
924
|
+
attempt >= policy.maxAttempts ||
|
|
925
|
+
(isErrorInstance(error, DurableAlarmTimeoutError) &&
|
|
926
|
+
!policy.retryTimeouts);
|
|
927
|
+
let timestamp = Date.now();
|
|
928
|
+
let delay = 0;
|
|
929
|
+
if (!terminal) {
|
|
930
|
+
try {
|
|
931
|
+
const rawDelay = policy.delay(attempt);
|
|
932
|
+
if (!Number.isFinite(rawDelay) || rawDelay < 0) {
|
|
933
|
+
throw new DurableRetryPolicyError(`named alarm "${row.name}"`);
|
|
934
|
+
}
|
|
935
|
+
delay = Math.round(rawDelay);
|
|
936
|
+
timestamp = Date.now();
|
|
937
|
+
if (!Number.isSafeInteger(delay) ||
|
|
938
|
+
!Number.isSafeInteger(timestamp + delay)) {
|
|
939
|
+
throw new DurableRetryPolicyError(`named alarm "${row.name}"`);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
catch (policyError) {
|
|
943
|
+
error = isErrorInstance(policyError, DurableRetryPolicyError)
|
|
944
|
+
? policyError
|
|
945
|
+
: new DurableRetryPolicyError(`named alarm "${row.name}"`, policyError);
|
|
946
|
+
delay = 0;
|
|
947
|
+
terminal = true;
|
|
948
|
+
}
|
|
390
949
|
}
|
|
391
|
-
|
|
950
|
+
const nextAttemptAt = timestamp + delay;
|
|
951
|
+
const serializedError = serializeError(error);
|
|
952
|
+
let updated = false;
|
|
953
|
+
await context.storage.transaction(async (transaction) => {
|
|
954
|
+
updated =
|
|
955
|
+
context.storage.sql
|
|
956
|
+
.exec(`UPDATE durability_alarms
|
|
957
|
+
SET status = ?, next_attempt_at = ?, last_error = ?,
|
|
958
|
+
last_error_name = ?
|
|
959
|
+
WHERE name = ? AND generation_id = ? AND status = 'pending'
|
|
960
|
+
AND attempt = ?
|
|
961
|
+
RETURNING name`, terminal ? 'failed' : 'pending', nextAttemptAt, serializedError.message, serializedError.name, row.name, row.generation_id, attempt)
|
|
962
|
+
.toArray().length > 0;
|
|
963
|
+
await reconcileAlarm(transaction);
|
|
964
|
+
});
|
|
965
|
+
if (updated) {
|
|
966
|
+
emit(terminal
|
|
967
|
+
? {
|
|
968
|
+
...entity,
|
|
969
|
+
type: 'attempt_settled',
|
|
970
|
+
timestamp,
|
|
971
|
+
attempt,
|
|
972
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
973
|
+
outcome: 'failed',
|
|
974
|
+
error: serializedError,
|
|
975
|
+
}
|
|
976
|
+
: {
|
|
977
|
+
...entity,
|
|
978
|
+
type: 'attempt_settled',
|
|
979
|
+
timestamp,
|
|
980
|
+
attempt,
|
|
981
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
982
|
+
outcome: 'retry_scheduled',
|
|
983
|
+
error: serializedError,
|
|
984
|
+
nextAttemptAt,
|
|
985
|
+
});
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
finally {
|
|
989
|
+
if (timeout !== undefined) {
|
|
990
|
+
clearTimeout(timeout);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
})().finally(() => {
|
|
994
|
+
policySettled = true;
|
|
995
|
+
releaseIfSettled();
|
|
996
|
+
});
|
|
997
|
+
void settled.finally(() => {
|
|
998
|
+
if (activeAlarms.get(row.name) === activeExecution) {
|
|
999
|
+
activeAlarms.delete(row.name);
|
|
392
1000
|
}
|
|
393
|
-
|
|
1001
|
+
});
|
|
1002
|
+
return execution;
|
|
1003
|
+
};
|
|
1004
|
+
const execute = (id, permitMode = 'wait') => {
|
|
1005
|
+
const call = getCall(id);
|
|
1006
|
+
if (!call || call.status !== 'pending') {
|
|
1007
|
+
return permitMode === 'wait' ? Promise.resolve() : undefined;
|
|
1008
|
+
}
|
|
1009
|
+
const running = active.get(id);
|
|
1010
|
+
if (running?.generation === call.generation_id) {
|
|
1011
|
+
return permitMode === 'wait' ? running.settled : undefined;
|
|
1012
|
+
}
|
|
1013
|
+
let releasePermit;
|
|
1014
|
+
if (permitMode === 'immediate') {
|
|
1015
|
+
releasePermit = tryAcquirePermit();
|
|
1016
|
+
if (!releasePermit) {
|
|
394
1017
|
return undefined;
|
|
395
1018
|
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
1019
|
+
}
|
|
1020
|
+
const policy = executionPolicy(call.operation);
|
|
1021
|
+
let settleActive;
|
|
1022
|
+
const settled = new Promise((resolve) => {
|
|
1023
|
+
settleActive = resolve;
|
|
1024
|
+
});
|
|
1025
|
+
const activeExecution = {
|
|
1026
|
+
generation: call.generation_id,
|
|
1027
|
+
createdAt: call.created_at,
|
|
1028
|
+
settled,
|
|
1029
|
+
};
|
|
1030
|
+
active.set(id, activeExecution);
|
|
1031
|
+
let handlerStarted = false;
|
|
1032
|
+
let handlerSettled = false;
|
|
1033
|
+
let policySettled = false;
|
|
1034
|
+
const releaseIfSettled = () => {
|
|
1035
|
+
if (policySettled && (!handlerStarted || handlerSettled)) {
|
|
1036
|
+
releasePermit?.();
|
|
1037
|
+
settleActive?.();
|
|
1038
|
+
}
|
|
1039
|
+
};
|
|
1040
|
+
const execution = (async () => {
|
|
1041
|
+
releasePermit ?? (releasePermit = await acquirePermit());
|
|
1042
|
+
const claimed = callAttemptSchema.optional().parse(context.storage.sql
|
|
1043
|
+
.exec(`UPDATE durability_calls
|
|
1044
|
+
SET attempt = attempt + 1
|
|
1045
|
+
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1046
|
+
AND attempt < ?
|
|
1047
|
+
RETURNING attempt`, id, call.generation_id, policy.maxAttempts)
|
|
1048
|
+
.toArray()[0]);
|
|
1049
|
+
if (!claimed) {
|
|
1050
|
+
const current = getCall(id);
|
|
1051
|
+
if (current?.generation_id === call.generation_id &&
|
|
1052
|
+
current.status === 'pending' &&
|
|
1053
|
+
current.attempt >= policy.maxAttempts) {
|
|
1054
|
+
const exhausted = serializeError(new DurableAttemptsExhaustedError('operation', call.operation, policy.maxAttempts));
|
|
1055
|
+
let updated = false;
|
|
1056
|
+
await context.storage.transaction(async (transaction) => {
|
|
1057
|
+
updated =
|
|
1058
|
+
context.storage.sql
|
|
1059
|
+
.exec(`UPDATE durability_calls
|
|
1060
|
+
SET status = 'failed', last_error = ?, last_error_name = ?
|
|
1061
|
+
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1062
|
+
AND attempt >= ?
|
|
1063
|
+
RETURNING id`, exhausted.message, exhausted.name, id, call.generation_id, policy.maxAttempts)
|
|
1064
|
+
.toArray().length > 0;
|
|
1065
|
+
await reconcileAlarm(transaction);
|
|
1066
|
+
});
|
|
1067
|
+
if (updated) {
|
|
1068
|
+
emit({
|
|
1069
|
+
...operationEntity(current),
|
|
1070
|
+
type: 'terminal',
|
|
1071
|
+
timestamp: Date.now(),
|
|
1072
|
+
attempt: current.attempt,
|
|
1073
|
+
reason: 'attempts_exhausted',
|
|
1074
|
+
error: exhausted,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
const attempt = claimed.attempt;
|
|
1081
|
+
const startedAt = Date.now();
|
|
1082
|
+
const entity = operationEntity(call);
|
|
1083
|
+
emit({
|
|
1084
|
+
...entity,
|
|
1085
|
+
type: 'attempt_started',
|
|
1086
|
+
timestamp: startedAt,
|
|
1087
|
+
attempt,
|
|
1088
|
+
});
|
|
1089
|
+
const controller = new AbortController();
|
|
1090
|
+
activeExecution.controller = controller;
|
|
404
1091
|
let timeout;
|
|
405
1092
|
try {
|
|
1093
|
+
const handler = handlers[call.operation];
|
|
406
1094
|
if (!handler) {
|
|
407
1095
|
throw new NonRetryableError(`No handler registered for operation "${call.operation}"`);
|
|
408
1096
|
}
|
|
409
|
-
|
|
1097
|
+
handlerStarted = true;
|
|
410
1098
|
const handlerResult = Promise.resolve().then(() => handler({
|
|
411
1099
|
id,
|
|
412
1100
|
operation: call.operation,
|
|
@@ -414,6 +1102,12 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
414
1102
|
attempt,
|
|
415
1103
|
signal: controller.signal,
|
|
416
1104
|
}));
|
|
1105
|
+
void handlerResult
|
|
1106
|
+
.finally(() => {
|
|
1107
|
+
handlerSettled = true;
|
|
1108
|
+
releaseIfSettled();
|
|
1109
|
+
})
|
|
1110
|
+
.catch(() => undefined);
|
|
417
1111
|
const timeoutResult = new Promise((_resolve, reject) => {
|
|
418
1112
|
timeout = setTimeout(() => {
|
|
419
1113
|
const error = new DurableAttemptTimeoutError(call.operation, policy.attemptTimeoutMs);
|
|
@@ -422,97 +1116,217 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
422
1116
|
}, policy.attemptTimeoutMs);
|
|
423
1117
|
});
|
|
424
1118
|
const result = await Promise.race([handlerResult, timeoutResult]);
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
1119
|
+
let serialized;
|
|
1120
|
+
try {
|
|
1121
|
+
serialized = serialize(result);
|
|
1122
|
+
}
|
|
1123
|
+
catch (error) {
|
|
1124
|
+
throw new DurableResultSerializationError(call.operation, error);
|
|
1125
|
+
}
|
|
1126
|
+
const timestamp = Date.now();
|
|
1127
|
+
const updated = context.storage.sql
|
|
1128
|
+
.exec(`UPDATE durability_calls
|
|
1129
|
+
SET status = 'completed', result = ?, last_error = NULL,
|
|
1130
|
+
last_error_name = NULL, completed_at = ?
|
|
1131
|
+
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1132
|
+
AND attempt = ?
|
|
1133
|
+
RETURNING id`, serialized, timestamp, id, call.generation_id, attempt)
|
|
1134
|
+
.toArray().length > 0;
|
|
1135
|
+
if (updated) {
|
|
1136
|
+
emit({
|
|
1137
|
+
...entity,
|
|
1138
|
+
type: 'attempt_settled',
|
|
1139
|
+
timestamp,
|
|
1140
|
+
attempt,
|
|
1141
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
1142
|
+
outcome: 'completed',
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
437
1145
|
}
|
|
438
|
-
catch (
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
1146
|
+
catch (caught) {
|
|
1147
|
+
let error = caught;
|
|
1148
|
+
let terminal = isNonRetryable(error) ||
|
|
1149
|
+
isErrorInstance(error, DurableResultSerializationError) ||
|
|
1150
|
+
attempt >= policy.maxAttempts;
|
|
1151
|
+
let timestamp = Date.now();
|
|
1152
|
+
let delay = 0;
|
|
1153
|
+
if (!terminal) {
|
|
1154
|
+
try {
|
|
1155
|
+
const rawDelay = policy.delay(attempt);
|
|
1156
|
+
if (!Number.isFinite(rawDelay) || rawDelay < 0) {
|
|
1157
|
+
throw new DurableRetryPolicyError(`operation "${call.operation}"`);
|
|
1158
|
+
}
|
|
1159
|
+
delay = Math.round(rawDelay);
|
|
1160
|
+
timestamp = Date.now();
|
|
1161
|
+
if (!Number.isSafeInteger(delay) ||
|
|
1162
|
+
!Number.isSafeInteger(timestamp + delay)) {
|
|
1163
|
+
throw new DurableRetryPolicyError(`operation "${call.operation}"`);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
catch (policyError) {
|
|
1167
|
+
error = isErrorInstance(policyError, DurableRetryPolicyError)
|
|
1168
|
+
? policyError
|
|
1169
|
+
: new DurableRetryPolicyError(`operation "${call.operation}"`, policyError);
|
|
1170
|
+
delay = 0;
|
|
1171
|
+
terminal = true;
|
|
1172
|
+
}
|
|
443
1173
|
}
|
|
444
|
-
const nextAttemptAt =
|
|
1174
|
+
const nextAttemptAt = timestamp + delay;
|
|
1175
|
+
const serializedError = serializeError(error);
|
|
1176
|
+
let updated = false;
|
|
445
1177
|
await context.storage.transaction(async (transaction) => {
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
}).execute();
|
|
457
|
-
if (!terminal && (await transaction.getAlarm()) === null) {
|
|
458
|
-
await transaction.setAlarm(nextAttemptAt);
|
|
459
|
-
}
|
|
1178
|
+
updated =
|
|
1179
|
+
context.storage.sql
|
|
1180
|
+
.exec(`UPDATE durability_calls
|
|
1181
|
+
SET status = ?, next_attempt_at = ?, last_error = ?,
|
|
1182
|
+
last_error_name = ?
|
|
1183
|
+
WHERE id = ? AND generation_id = ? AND status = 'pending'
|
|
1184
|
+
AND attempt = ?
|
|
1185
|
+
RETURNING id`, terminal ? 'failed' : 'pending', nextAttemptAt, serializedError.message, serializedError.name, id, call.generation_id, attempt)
|
|
1186
|
+
.toArray().length > 0;
|
|
1187
|
+
await reconcileAlarm(transaction);
|
|
460
1188
|
});
|
|
461
|
-
|
|
1189
|
+
if (updated) {
|
|
1190
|
+
emit(terminal
|
|
1191
|
+
? {
|
|
1192
|
+
...entity,
|
|
1193
|
+
type: 'attempt_settled',
|
|
1194
|
+
timestamp,
|
|
1195
|
+
attempt,
|
|
1196
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
1197
|
+
outcome: 'failed',
|
|
1198
|
+
error: serializedError,
|
|
1199
|
+
}
|
|
1200
|
+
: {
|
|
1201
|
+
...entity,
|
|
1202
|
+
type: 'attempt_settled',
|
|
1203
|
+
timestamp,
|
|
1204
|
+
attempt,
|
|
1205
|
+
durationMs: Math.max(0, timestamp - startedAt),
|
|
1206
|
+
outcome: 'retry_scheduled',
|
|
1207
|
+
error: serializedError,
|
|
1208
|
+
nextAttemptAt,
|
|
1209
|
+
});
|
|
1210
|
+
}
|
|
462
1211
|
}
|
|
463
1212
|
finally {
|
|
464
1213
|
if (timeout !== undefined) {
|
|
465
1214
|
clearTimeout(timeout);
|
|
466
1215
|
}
|
|
467
1216
|
}
|
|
468
|
-
})()
|
|
469
|
-
|
|
470
|
-
|
|
1217
|
+
})().finally(() => {
|
|
1218
|
+
policySettled = true;
|
|
1219
|
+
releaseIfSettled();
|
|
1220
|
+
});
|
|
1221
|
+
void settled.finally(() => {
|
|
1222
|
+
if (active.get(id) === activeExecution) {
|
|
1223
|
+
active.delete(id);
|
|
1224
|
+
}
|
|
1225
|
+
});
|
|
471
1226
|
return execution;
|
|
472
1227
|
};
|
|
473
1228
|
const run = async (input) => {
|
|
474
|
-
const
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
1229
|
+
const payload = serialize(input.payload);
|
|
1230
|
+
const now = Date.now();
|
|
1231
|
+
const generation = crypto.randomUUID();
|
|
1232
|
+
let inserted = false;
|
|
1233
|
+
let winner;
|
|
1234
|
+
await context.storage.transaction(async (transaction) => {
|
|
1235
|
+
inserted =
|
|
1236
|
+
context.storage.sql
|
|
1237
|
+
.exec(`INSERT INTO durability_calls
|
|
1238
|
+
(id, operation, payload, status, attempt, next_attempt_at,
|
|
1239
|
+
generation_id, created_at)
|
|
1240
|
+
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?)
|
|
1241
|
+
ON CONFLICT(id) DO NOTHING
|
|
1242
|
+
RETURNING id`, input.id, input.operation, payload, now, generation, now)
|
|
1243
|
+
.toArray().length > 0;
|
|
1244
|
+
winner = getCall(input.id);
|
|
1245
|
+
if (!winner) {
|
|
1246
|
+
throw new Error(`Durable call "${input.id}" was not persisted`);
|
|
479
1247
|
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
1248
|
+
assertOperation(input.id, winner.operation, input.operation);
|
|
1249
|
+
await reconcileAlarm(transaction);
|
|
1250
|
+
});
|
|
1251
|
+
if (inserted && winner) {
|
|
1252
|
+
emit({
|
|
1253
|
+
...operationEntity(winner),
|
|
1254
|
+
type: 'registered',
|
|
1255
|
+
timestamp: now,
|
|
1256
|
+
attempt: 0,
|
|
484
1257
|
});
|
|
485
1258
|
}
|
|
486
|
-
|
|
487
|
-
const
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
status: 'pending',
|
|
496
|
-
attempt: 0,
|
|
497
|
-
next_attempt_at: now,
|
|
498
|
-
},
|
|
499
|
-
}).execute();
|
|
500
|
-
if ((await transaction.getAlarm()) === null) {
|
|
501
|
-
await transaction.setAlarm(now);
|
|
502
|
-
}
|
|
1259
|
+
if (inserted && winner?.status === 'pending') {
|
|
1260
|
+
const eagerExecution = execute(input.id, 'immediate');
|
|
1261
|
+
if (!eagerExecution) {
|
|
1262
|
+
return;
|
|
1263
|
+
}
|
|
1264
|
+
const background = eagerExecution
|
|
1265
|
+
.then(scheduleNextAlarm)
|
|
1266
|
+
.catch((error) => {
|
|
1267
|
+
reportFailure('durability.background_execution.failed', { operation: input.operation, id: input.id }, error);
|
|
503
1268
|
});
|
|
1269
|
+
if (context.waitUntil) {
|
|
1270
|
+
try {
|
|
1271
|
+
context.waitUntil(background);
|
|
1272
|
+
}
|
|
1273
|
+
catch (error) {
|
|
1274
|
+
reportFailure('durability.background_execution.failed', { operation: input.operation, id: input.id }, error);
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
void background;
|
|
504
1278
|
}
|
|
505
|
-
const execution = execute(input.id);
|
|
506
|
-
void execution.then(() => scheduleNextAlarm()).catch(() => undefined);
|
|
507
1279
|
};
|
|
508
|
-
const
|
|
1280
|
+
const purgeBefore = async (before) => {
|
|
1281
|
+
if (!Number.isSafeInteger(before) || before < 0) {
|
|
1282
|
+
throw new RangeError('timestamp must be a non-negative safe integer');
|
|
1283
|
+
}
|
|
1284
|
+
let operations = 0;
|
|
1285
|
+
let namedAlarms = 0;
|
|
1286
|
+
await context.storage.transaction(async (transaction) => {
|
|
1287
|
+
operations = countSchema.parse(context.storage.sql
|
|
1288
|
+
.exec('SELECT COUNT(*) AS count FROM durability_calls WHERE created_at < ?', before)
|
|
1289
|
+
.toArray()[0]).count;
|
|
1290
|
+
namedAlarms = countSchema.parse(context.storage.sql
|
|
1291
|
+
.exec('SELECT COUNT(*) AS count FROM durability_alarms WHERE created_at < ?', before)
|
|
1292
|
+
.toArray()[0]).count;
|
|
1293
|
+
for (const running of active.values()) {
|
|
1294
|
+
if (running.createdAt < before) {
|
|
1295
|
+
running.controller?.abort(new Error('Durable operation was purged'));
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
for (const running of activeAlarms.values()) {
|
|
1299
|
+
if (running.createdAt < before) {
|
|
1300
|
+
running.controller?.abort(new Error('Durable alarm was purged'));
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
context.storage.sql.exec('DELETE FROM durability_calls WHERE created_at < ?', before);
|
|
1304
|
+
context.storage.sql.exec('DELETE FROM durability_alarms WHERE created_at < ?', before);
|
|
1305
|
+
await reconcileAlarm(transaction);
|
|
1306
|
+
});
|
|
1307
|
+
const result = {
|
|
1308
|
+
operations,
|
|
1309
|
+
namedAlarms,
|
|
1310
|
+
total: operations + namedAlarms,
|
|
1311
|
+
};
|
|
1312
|
+
emit({
|
|
1313
|
+
type: 'purged',
|
|
1314
|
+
entityKind: 'durability',
|
|
1315
|
+
timestamp: Date.now(),
|
|
1316
|
+
before,
|
|
1317
|
+
...result,
|
|
1318
|
+
});
|
|
1319
|
+
return result;
|
|
1320
|
+
};
|
|
1321
|
+
const alarm = async (alarmInfo) => {
|
|
509
1322
|
const startedAt = Date.now();
|
|
510
1323
|
const due = listDueIds(startedAt, 100);
|
|
511
|
-
|
|
1324
|
+
const dueAlarms = listDueAlarms(startedAt, 100);
|
|
1325
|
+
if (due.length === 0 && dueAlarms.length === 0) {
|
|
512
1326
|
await scheduleNextAlarm();
|
|
513
1327
|
return;
|
|
514
1328
|
}
|
|
515
|
-
const
|
|
1329
|
+
const operationExecution = runConcurrent(due, alarmConcurrency, async (id) => {
|
|
516
1330
|
try {
|
|
517
1331
|
await execute(id);
|
|
518
1332
|
}
|
|
@@ -520,32 +1334,87 @@ export const createDurability = (context, handlers, options = {}) => {
|
|
|
520
1334
|
return;
|
|
521
1335
|
}
|
|
522
1336
|
});
|
|
1337
|
+
const namedAlarmFailures = [];
|
|
1338
|
+
const namedAlarmExecution = runConcurrent(dueAlarms, alarmConcurrency, async (row) => {
|
|
1339
|
+
try {
|
|
1340
|
+
await executeNamedAlarm(row, alarmInfo);
|
|
1341
|
+
}
|
|
1342
|
+
catch (error) {
|
|
1343
|
+
namedAlarmFailures.push(error);
|
|
1344
|
+
}
|
|
1345
|
+
}).then(() => {
|
|
1346
|
+
if (namedAlarmFailures.length > 0) {
|
|
1347
|
+
throw namedAlarmFailures[0];
|
|
1348
|
+
}
|
|
1349
|
+
return undefined;
|
|
1350
|
+
});
|
|
1351
|
+
const execution = Promise.all([
|
|
1352
|
+
operationExecution,
|
|
1353
|
+
namedAlarmExecution,
|
|
1354
|
+
]).then(() => undefined);
|
|
1355
|
+
void execution.catch(() => undefined);
|
|
523
1356
|
let timeout;
|
|
524
1357
|
const handoff = Symbol('alarm handoff');
|
|
525
1358
|
const remaining = alarmHandoffMs - (Date.now() - startedAt);
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
1359
|
+
let result;
|
|
1360
|
+
try {
|
|
1361
|
+
result = await Promise.race([
|
|
1362
|
+
execution,
|
|
1363
|
+
new Promise((resolve) => {
|
|
1364
|
+
timeout = setTimeout(() => resolve(handoff), Math.max(remaining, 0));
|
|
1365
|
+
}),
|
|
1366
|
+
]);
|
|
1367
|
+
}
|
|
1368
|
+
finally {
|
|
1369
|
+
if (timeout !== undefined) {
|
|
1370
|
+
clearTimeout(timeout);
|
|
1371
|
+
}
|
|
534
1372
|
}
|
|
535
1373
|
if (result === handoff) {
|
|
536
|
-
await
|
|
1374
|
+
await armForHandoff(Date.now());
|
|
537
1375
|
return;
|
|
538
1376
|
}
|
|
539
1377
|
await scheduleNextAlarm();
|
|
540
1378
|
};
|
|
541
|
-
const
|
|
1379
|
+
for (const name of alarmHandlers.keys()) {
|
|
1380
|
+
if (name in alarm) {
|
|
1381
|
+
throw new Error(`"${name}" is reserved by durability alarms`);
|
|
1382
|
+
}
|
|
1383
|
+
Object.assign(alarm, {
|
|
1384
|
+
[name]: (scheduledTime) => scheduleNamedAlarm(name, scheduledTime),
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
const durability = { alarm, purgeBefore };
|
|
542
1388
|
for (const operation of Object.keys(handlers)) {
|
|
543
|
-
if (operation === 'alarm') {
|
|
1389
|
+
if (operation === 'alarm' || operation === 'purgeBefore') {
|
|
544
1390
|
throw new Error(`"${operation}" is reserved by durability`);
|
|
545
1391
|
}
|
|
546
1392
|
const durableOperation = (input) => run({ ...input, operation });
|
|
547
1393
|
durableOperation.getResult = async (idempotencyKey) => getResult(operation, idempotencyKey);
|
|
548
1394
|
durability[operation] = durableOperation;
|
|
549
1395
|
}
|
|
1396
|
+
const isCompleteDurability = (value) => {
|
|
1397
|
+
const alarmValue = value['alarm'];
|
|
1398
|
+
if (typeof alarmValue !== 'function' ||
|
|
1399
|
+
typeof value['purgeBefore'] !== 'function') {
|
|
1400
|
+
return false;
|
|
1401
|
+
}
|
|
1402
|
+
for (const name of alarmHandlers.keys()) {
|
|
1403
|
+
if (typeof Reflect.get(alarmValue, name) !== 'function') {
|
|
1404
|
+
return false;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
for (const operation of Object.keys(handlers)) {
|
|
1408
|
+
const method = value[operation];
|
|
1409
|
+
if (typeof method !== 'function' ||
|
|
1410
|
+
typeof Reflect.get(method, 'getResult') !== 'function') {
|
|
1411
|
+
return false;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
return true;
|
|
1415
|
+
};
|
|
1416
|
+
if (!isCompleteDurability(durability)) {
|
|
1417
|
+
throw new Error('Durability methods were not initialized');
|
|
1418
|
+
}
|
|
550
1419
|
return durability;
|
|
551
1420
|
};
|