durability 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +300 -116
- package/dist/durability-alarms.d.ts +90 -0
- package/dist/durability-alarms.js +165 -0
- package/dist/durability.d.ts +158 -0
- package/dist/durability.js +260 -0
- package/dist/errors.d.ts +83 -0
- package/dist/errors.js +184 -0
- package/dist/fanout.d.ts +61 -0
- package/dist/fanout.js +528 -0
- package/dist/index.d.ts +10 -435
- package/dist/index.js +7 -1420
- package/dist/kv-storage.d.ts +2 -0
- package/dist/kv-storage.js +421 -0
- package/dist/lifecycle.d.ts +70 -0
- package/dist/lifecycle.js +36 -0
- package/dist/load.d.ts +40 -0
- package/dist/load.js +68 -0
- package/dist/migrations.d.ts +50 -0
- package/dist/migrations.js +379 -0
- package/dist/policy.d.ts +74 -0
- package/dist/policy.js +49 -0
- package/dist/routing-client.d.ts +28 -0
- package/dist/routing-client.js +109 -0
- package/dist/routing.d.ts +33 -0
- package/dist/routing.js +58 -0
- package/dist/scheduler.d.ts +149 -0
- package/dist/scheduler.js +424 -0
- package/dist/serialization.d.ts +2 -0
- package/dist/serialization.js +13 -0
- package/dist/sqlite-storage.d.ts +2 -0
- package/dist/sqlite-storage.js +251 -0
- package/dist/storage.d.ts +134 -0
- package/dist/storage.js +69 -0
- package/dist/stored-message.d.ts +29 -0
- package/dist/stored-message.js +1 -0
- package/package.json +7 -3
package/dist/fanout.js
ADDED
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
2
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
3
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
4
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
5
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
6
|
+
};
|
|
7
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
|
+
};
|
|
12
|
+
var _DurabilityFanout_instances, _DurabilityFanout_engine, _DurabilityFanout_routing, _DurabilityFanout_targets, _DurabilityFanout_policies, _DurabilityFanout_dlq, _DurabilityFanout_maxBatchSize, _DurabilityFanout_emit, _DurabilityFanout_running, _DurabilityFanout_runDue, _DurabilityFanout_runTarget, _DurabilityFanout_postpone, _DurabilityFanout_deliver, _DurabilityFanout_message, _DurabilityFanout_remove, _DurabilityFanout_retry, _DurabilityFanout_deadLetter, _DurabilityFanout_deadline;
|
|
13
|
+
import { DurableRetryPolicyError, FanoutEnqueueError, FanoutSettlementError, FanoutTimeoutError, reportFailure, serializeError, } from './errors.js';
|
|
14
|
+
import { RoutingLoad } from './load.js';
|
|
15
|
+
import { durabilityFanoutMigrations, migrate } from './migrations.js';
|
|
16
|
+
import { assertPositiveInteger, createPolicyResolver, methodOptionsSchema, } from './policy.js';
|
|
17
|
+
import { engineFor, } from './scheduler.js';
|
|
18
|
+
import { deserialize, serialize } from './serialization.js';
|
|
19
|
+
const infrastructureRetryMs = 60000;
|
|
20
|
+
const entity = (claim) => ({
|
|
21
|
+
entityKind: 'fanout_delivery',
|
|
22
|
+
id: claim.message.id,
|
|
23
|
+
target: claim.row.target_id,
|
|
24
|
+
generation: claim.row.generation_id,
|
|
25
|
+
});
|
|
26
|
+
const deliveryIdentity = (row) => JSON.stringify([row.generation_id, row.target_id]);
|
|
27
|
+
/**
|
|
28
|
+
* A composable DO capability: durable, independent delivery to static named
|
|
29
|
+
* targets. Share a scheduler with operations/alarms on the same object.
|
|
30
|
+
* Register targets under stable IDs; removing a target pauses its pending work.
|
|
31
|
+
*/
|
|
32
|
+
export class DurabilityFanout {
|
|
33
|
+
constructor(config) {
|
|
34
|
+
_DurabilityFanout_instances.add(this);
|
|
35
|
+
_DurabilityFanout_engine.set(this, void 0);
|
|
36
|
+
_DurabilityFanout_routing.set(this, void 0);
|
|
37
|
+
_DurabilityFanout_targets.set(this, void 0);
|
|
38
|
+
_DurabilityFanout_policies.set(this, void 0);
|
|
39
|
+
_DurabilityFanout_dlq.set(this, void 0);
|
|
40
|
+
_DurabilityFanout_maxBatchSize.set(this, void 0);
|
|
41
|
+
_DurabilityFanout_emit.set(this, void 0);
|
|
42
|
+
_DurabilityFanout_running.set(this, new Map());
|
|
43
|
+
__classPrivateFieldSet(this, _DurabilityFanout_maxBatchSize, config.maxBatchSize ?? 10, "f");
|
|
44
|
+
assertPositiveInteger('maxBatchSize', __classPrivateFieldGet(this, _DurabilityFanout_maxBatchSize, "f"));
|
|
45
|
+
__classPrivateFieldSet(this, _DurabilityFanout_targets, new Map(Object.entries(config.targets).map(([id, target]) => [id, { ...target }])), "f");
|
|
46
|
+
if (__classPrivateFieldGet(this, _DurabilityFanout_targets, "f").size === 0) {
|
|
47
|
+
throw new TypeError('Fanout requires at least one static target');
|
|
48
|
+
}
|
|
49
|
+
__classPrivateFieldSet(this, _DurabilityFanout_policies, new Map(), "f");
|
|
50
|
+
for (const [id, target] of __classPrivateFieldGet(this, _DurabilityFanout_targets, "f")) {
|
|
51
|
+
if (!id ||
|
|
52
|
+
(typeof target.deliver !== 'function' &&
|
|
53
|
+
typeof target.storage !== 'function')) {
|
|
54
|
+
throw new TypeError(`Invalid fanout target "${id}"`);
|
|
55
|
+
}
|
|
56
|
+
__classPrivateFieldGet(this, _DurabilityFanout_policies, "f").set(id, createPolicyResolver(config, { [id]: target }, methodOptionsSchema)(id));
|
|
57
|
+
}
|
|
58
|
+
__classPrivateFieldSet(this, _DurabilityFanout_engine, engineFor(config), "f");
|
|
59
|
+
__classPrivateFieldSet(this, _DurabilityFanout_routing, config.routing ?? new RoutingLoad(), "f");
|
|
60
|
+
__classPrivateFieldSet(this, _DurabilityFanout_dlq, config.dlq, "f");
|
|
61
|
+
__classPrivateFieldSet(this, _DurabilityFanout_emit, __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").emitter(config.onLifecycleEvent), "f");
|
|
62
|
+
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").migrateSchema('fanout', durabilityFanoutMigrations);
|
|
63
|
+
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").register({ kind: 'deliveries', runDue: () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_runDue).call(this) });
|
|
64
|
+
}
|
|
65
|
+
static migrate(context, target) {
|
|
66
|
+
return migrate(context.storage, 'fanout', durabilityFanoutMigrations, target);
|
|
67
|
+
}
|
|
68
|
+
alarm(info) {
|
|
69
|
+
return __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").alarm(info);
|
|
70
|
+
}
|
|
71
|
+
async load() {
|
|
72
|
+
return __classPrivateFieldGet(this, _DurabilityFanout_routing, "f").snapshot(await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.deliveries.pendingCount());
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Atomically accepts the whole batch, including its frozen target set.
|
|
76
|
+
* Definite input conflicts return success:false. Storage/commit uncertainty
|
|
77
|
+
* throws: a transport failure is never represented as definite rejection.
|
|
78
|
+
* IDs deduplicate while their registration is retained, not forever.
|
|
79
|
+
*/
|
|
80
|
+
async enqueue(messages, options = {}) {
|
|
81
|
+
const finish = __classPrivateFieldGet(this, _DurabilityFanout_routing, "f").begin('inbound');
|
|
82
|
+
let entries;
|
|
83
|
+
let targets;
|
|
84
|
+
try {
|
|
85
|
+
targets = [...(options.targets ?? __classPrivateFieldGet(this, _DurabilityFanout_targets, "f").keys())].sort();
|
|
86
|
+
if (targets.length === 0 ||
|
|
87
|
+
new Set(targets).size !== targets.length ||
|
|
88
|
+
targets.some((id) => !__classPrivateFieldGet(this, _DurabilityFanout_targets, "f").has(id))) {
|
|
89
|
+
throw new FanoutEnqueueError('Enqueue targets must be distinct configured target IDs');
|
|
90
|
+
}
|
|
91
|
+
entries = (Array.isArray(messages) ? messages : [messages]).map((message) => {
|
|
92
|
+
const id = message.id ?? crypto.randomUUID();
|
|
93
|
+
const key = message.deduplicationKey ?? id;
|
|
94
|
+
if (typeof id !== 'string' ||
|
|
95
|
+
!id ||
|
|
96
|
+
typeof key !== 'string' ||
|
|
97
|
+
!key) {
|
|
98
|
+
throw new FanoutEnqueueError('Message IDs and deduplication keys must be non-empty strings');
|
|
99
|
+
}
|
|
100
|
+
return { key, id, payload: serialize(message.body) };
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
finish();
|
|
105
|
+
return {
|
|
106
|
+
success: false,
|
|
107
|
+
error: serializeError(error),
|
|
108
|
+
load: await this.load(),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const registered = [];
|
|
112
|
+
try {
|
|
113
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
114
|
+
registered.length = 0;
|
|
115
|
+
const store = transaction.deliveries;
|
|
116
|
+
for (const entry of entries) {
|
|
117
|
+
// eslint-disable-next-line no-await-in-loop
|
|
118
|
+
const existing = await store.getMessage(entry.key);
|
|
119
|
+
if (existing) {
|
|
120
|
+
if (existing.id !== entry.id ||
|
|
121
|
+
existing.payload !== entry.payload ||
|
|
122
|
+
existing.targets !== JSON.stringify(targets)) {
|
|
123
|
+
throw new FanoutEnqueueError(`Deduplication key "${entry.key}" already identifies different work`);
|
|
124
|
+
}
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const message = {
|
|
128
|
+
...entry,
|
|
129
|
+
targets: JSON.stringify(targets),
|
|
130
|
+
remaining: targets.length,
|
|
131
|
+
// eslint-disable-next-line no-await-in-loop
|
|
132
|
+
seq: await store.nextSeq(),
|
|
133
|
+
created_at: Date.now(),
|
|
134
|
+
generation_id: crypto.randomUUID(),
|
|
135
|
+
};
|
|
136
|
+
// eslint-disable-next-line no-await-in-loop
|
|
137
|
+
await store.insertMessage(message);
|
|
138
|
+
for (const target of targets) {
|
|
139
|
+
const row = {
|
|
140
|
+
id: JSON.stringify([entry.key, target]),
|
|
141
|
+
message_key: entry.key,
|
|
142
|
+
target_id: target,
|
|
143
|
+
seq: message.seq,
|
|
144
|
+
generation_id: message.generation_id,
|
|
145
|
+
created_at: message.created_at,
|
|
146
|
+
status: 'pending',
|
|
147
|
+
phase: 'delivery',
|
|
148
|
+
attempt: 0,
|
|
149
|
+
next_attempt_at: message.created_at,
|
|
150
|
+
last_error: null,
|
|
151
|
+
last_error_name: null,
|
|
152
|
+
dead_lettered_at: null,
|
|
153
|
+
dead_letter_reason: null,
|
|
154
|
+
};
|
|
155
|
+
// eslint-disable-next-line no-await-in-loop
|
|
156
|
+
if (!(await store.insert(row))) {
|
|
157
|
+
throw new Error('Fanout manifest and delivery state disagree');
|
|
158
|
+
}
|
|
159
|
+
registered.push({ row, message });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
if (!(error instanceof FanoutEnqueueError)) {
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
finish();
|
|
170
|
+
return {
|
|
171
|
+
success: false,
|
|
172
|
+
error: serializeError(error),
|
|
173
|
+
load: await this.load(),
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
finish();
|
|
178
|
+
}
|
|
179
|
+
for (const claim of registered) {
|
|
180
|
+
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
181
|
+
...entity(claim),
|
|
182
|
+
type: 'registered',
|
|
183
|
+
timestamp: claim.message.created_at,
|
|
184
|
+
attempt: 0,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const load = await this.load();
|
|
188
|
+
if (registered.length > 0) {
|
|
189
|
+
const onError = (error) => reportFailure('durability.fanout.failed', {}, error);
|
|
190
|
+
const work = __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_runDue).call(this)
|
|
191
|
+
.then(() => __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").scheduleNextAlarm())
|
|
192
|
+
.catch(onError);
|
|
193
|
+
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").waitUntil(work, onError);
|
|
194
|
+
}
|
|
195
|
+
return { success: true, value: entries.map(({ id }) => id), load };
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
_DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMap(), _DurabilityFanout_targets = new WeakMap(), _DurabilityFanout_policies = new WeakMap(), _DurabilityFanout_dlq = new WeakMap(), _DurabilityFanout_maxBatchSize = new WeakMap(), _DurabilityFanout_emit = new WeakMap(), _DurabilityFanout_running = new WeakMap(), _DurabilityFanout_instances = new WeakSet(), _DurabilityFanout_runDue = async function _DurabilityFanout_runDue() {
|
|
199
|
+
const targets = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.deliveries.listTargets();
|
|
200
|
+
// Each target advances independently; a missing or slow target cannot hide
|
|
201
|
+
// another target behind a global LIMIT. Each pass processes one bounded batch.
|
|
202
|
+
await Promise.all(targets.map((id) => {
|
|
203
|
+
const running = __classPrivateFieldGet(this, _DurabilityFanout_running, "f").get(id);
|
|
204
|
+
if (running) {
|
|
205
|
+
return running;
|
|
206
|
+
}
|
|
207
|
+
const work = __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_runTarget).call(this, id).finally(() => __classPrivateFieldGet(this, _DurabilityFanout_running, "f").delete(id));
|
|
208
|
+
__classPrivateFieldGet(this, _DurabilityFanout_running, "f").set(id, work);
|
|
209
|
+
return work;
|
|
210
|
+
}));
|
|
211
|
+
}, _DurabilityFanout_runTarget = async function _DurabilityFanout_runTarget(id) {
|
|
212
|
+
const release = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").acquirePermit();
|
|
213
|
+
try {
|
|
214
|
+
const due = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.deliveries.listDueForTarget(id, Date.now(), __classPrivateFieldGet(this, _DurabilityFanout_maxBatchSize, "f"));
|
|
215
|
+
const target = __classPrivateFieldGet(this, _DurabilityFanout_targets, "f").get(id);
|
|
216
|
+
const policy = __classPrivateFieldGet(this, _DurabilityFanout_policies, "f").get(id);
|
|
217
|
+
if (!target || !policy) {
|
|
218
|
+
if (due.length > 0) {
|
|
219
|
+
reportFailure('durability.fanout.target_missing', { target: id }, new Error(`Target "${id}" is no longer configured; retaining work`));
|
|
220
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_postpone).call(this, due);
|
|
221
|
+
}
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
const claims = [];
|
|
225
|
+
for (const row of due) {
|
|
226
|
+
// eslint-disable-next-line no-await-in-loop
|
|
227
|
+
const message = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.deliveries.getMessage(row.message_key);
|
|
228
|
+
if (!message || message.generation_id !== row.generation_id) {
|
|
229
|
+
throw new Error('Fanout delivery is missing its message manifest');
|
|
230
|
+
}
|
|
231
|
+
if (row.phase === 'dead_letter' || row.attempt >= policy.maxAttempts) {
|
|
232
|
+
// eslint-disable-next-line no-await-in-loop
|
|
233
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, { row, message }, row.dead_letter_reason ?? 'exhausted');
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
// eslint-disable-next-line no-await-in-loop
|
|
237
|
+
const attempt = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
238
|
+
const claimed = await transaction.deliveries.claimAttempt(row.id, row.generation_id, policy.maxAttempts);
|
|
239
|
+
if (claimed !== undefined) {
|
|
240
|
+
await transaction.deliveries.settle(row.id, row.generation_id, claimed, {
|
|
241
|
+
next_attempt_at: Date.now() + policy.attemptTimeoutMs,
|
|
242
|
+
});
|
|
243
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
244
|
+
}
|
|
245
|
+
return claimed;
|
|
246
|
+
});
|
|
247
|
+
if (attempt !== undefined) {
|
|
248
|
+
claims.push({ row: { ...row, attempt }, message });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (claims.length > 0) {
|
|
252
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deliver).call(this, target, claims, policy);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
finally {
|
|
256
|
+
release();
|
|
257
|
+
}
|
|
258
|
+
}, _DurabilityFanout_postpone = async function _DurabilityFanout_postpone(rows) {
|
|
259
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
// eslint-disable-next-line no-await-in-loop
|
|
262
|
+
await transaction.deliveries.settle(row.id, row.generation_id, row.attempt, {
|
|
263
|
+
next_attempt_at: Date.now() + infrastructureRetryMs,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
267
|
+
});
|
|
268
|
+
}, _DurabilityFanout_deliver = async function _DurabilityFanout_deliver(target, claims, policy) {
|
|
269
|
+
const startedAt = Date.now();
|
|
270
|
+
const finish = __classPrivateFieldGet(this, _DurabilityFanout_routing, "f").begin('outbound');
|
|
271
|
+
const states = claims.map((claim) => ({
|
|
272
|
+
claim,
|
|
273
|
+
closed: false,
|
|
274
|
+
}));
|
|
275
|
+
const messages = states.map((state) => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_message).call(this, state, policy, startedAt));
|
|
276
|
+
for (const claim of claims) {
|
|
277
|
+
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
278
|
+
...entity(claim),
|
|
279
|
+
type: 'attempt_started',
|
|
280
|
+
timestamp: startedAt,
|
|
281
|
+
attempt: claim.row.attempt,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
let failure = new Error('Consumer returned without settling the message');
|
|
285
|
+
try {
|
|
286
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadline).call(this, async () => {
|
|
287
|
+
let invocationFailed = false;
|
|
288
|
+
let invocationError;
|
|
289
|
+
try {
|
|
290
|
+
if (target.deliver) {
|
|
291
|
+
await target.deliver(messages);
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
const writes = await Promise.allSettled(messages.map(async (message) => {
|
|
295
|
+
await target.storage({
|
|
296
|
+
id: message.deliveryId,
|
|
297
|
+
messageId: message.id,
|
|
298
|
+
target: message.target,
|
|
299
|
+
body: message.body,
|
|
300
|
+
enqueuedAt: message.enqueuedAt,
|
|
301
|
+
storedAt: Date.now(),
|
|
302
|
+
attempts: message.attempt,
|
|
303
|
+
});
|
|
304
|
+
await message.ack();
|
|
305
|
+
}));
|
|
306
|
+
const rejected = writes.find((write) => write.status === 'rejected');
|
|
307
|
+
if (rejected?.status === 'rejected') {
|
|
308
|
+
throw rejected.reason;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
invocationFailed = true;
|
|
314
|
+
invocationError = error;
|
|
315
|
+
}
|
|
316
|
+
for (const state of states) {
|
|
317
|
+
state.closed = true;
|
|
318
|
+
}
|
|
319
|
+
// A callback failure must not bypass already-started settlements.
|
|
320
|
+
// The outer deadline still bounds a hung storage or RPC operation.
|
|
321
|
+
const settlements = await Promise.allSettled(states.map((state) => state.pending));
|
|
322
|
+
if (invocationFailed) {
|
|
323
|
+
throw invocationError;
|
|
324
|
+
}
|
|
325
|
+
const rejected = settlements.find((settlement) => settlement.status === 'rejected');
|
|
326
|
+
if (rejected?.status === 'rejected') {
|
|
327
|
+
throw rejected.reason;
|
|
328
|
+
}
|
|
329
|
+
}, policy.attemptTimeoutMs);
|
|
330
|
+
}
|
|
331
|
+
catch (error) {
|
|
332
|
+
failure = error;
|
|
333
|
+
}
|
|
334
|
+
finally {
|
|
335
|
+
for (const state of states) {
|
|
336
|
+
state.closed = true;
|
|
337
|
+
}
|
|
338
|
+
finish();
|
|
339
|
+
}
|
|
340
|
+
for (const state of states) {
|
|
341
|
+
if (state.completed) {
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
if (state.outcome !== undefined) {
|
|
345
|
+
// Pending actions retain their persisted claim deadline. Failed actions
|
|
346
|
+
// may have committed despite throwing; consult state before retrying.
|
|
347
|
+
if (!state.finished) {
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
// eslint-disable-next-line no-await-in-loop
|
|
351
|
+
const current = await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.deliveries.get(state.claim.row.id);
|
|
352
|
+
if (!current ||
|
|
353
|
+
current.generation_id !== state.claim.row.generation_id ||
|
|
354
|
+
current.attempt !== state.claim.row.attempt ||
|
|
355
|
+
current.phase === 'dead_letter') {
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
state.outcome = 'retry';
|
|
360
|
+
// eslint-disable-next-line no-await-in-loop
|
|
361
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_retry).call(this, state.claim, policy, undefined, serializeError(failure), startedAt);
|
|
362
|
+
}
|
|
363
|
+
}, _DurabilityFanout_message = function _DurabilityFanout_message(state, policy, startedAt) {
|
|
364
|
+
const { row, message } = state.claim;
|
|
365
|
+
const settle = (outcome, action) => {
|
|
366
|
+
if (state.closed || state.outcome !== undefined) {
|
|
367
|
+
return Promise.reject(new FanoutSettlementError(message.id, state.outcome));
|
|
368
|
+
}
|
|
369
|
+
state.outcome = outcome;
|
|
370
|
+
state.pending = action()
|
|
371
|
+
.then(() => {
|
|
372
|
+
state.completed = true;
|
|
373
|
+
return undefined;
|
|
374
|
+
})
|
|
375
|
+
.finally(() => {
|
|
376
|
+
state.finished = true;
|
|
377
|
+
});
|
|
378
|
+
void state.pending.catch(() => undefined);
|
|
379
|
+
return state.pending;
|
|
380
|
+
};
|
|
381
|
+
return {
|
|
382
|
+
id: message.id,
|
|
383
|
+
deliveryId: deliveryIdentity(row),
|
|
384
|
+
target: row.target_id,
|
|
385
|
+
body: deserialize(message.payload),
|
|
386
|
+
enqueuedAt: message.created_at,
|
|
387
|
+
attempt: row.attempt,
|
|
388
|
+
ack: () => settle('ack', async () => {
|
|
389
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_remove).call(this, row);
|
|
390
|
+
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
391
|
+
...entity(state.claim),
|
|
392
|
+
type: 'attempt_settled',
|
|
393
|
+
timestamp: Date.now(),
|
|
394
|
+
attempt: row.attempt,
|
|
395
|
+
durationMs: Math.max(0, Date.now() - startedAt),
|
|
396
|
+
outcome: 'completed',
|
|
397
|
+
});
|
|
398
|
+
}),
|
|
399
|
+
retry: (delayMs) => {
|
|
400
|
+
if (delayMs !== undefined &&
|
|
401
|
+
(!Number.isSafeInteger(delayMs) ||
|
|
402
|
+
delayMs < 0 ||
|
|
403
|
+
!Number.isSafeInteger(Date.now() + delayMs))) {
|
|
404
|
+
return Promise.reject(new RangeError('delayMs must produce a non-negative safe-integer timestamp'));
|
|
405
|
+
}
|
|
406
|
+
return settle('retry', () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_retry).call(this, state.claim, policy, delayMs, { name: 'Retry', message: 'Retried by consumer' }, startedAt));
|
|
407
|
+
},
|
|
408
|
+
deadLetter: () => settle('deadLetter', () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, state.claim, 'explicit')),
|
|
409
|
+
};
|
|
410
|
+
}, _DurabilityFanout_remove = async function _DurabilityFanout_remove(row) {
|
|
411
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
412
|
+
if (!(await transaction.deliveries.remove(row.id, row.generation_id, row.attempt))) {
|
|
413
|
+
throw new FanoutSettlementError(row.id);
|
|
414
|
+
}
|
|
415
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
416
|
+
});
|
|
417
|
+
}, _DurabilityFanout_retry = async function _DurabilityFanout_retry(claim, policy, override, error, startedAt) {
|
|
418
|
+
const { row } = claim;
|
|
419
|
+
if (row.attempt >= policy.maxAttempts) {
|
|
420
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, claim, 'exhausted', error);
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
let nextAttemptAt;
|
|
424
|
+
try {
|
|
425
|
+
const rawDelay = override ?? policy.delay(row.attempt);
|
|
426
|
+
const delay = Math.round(rawDelay);
|
|
427
|
+
nextAttemptAt = Date.now() + delay;
|
|
428
|
+
if (!Number.isFinite(rawDelay) ||
|
|
429
|
+
rawDelay < 0 ||
|
|
430
|
+
!Number.isSafeInteger(nextAttemptAt)) {
|
|
431
|
+
throw new DurableRetryPolicyError('fanout');
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
catch (caught) {
|
|
435
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, claim, 'exhausted', serializeError(caught));
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
439
|
+
const updated = await transaction.deliveries.settle(row.id, row.generation_id, row.attempt, {
|
|
440
|
+
next_attempt_at: nextAttemptAt,
|
|
441
|
+
last_error: error.message,
|
|
442
|
+
last_error_name: error.name,
|
|
443
|
+
});
|
|
444
|
+
if (!updated) {
|
|
445
|
+
throw new FanoutSettlementError(row.id);
|
|
446
|
+
}
|
|
447
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
448
|
+
});
|
|
449
|
+
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
450
|
+
...entity(claim),
|
|
451
|
+
type: 'attempt_settled',
|
|
452
|
+
timestamp: Date.now(),
|
|
453
|
+
attempt: row.attempt,
|
|
454
|
+
durationMs: Math.max(0, Date.now() - startedAt),
|
|
455
|
+
outcome: 'retry_scheduled',
|
|
456
|
+
nextAttemptAt,
|
|
457
|
+
error,
|
|
458
|
+
});
|
|
459
|
+
}, _DurabilityFanout_deadLetter = async function _DurabilityFanout_deadLetter(claim, reason, failure) {
|
|
460
|
+
const { row, message } = claim;
|
|
461
|
+
const timestamp = row.dead_lettered_at ?? Date.now();
|
|
462
|
+
const error = failure ??
|
|
463
|
+
(row.last_error === null
|
|
464
|
+
? null
|
|
465
|
+
: { name: row.last_error_name ?? 'Error', message: row.last_error });
|
|
466
|
+
// Persist intent BEFORE external I/O: eviction or timeout must never turn
|
|
467
|
+
// a terminal decision into another consumer delivery.
|
|
468
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").storage.transaction(async (transaction) => {
|
|
469
|
+
const updated = await transaction.deliveries.settle(row.id, row.generation_id, row.attempt, {
|
|
470
|
+
phase: 'dead_letter',
|
|
471
|
+
dead_lettered_at: timestamp,
|
|
472
|
+
dead_letter_reason: reason,
|
|
473
|
+
next_attempt_at: Date.now() + infrastructureRetryMs,
|
|
474
|
+
last_error: error?.message ?? null,
|
|
475
|
+
last_error_name: error?.name ?? null,
|
|
476
|
+
});
|
|
477
|
+
if (!updated) {
|
|
478
|
+
throw new FanoutSettlementError(row.id);
|
|
479
|
+
}
|
|
480
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").reconcile(transaction);
|
|
481
|
+
});
|
|
482
|
+
try {
|
|
483
|
+
if (!__classPrivateFieldGet(this, _DurabilityFanout_dlq, "f")) {
|
|
484
|
+
throw new Error('No DLQ configured; retaining terminal delivery');
|
|
485
|
+
}
|
|
486
|
+
const stored = {
|
|
487
|
+
id: deliveryIdentity(row),
|
|
488
|
+
messageId: message.id,
|
|
489
|
+
target: row.target_id,
|
|
490
|
+
body: deserialize(message.payload),
|
|
491
|
+
enqueuedAt: message.created_at,
|
|
492
|
+
storedAt: timestamp,
|
|
493
|
+
attempts: row.attempt,
|
|
494
|
+
failure: { reason, error },
|
|
495
|
+
};
|
|
496
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadline).call(this, () => __classPrivateFieldGet(this, _DurabilityFanout_dlq, "f")(stored), infrastructureRetryMs);
|
|
497
|
+
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_remove).call(this, row);
|
|
498
|
+
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
499
|
+
...entity(claim),
|
|
500
|
+
type: 'terminal',
|
|
501
|
+
timestamp: Date.now(),
|
|
502
|
+
attempt: row.attempt,
|
|
503
|
+
reason: reason === 'explicit' ? 'dead_lettered' : 'attempts_exhausted',
|
|
504
|
+
error: error ?? {
|
|
505
|
+
name: 'DeadLetter',
|
|
506
|
+
message: 'Dead-lettered by consumer',
|
|
507
|
+
},
|
|
508
|
+
});
|
|
509
|
+
}
|
|
510
|
+
catch (caught) {
|
|
511
|
+
reportFailure('durability.fanout.dead_letter_failed', { id: message.id, target: row.target_id }, caught);
|
|
512
|
+
}
|
|
513
|
+
}, _DurabilityFanout_deadline = async function _DurabilityFanout_deadline(run, timeoutMs) {
|
|
514
|
+
let timer;
|
|
515
|
+
try {
|
|
516
|
+
await Promise.race([
|
|
517
|
+
Promise.resolve().then(run),
|
|
518
|
+
new Promise((_resolve, reject) => {
|
|
519
|
+
timer = setTimeout(() => reject(new FanoutTimeoutError(timeoutMs)), timeoutMs);
|
|
520
|
+
}),
|
|
521
|
+
]);
|
|
522
|
+
}
|
|
523
|
+
finally {
|
|
524
|
+
if (timer !== undefined) {
|
|
525
|
+
clearTimeout(timer);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
};
|