@relaybus/testing 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.cjs +1266 -1187
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +19 -6
- package/dist/index.d.ts +19 -6
- package/dist/index.js +1266 -1187
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createUuidV7 } from '@relaybus/core';
|
|
2
1
|
import { describe, beforeEach, afterEach, it, expect } from 'vitest';
|
|
2
|
+
import { createUuidV7 } from '@relaybus/core';
|
|
3
3
|
|
|
4
4
|
// src/event-store-contract.ts
|
|
5
5
|
function createDeferred() {
|
|
@@ -31,1251 +31,1327 @@ function createEnvelope(overrides = {}) {
|
|
|
31
31
|
function createDelivery(handler, availableAt = at(0)) {
|
|
32
32
|
return { id: createUuidV7(), handler, availableAt };
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
|
|
35
|
+
// src/contract/append-and-claim.ts
|
|
36
|
+
function appendAndClaimContract(suite) {
|
|
37
|
+
const { append, claim } = suite;
|
|
38
|
+
let store;
|
|
39
|
+
beforeEach(() => {
|
|
40
|
+
store = suite.store();
|
|
41
|
+
});
|
|
42
|
+
describe("setup", () => {
|
|
43
|
+
it("can run more than once", async () => {
|
|
44
|
+
await store.setup?.();
|
|
45
|
+
await store.setup?.();
|
|
46
|
+
expect(await store.statistics(at(0))).toMatchObject({ pending: 0 });
|
|
40
47
|
});
|
|
41
|
-
|
|
42
|
-
|
|
48
|
+
});
|
|
49
|
+
describe("append", () => {
|
|
50
|
+
it("stores events and deliveries when the transaction commits", async () => {
|
|
51
|
+
const event = createEnvelope();
|
|
52
|
+
const result = await append([
|
|
53
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
54
|
+
]);
|
|
55
|
+
expect(result).toEqual({
|
|
56
|
+
appendedEventIds: [event.id],
|
|
57
|
+
duplicateEventIds: [],
|
|
58
|
+
appendedDeliveryCount: 2
|
|
59
|
+
});
|
|
60
|
+
const records = await store.listDeliveries({ eventId: event.id });
|
|
61
|
+
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
62
|
+
expect(records.every((record) => record.status === "pending" && record.attempts === 0)).toBe(
|
|
63
|
+
true
|
|
64
|
+
);
|
|
43
65
|
});
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
handlers,
|
|
60
|
-
limit,
|
|
61
|
-
leaseOwner: owner,
|
|
62
|
-
leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
|
|
63
|
-
now
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
describe("setup", () => {
|
|
67
|
-
it("can run more than once", async () => {
|
|
68
|
-
await store.setup?.();
|
|
69
|
-
await store.setup?.();
|
|
70
|
-
expect(await store.statistics(at(0))).toMatchObject({ pending: 0 });
|
|
71
|
-
});
|
|
66
|
+
it("stores nothing when the transaction rolls back", async () => {
|
|
67
|
+
const event = createEnvelope();
|
|
68
|
+
const failure = new Error("rollback");
|
|
69
|
+
await expect(
|
|
70
|
+
store.transaction(async (transaction) => {
|
|
71
|
+
await store.append(
|
|
72
|
+
transaction,
|
|
73
|
+
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
74
|
+
at(0)
|
|
75
|
+
);
|
|
76
|
+
throw failure;
|
|
77
|
+
})
|
|
78
|
+
).rejects.toBe(failure);
|
|
79
|
+
expect(await store.listDeliveries({})).toEqual([]);
|
|
80
|
+
expect(await claim(["handler-a"])).toEqual([]);
|
|
72
81
|
});
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const event = createEnvelope();
|
|
76
|
-
const result = await append([
|
|
77
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
78
|
-
]);
|
|
79
|
-
expect(result).toEqual({
|
|
80
|
-
appendedEventIds: [event.id],
|
|
81
|
-
duplicateEventIds: [],
|
|
82
|
-
appendedDeliveryCount: 2
|
|
83
|
-
});
|
|
84
|
-
const records = await store.listDeliveries({ eventId: event.id });
|
|
85
|
-
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
86
|
-
expect(
|
|
87
|
-
records.every((record) => record.status === "pending" && record.attempts === 0)
|
|
88
|
-
).toBe(true);
|
|
89
|
-
});
|
|
90
|
-
it("stores nothing when the transaction rolls back", async () => {
|
|
91
|
-
const event = createEnvelope();
|
|
92
|
-
const failure = new Error("rollback");
|
|
93
|
-
await expect(
|
|
94
|
-
store.transaction(async (transaction) => {
|
|
95
|
-
await store.append(
|
|
96
|
-
transaction,
|
|
97
|
-
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
98
|
-
at(0)
|
|
99
|
-
);
|
|
100
|
-
throw failure;
|
|
101
|
-
})
|
|
102
|
-
).rejects.toBe(failure);
|
|
103
|
-
expect(await store.listDeliveries({})).toEqual([]);
|
|
104
|
-
expect(await claim(["handler-a"])).toEqual([]);
|
|
105
|
-
});
|
|
106
|
-
it("returns the value of the transaction work", async () => {
|
|
107
|
-
await expect(store.transaction(async () => "result")).resolves.toBe("result");
|
|
108
|
-
});
|
|
109
|
-
it("round-trips the envelope exactly as JSON would", async () => {
|
|
110
|
-
const event = createEnvelope({
|
|
111
|
-
key: "order-42",
|
|
112
|
-
payload: { text: "\xE7\u011F\u0131\xF6\u015F\xFC \u{1F680}", nested: { list: [1, "two", null, true] }, amount: 12.5 },
|
|
113
|
-
headers: { traceparent: "00-abc-def-01", tenant: "acme" },
|
|
114
|
-
correlationId: createUuidV7(),
|
|
115
|
-
causationId: createUuidV7(),
|
|
116
|
-
occurredAt: /* @__PURE__ */ new Date("2026-03-04T05:06:07.089Z")
|
|
117
|
-
});
|
|
118
|
-
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
119
|
-
const [claimed] = await claim(["handler-a"]);
|
|
120
|
-
expect(claimed.event).toEqual(event);
|
|
121
|
-
expect(claimed.event.occurredAt).toBeInstanceOf(Date);
|
|
122
|
-
});
|
|
123
|
-
it("skips an event id that already exists but adds its missing deliveries", async () => {
|
|
124
|
-
const event = createEnvelope();
|
|
125
|
-
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
126
|
-
const result = await append([
|
|
127
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
128
|
-
]);
|
|
129
|
-
expect(result).toEqual({
|
|
130
|
-
appendedEventIds: [],
|
|
131
|
-
duplicateEventIds: [event.id],
|
|
132
|
-
appendedDeliveryCount: 1
|
|
133
|
-
});
|
|
134
|
-
const records = await store.listDeliveries({ eventId: event.id });
|
|
135
|
-
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
136
|
-
});
|
|
137
|
-
it("deduplicates repeated events and handlers inside one call", async () => {
|
|
138
|
-
const event = createEnvelope();
|
|
139
|
-
const result = await append([
|
|
140
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-a")] },
|
|
141
|
-
{ event, deliveries: [createDelivery("handler-a")] }
|
|
142
|
-
]);
|
|
143
|
-
expect(result.appendedEventIds).toEqual([event.id]);
|
|
144
|
-
expect(result.duplicateEventIds).toEqual([event.id]);
|
|
145
|
-
expect(result.appendedDeliveryCount).toBe(1);
|
|
146
|
-
expect(await store.listDeliveries({})).toHaveLength(1);
|
|
147
|
-
});
|
|
148
|
-
it("keeps events written by concurrent transactions apart", async () => {
|
|
149
|
-
const events = Array.from({ length: 20 }, () => createEnvelope());
|
|
150
|
-
await Promise.all(
|
|
151
|
-
events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
|
|
152
|
-
);
|
|
153
|
-
expect(await store.listDeliveries({ limit: 1e3 })).toHaveLength(20);
|
|
154
|
-
});
|
|
82
|
+
it("returns the value of the transaction work", async () => {
|
|
83
|
+
await expect(store.transaction(async () => "result")).resolves.toBe("result");
|
|
155
84
|
});
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
expect(claimed[0].leaseToken).toEqual(expect.any(String));
|
|
170
|
-
const [record] = await store.listDeliveries({ handler: "handler-a" });
|
|
171
|
-
expect(record).toMatchObject({ status: "processing", attempts: 1, leaseOwner: "worker-1" });
|
|
172
|
-
expect(record.leaseExpiresAt).toEqual(at(1e3 + LEASE_MS));
|
|
173
|
-
});
|
|
174
|
-
it("ignores deliveries that are not due yet", async () => {
|
|
175
|
-
await append([
|
|
176
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(5e3))] }
|
|
177
|
-
]);
|
|
178
|
-
expect(await claim(["handler-a"], at(4999))).toEqual([]);
|
|
179
|
-
expect(await claim(["handler-a"], at(5e3))).toHaveLength(1);
|
|
180
|
-
});
|
|
181
|
-
it("returns nothing for an empty handler list", async () => {
|
|
182
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
183
|
-
expect(await claim([])).toEqual([]);
|
|
184
|
-
});
|
|
185
|
-
it("respects the limit and claims the earliest due first", async () => {
|
|
186
|
-
const late = createEnvelope();
|
|
187
|
-
const early = createEnvelope();
|
|
188
|
-
const middle = createEnvelope();
|
|
189
|
-
await append([
|
|
190
|
-
{ event: late, deliveries: [createDelivery("handler-a", at(300))] },
|
|
191
|
-
{ event: early, deliveries: [createDelivery("handler-a", at(100))] },
|
|
192
|
-
{ event: middle, deliveries: [createDelivery("handler-a", at(200))] }
|
|
193
|
-
]);
|
|
194
|
-
const first = await claim(["handler-a"], at(1e3), 2);
|
|
195
|
-
expect(first.map((delivery) => delivery.event.id)).toEqual([early.id, middle.id]);
|
|
196
|
-
const second = await claim(["handler-a"], at(1e3), 2);
|
|
197
|
-
expect(second.map((delivery) => delivery.event.id)).toEqual([late.id]);
|
|
198
|
-
});
|
|
199
|
-
it("breaks ties between equally due deliveries by delivery id", async () => {
|
|
200
|
-
const ids = [createUuidV7(), createUuidV7(), createUuidV7()].sort();
|
|
201
|
-
await append([
|
|
202
|
-
{
|
|
203
|
-
event: createEnvelope(),
|
|
204
|
-
deliveries: [{ id: ids[2], handler: "handler-a", availableAt: at(0) }]
|
|
205
|
-
},
|
|
206
|
-
{
|
|
207
|
-
event: createEnvelope(),
|
|
208
|
-
deliveries: [{ id: ids[0], handler: "handler-a", availableAt: at(0) }]
|
|
209
|
-
},
|
|
210
|
-
{
|
|
211
|
-
event: createEnvelope(),
|
|
212
|
-
deliveries: [{ id: ids[1], handler: "handler-a", availableAt: at(0) }]
|
|
213
|
-
}
|
|
214
|
-
]);
|
|
215
|
-
const claimed = await claim(["handler-a"], at(0), 3);
|
|
216
|
-
expect(claimed.map((delivery) => delivery.deliveryId)).toEqual(ids);
|
|
217
|
-
});
|
|
218
|
-
it("does not hand out a leased delivery until its lease expires", async () => {
|
|
219
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
220
|
-
const [first] = await claim(["handler-a"], at(0));
|
|
221
|
-
expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
|
|
222
|
-
const [second] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
223
|
-
expect(second).toMatchObject({ deliveryId: first.deliveryId, attempt: 2 });
|
|
224
|
-
expect(second.leaseToken).not.toBe(first.leaseToken);
|
|
225
|
-
});
|
|
226
|
-
it("never gives the same delivery to two concurrent claimers", async () => {
|
|
227
|
-
const entries = Array.from({ length: 40 }, () => ({
|
|
228
|
-
event: createEnvelope(),
|
|
229
|
-
deliveries: [createDelivery("handler-a")]
|
|
230
|
-
}));
|
|
231
|
-
await append(entries);
|
|
232
|
-
const batches = await Promise.all(
|
|
233
|
-
Array.from({ length: 8 }, async (_, index) => {
|
|
234
|
-
const own = [];
|
|
235
|
-
for (; ; ) {
|
|
236
|
-
const batch = await claim(["handler-a"], at(0), 10, `worker-${index}`);
|
|
237
|
-
if (batch.length === 0) {
|
|
238
|
-
return own;
|
|
239
|
-
}
|
|
240
|
-
own.push(...batch);
|
|
241
|
-
}
|
|
242
|
-
})
|
|
243
|
-
);
|
|
244
|
-
const leftovers = await claim(["handler-a"], at(0), 100, "worker-late");
|
|
245
|
-
const claimedIds = [...batches.flat(), ...leftovers].map((delivery) => delivery.deliveryId);
|
|
246
|
-
expect(new Set(claimedIds).size).toBe(claimedIds.length);
|
|
247
|
-
expect(claimedIds).toHaveLength(40);
|
|
248
|
-
});
|
|
85
|
+
it("round-trips the envelope exactly as JSON would", async () => {
|
|
86
|
+
const event = createEnvelope({
|
|
87
|
+
key: "order-42",
|
|
88
|
+
payload: { text: "\xE7\u011F\u0131\xF6\u015F\xFC \u{1F680}", nested: { list: [1, "two", null, true] }, amount: 12.5 },
|
|
89
|
+
headers: { traceparent: "00-abc-def-01", tenant: "acme" },
|
|
90
|
+
correlationId: createUuidV7(),
|
|
91
|
+
causationId: createUuidV7(),
|
|
92
|
+
occurredAt: /* @__PURE__ */ new Date("2026-03-04T05:06:07.089Z")
|
|
93
|
+
});
|
|
94
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
95
|
+
const [claimed] = await claim(["handler-a"]);
|
|
96
|
+
expect(claimed.event).toEqual(event);
|
|
97
|
+
expect(claimed.event.occurredAt).toBeInstanceOf(Date);
|
|
249
98
|
});
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
expect(await store.complete(current, at(LEASE_MS + 2))).toBe(true);
|
|
264
|
-
if (!keepsCompleted) {
|
|
265
|
-
expect(await store.listDeliveries({})).toEqual([]);
|
|
266
|
-
return;
|
|
267
|
-
}
|
|
268
|
-
const [record] = await store.listDeliveries({});
|
|
269
|
-
expect(record).toMatchObject({
|
|
270
|
-
status: "completed",
|
|
271
|
-
leaseOwner: null,
|
|
272
|
-
leaseExpiresAt: null
|
|
273
|
-
});
|
|
274
|
-
expect(record.completedAt).toEqual(at(LEASE_MS + 2));
|
|
275
|
-
});
|
|
276
|
-
it("refuses to complete twice", async () => {
|
|
277
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
278
|
-
const [delivery] = await claim(["handler-a"]);
|
|
279
|
-
expect(await store.complete(delivery, at(1))).toBe(true);
|
|
280
|
-
expect(await store.complete(delivery, at(2))).toBe(false);
|
|
281
|
-
});
|
|
282
|
-
it("refuses leases of unknown deliveries", async () => {
|
|
283
|
-
const lease = { deliveryId: createUuidV7(), leaseToken: createUuidV7() };
|
|
284
|
-
expect(await store.complete(lease, at(0))).toBe(false);
|
|
285
|
-
expect(await store.extendLease(lease, at(1), at(0))).toBe(false);
|
|
286
|
-
});
|
|
287
|
-
it("extends a held lease", async () => {
|
|
288
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
289
|
-
const [delivery] = await claim(["handler-a"]);
|
|
290
|
-
expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
|
|
291
|
-
expect(await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2")).toEqual([]);
|
|
292
|
-
const [record] = await store.listDeliveries({});
|
|
293
|
-
expect(record.leaseExpiresAt).toEqual(at(LEASE_MS * 2));
|
|
294
|
-
});
|
|
295
|
-
it("completes inside a transaction together with it", async () => {
|
|
296
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
297
|
-
const [delivery] = await claim(["handler-a"]);
|
|
298
|
-
await expect(
|
|
299
|
-
store.transaction(async (transaction) => {
|
|
300
|
-
expect(await store.complete(delivery, at(1), transaction)).toBe(true);
|
|
301
|
-
throw new Error("rollback");
|
|
302
|
-
})
|
|
303
|
-
).rejects.toThrow("rollback");
|
|
304
|
-
expect((await store.listDeliveries({}))[0].status).toBe("processing");
|
|
305
|
-
await store.transaction(async (transaction) => {
|
|
306
|
-
expect(await store.complete(delivery, at(2), transaction)).toBe(true);
|
|
307
|
-
});
|
|
308
|
-
if (keepsCompleted) {
|
|
309
|
-
expect((await store.listDeliveries({}))[0].status).toBe("completed");
|
|
310
|
-
} else {
|
|
311
|
-
expect(await store.listDeliveries({})).toEqual([]);
|
|
312
|
-
}
|
|
313
|
-
});
|
|
314
|
-
it("refuses a transactional completion with a lost lease", async () => {
|
|
315
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
316
|
-
const [stale] = await claim(["handler-a"], at(0));
|
|
317
|
-
await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
318
|
-
await store.transaction(async (transaction) => {
|
|
319
|
-
expect(await store.complete(stale, at(LEASE_MS + 1), transaction)).toBe(false);
|
|
320
|
-
});
|
|
321
|
-
expect((await store.listDeliveries({}))[0].status).toBe("processing");
|
|
322
|
-
});
|
|
99
|
+
it("skips an event id that already exists but adds its missing deliveries", async () => {
|
|
100
|
+
const event = createEnvelope();
|
|
101
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
102
|
+
const result = await append([
|
|
103
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
104
|
+
]);
|
|
105
|
+
expect(result).toEqual({
|
|
106
|
+
appendedEventIds: [],
|
|
107
|
+
duplicateEventIds: [event.id],
|
|
108
|
+
appendedDeliveryCount: 1
|
|
109
|
+
});
|
|
110
|
+
const records = await store.listDeliveries({ eventId: event.id });
|
|
111
|
+
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
323
112
|
});
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
lastError: error,
|
|
335
|
-
leaseOwner: null
|
|
336
|
-
});
|
|
337
|
-
expect(record.availableAt).toEqual(at(1e4));
|
|
338
|
-
expect(await claim(["handler-a"], at(9999))).toEqual([]);
|
|
339
|
-
const [retry] = await claim(["handler-a"], at(1e4));
|
|
340
|
-
expect(retry.attempt).toBe(2);
|
|
341
|
-
});
|
|
342
|
-
it("buries a delivery so it is never claimed again", async () => {
|
|
343
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
344
|
-
const [delivery] = await claim(["handler-a"]);
|
|
345
|
-
const error = { name: "Error", message: "fatal" };
|
|
346
|
-
expect(await store.bury(delivery, error, at(5))).toBe(true);
|
|
347
|
-
expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
|
|
348
|
-
const [record] = await store.listDeliveries({ status: "dead" });
|
|
349
|
-
expect(record).toMatchObject({ status: "dead", lastError: error });
|
|
350
|
-
expect(record.updatedAt).toEqual(at(5));
|
|
351
|
-
});
|
|
352
|
-
it("requeues only dead deliveries, with a fresh attempt budget", async () => {
|
|
353
|
-
const event = createEnvelope();
|
|
354
|
-
await append([
|
|
355
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
356
|
-
]);
|
|
357
|
-
const [dead] = await claim(["handler-a"]);
|
|
358
|
-
await store.bury(dead, { name: "Error", message: "fatal" }, at(1));
|
|
359
|
-
const [pending] = await store.listDeliveries({ handler: "handler-b" });
|
|
360
|
-
const moved = await store.requeue(
|
|
361
|
-
[dead.deliveryId, dead.deliveryId, pending.id, createUuidV7()],
|
|
362
|
-
at(100)
|
|
363
|
-
);
|
|
364
|
-
expect(moved).toBe(1);
|
|
365
|
-
const [record] = await store.listDeliveries({ handler: "handler-a" });
|
|
366
|
-
expect(record).toMatchObject({ status: "pending", attempts: 0 });
|
|
367
|
-
expect(record.availableAt).toEqual(at(100));
|
|
368
|
-
const [again] = await claim(["handler-a"], at(100));
|
|
369
|
-
expect(again.attempt).toBe(1);
|
|
370
|
-
});
|
|
113
|
+
it("deduplicates repeated events and handlers inside one call", async () => {
|
|
114
|
+
const event = createEnvelope();
|
|
115
|
+
const result = await append([
|
|
116
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-a")] },
|
|
117
|
+
{ event, deliveries: [createDelivery("handler-a")] }
|
|
118
|
+
]);
|
|
119
|
+
expect(result.appendedEventIds).toEqual([event.id]);
|
|
120
|
+
expect(result.duplicateEventIds).toEqual([event.id]);
|
|
121
|
+
expect(result.appendedDeliveryCount).toBe(1);
|
|
122
|
+
expect(await store.listDeliveries({})).toHaveLength(1);
|
|
371
123
|
});
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
return events;
|
|
379
|
-
}
|
|
380
|
-
it("hands out one delivery per handler and key, in publish order", async () => {
|
|
381
|
-
const [first, second, third] = await appendKeyed("order-1", 3);
|
|
382
|
-
const batch1 = await claim(["handler-a"]);
|
|
383
|
-
expect(batch1.map((delivery) => delivery.event.id)).toEqual([first.id]);
|
|
384
|
-
expect(await claim(["handler-a"])).toEqual([]);
|
|
385
|
-
await store.complete(batch1[0], at(1));
|
|
386
|
-
const batch2 = await claim(["handler-a"]);
|
|
387
|
-
expect(batch2.map((delivery) => delivery.event.id)).toEqual([second.id]);
|
|
388
|
-
await store.complete(batch2[0], at(2));
|
|
389
|
-
expect((await claim(["handler-a"])).map((delivery) => delivery.event.id)).toEqual([
|
|
390
|
-
third.id
|
|
391
|
-
]);
|
|
392
|
-
});
|
|
393
|
-
it("keeps later events waiting while the head is retrying", async () => {
|
|
394
|
-
const [first, second] = await appendKeyed("order-1", 2);
|
|
395
|
-
const [head] = await claim(["handler-a"]);
|
|
396
|
-
await store.reschedule(head, at(6e4), { name: "Error", message: "retry" }, at(1));
|
|
397
|
-
expect(await claim(["handler-a"], at(3e4))).toEqual([]);
|
|
398
|
-
const [retried] = await claim(["handler-a"], at(6e4));
|
|
399
|
-
expect(retried.event.id).toBe(first.id);
|
|
400
|
-
await store.complete(retried, at(60001));
|
|
401
|
-
expect((await claim(["handler-a"], at(60002)))[0].event.id).toBe(second.id);
|
|
402
|
-
});
|
|
403
|
-
it("releases the key when the head is dead-lettered", async () => {
|
|
404
|
-
const [, second] = await appendKeyed("order-1", 2);
|
|
405
|
-
const [head] = await claim(["handler-a"]);
|
|
406
|
-
await store.bury(head, { name: "Error", message: "fatal" }, at(1));
|
|
407
|
-
expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
|
|
408
|
-
});
|
|
409
|
-
it("orders each handler independently", async () => {
|
|
410
|
-
const [first, second] = await appendKeyed("order-1", 2, ["handler-a", "handler-b"]);
|
|
411
|
-
const [fromA] = await claim(["handler-a"]);
|
|
412
|
-
await store.complete(fromA, at(1));
|
|
413
|
-
const [nextA] = await claim(["handler-a"]);
|
|
414
|
-
expect(nextA.event.id).toBe(second.id);
|
|
415
|
-
const [fromB] = await claim(["handler-b"]);
|
|
416
|
-
expect(fromB.event.id).toBe(first.id);
|
|
417
|
-
});
|
|
418
|
-
it("runs different keys and unkeyed events in parallel", async () => {
|
|
419
|
-
await appendKeyed("order-1", 2);
|
|
420
|
-
await appendKeyed("order-2", 2);
|
|
421
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
422
|
-
expect(await claim(["handler-a"])).toHaveLength(3);
|
|
423
|
-
});
|
|
424
|
-
it("orders concurrent publishers of the same key by commit", async () => {
|
|
425
|
-
const events = Array.from(
|
|
426
|
-
{ length: 10 },
|
|
427
|
-
(_, index) => createEnvelope({ key: "shared", payload: { index } })
|
|
428
|
-
);
|
|
429
|
-
await Promise.all(
|
|
430
|
-
events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
|
|
431
|
-
);
|
|
432
|
-
const seen = [];
|
|
433
|
-
for (let round = 0; round < 10; round += 1) {
|
|
434
|
-
const batch = await claim(["handler-a"]);
|
|
435
|
-
expect(batch).toHaveLength(1);
|
|
436
|
-
seen.push(batch[0].event.id);
|
|
437
|
-
await store.complete(batch[0], at(round + 1));
|
|
438
|
-
}
|
|
439
|
-
expect(new Set(seen).size).toBe(10);
|
|
440
|
-
});
|
|
124
|
+
it("keeps events written by concurrent transactions apart", async () => {
|
|
125
|
+
const events = Array.from({ length: 20 }, () => createEnvelope());
|
|
126
|
+
await Promise.all(
|
|
127
|
+
events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
|
|
128
|
+
);
|
|
129
|
+
expect(await store.listDeliveries({ limit: 1e3 })).toHaveLength(20);
|
|
441
130
|
});
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
).toHaveLength(2);
|
|
461
|
-
expect(await store.listDeliveries({ status: "dead" })).toEqual([]);
|
|
462
|
-
const limited = await store.listDeliveries({ limit: 1 });
|
|
463
|
-
expect(limited).toHaveLength(1);
|
|
464
|
-
expect(limited[0].event.id).toBe(first.id);
|
|
465
|
-
expect(limited[0].createdAt).toEqual(at(1));
|
|
466
|
-
});
|
|
467
|
-
it("reports counts and the oldest due delivery", async () => {
|
|
468
|
-
await append([
|
|
469
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(100))] },
|
|
470
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(200))] },
|
|
471
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-b", at(5e4))] },
|
|
472
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-c", at(0))] }
|
|
473
|
-
]);
|
|
474
|
-
const [processing] = await claim(["handler-c"], at(0));
|
|
475
|
-
const [dead] = await claim(["handler-a"], at(100), 1);
|
|
476
|
-
await store.bury(dead, { name: "Error", message: "x" }, at(101));
|
|
477
|
-
expect(await store.statistics(at(1e3))).toEqual({
|
|
478
|
-
pending: 2,
|
|
479
|
-
processing: 1,
|
|
480
|
-
completed: 0,
|
|
481
|
-
dead: 1,
|
|
482
|
-
oldestDueAt: at(200)
|
|
483
|
-
});
|
|
484
|
-
expect((await store.statistics(at(LEASE_MS))).oldestDueAt).toEqual(at(200));
|
|
485
|
-
await store.complete(processing, at(1));
|
|
486
|
-
expect(await store.statistics(at(0))).toMatchObject({
|
|
487
|
-
completed: keepsCompleted ? 1 : 0,
|
|
488
|
-
processing: 0,
|
|
489
|
-
oldestDueAt: null
|
|
490
|
-
});
|
|
491
|
-
});
|
|
492
|
-
it("counts an expired lease as due", async () => {
|
|
493
|
-
await append([
|
|
494
|
-
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(LEASE_MS * 5))] }
|
|
495
|
-
]);
|
|
496
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
|
|
497
|
-
await claim(["handler-b"], at(0));
|
|
498
|
-
expect((await store.statistics(at(LEASE_MS + 1))).oldestDueAt).toEqual(at(LEASE_MS));
|
|
499
|
-
});
|
|
131
|
+
});
|
|
132
|
+
describe("claim", () => {
|
|
133
|
+
it("leases due deliveries of the requested handlers and counts the attempt", async () => {
|
|
134
|
+
const event = createEnvelope();
|
|
135
|
+
await append([
|
|
136
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
137
|
+
]);
|
|
138
|
+
const claimed = await claim(["handler-a"], at(1e3));
|
|
139
|
+
expect(claimed).toHaveLength(1);
|
|
140
|
+
expect(claimed[0]).toMatchObject({
|
|
141
|
+
handler: "handler-a",
|
|
142
|
+
attempt: 1,
|
|
143
|
+
leaseExpiresAt: at(1e3 + LEASE_MS)
|
|
144
|
+
});
|
|
145
|
+
expect(claimed[0].leaseToken).toEqual(expect.any(String));
|
|
146
|
+
const [record] = await store.listDeliveries({ handler: "handler-a" });
|
|
147
|
+
expect(record).toMatchObject({ status: "processing", attempts: 1, leaseOwner: "worker-1" });
|
|
148
|
+
expect(record.leaseExpiresAt).toEqual(at(1e3 + LEASE_MS));
|
|
500
149
|
});
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
150
|
+
it("ignores deliveries that are not due yet", async () => {
|
|
151
|
+
await append([
|
|
152
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(5e3))] }
|
|
153
|
+
]);
|
|
154
|
+
expect(await claim(["handler-a"], at(4999))).toEqual([]);
|
|
155
|
+
expect(await claim(["handler-a"], at(5e3))).toHaveLength(1);
|
|
156
|
+
});
|
|
157
|
+
it("returns nothing for an empty handler list", async () => {
|
|
158
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
159
|
+
expect(await claim([])).toEqual([]);
|
|
160
|
+
});
|
|
161
|
+
it("respects the limit and claims the earliest due first", async () => {
|
|
162
|
+
const late = createEnvelope();
|
|
163
|
+
const early = createEnvelope();
|
|
164
|
+
const middle = createEnvelope();
|
|
165
|
+
await append([
|
|
166
|
+
{ event: late, deliveries: [createDelivery("handler-a", at(300))] },
|
|
167
|
+
{ event: early, deliveries: [createDelivery("handler-a", at(100))] },
|
|
168
|
+
{ event: middle, deliveries: [createDelivery("handler-a", at(200))] }
|
|
169
|
+
]);
|
|
170
|
+
const first = await claim(["handler-a"], at(1e3), 2);
|
|
171
|
+
expect(first.map((delivery) => delivery.event.id)).toEqual([early.id, middle.id]);
|
|
172
|
+
const second = await claim(["handler-a"], at(1e3), 2);
|
|
173
|
+
expect(second.map((delivery) => delivery.event.id)).toEqual([late.id]);
|
|
174
|
+
});
|
|
175
|
+
it("breaks ties between equally due deliveries by delivery id", async () => {
|
|
176
|
+
const ids = [createUuidV7(), createUuidV7(), createUuidV7()].sort();
|
|
177
|
+
await append([
|
|
178
|
+
{
|
|
179
|
+
event: createEnvelope(),
|
|
180
|
+
deliveries: [{ id: ids[2], handler: "handler-a", availableAt: at(0) }]
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
event: createEnvelope(),
|
|
184
|
+
deliveries: [{ id: ids[0], handler: "handler-a", availableAt: at(0) }]
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
event: createEnvelope(),
|
|
188
|
+
deliveries: [{ id: ids[1], handler: "handler-a", availableAt: at(0) }]
|
|
539
189
|
}
|
|
190
|
+
]);
|
|
191
|
+
const claimed = await claim(["handler-a"], at(0), 3);
|
|
192
|
+
expect(claimed.map((delivery) => delivery.deliveryId)).toEqual(ids);
|
|
193
|
+
});
|
|
194
|
+
it("does not hand out a leased delivery until its lease expires", async () => {
|
|
195
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
196
|
+
const [first] = await claim(["handler-a"], at(0));
|
|
197
|
+
expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
|
|
198
|
+
const [second] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
199
|
+
expect(second).toMatchObject({ deliveryId: first.deliveryId, attempt: 2 });
|
|
200
|
+
expect(second.leaseToken).not.toBe(first.leaseToken);
|
|
201
|
+
});
|
|
202
|
+
it("never gives the same delivery to two concurrent claimers", async () => {
|
|
203
|
+
const entries = Array.from({ length: 40 }, () => ({
|
|
204
|
+
event: createEnvelope(),
|
|
205
|
+
deliveries: [createDelivery("handler-a")]
|
|
206
|
+
}));
|
|
207
|
+
await append(entries);
|
|
208
|
+
const batches = await Promise.all(
|
|
209
|
+
Array.from({ length: 8 }, async (_, index) => {
|
|
210
|
+
const own = [];
|
|
211
|
+
for (; ; ) {
|
|
212
|
+
const batch = await claim(["handler-a"], at(0), 10, `worker-${index}`);
|
|
213
|
+
if (batch.length === 0) {
|
|
214
|
+
return own;
|
|
215
|
+
}
|
|
216
|
+
own.push(...batch);
|
|
217
|
+
}
|
|
218
|
+
})
|
|
540
219
|
);
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
220
|
+
const leftovers = await claim(["handler-a"], at(0), 100, "worker-late");
|
|
221
|
+
const claimedIds = [...batches.flat(), ...leftovers].map((delivery) => delivery.deliveryId);
|
|
222
|
+
expect(new Set(claimedIds).size).toBe(claimedIds.length);
|
|
223
|
+
expect(claimedIds).toHaveLength(40);
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
function fencingContract(suite) {
|
|
228
|
+
const { append, claim, keepsCompleted } = suite;
|
|
229
|
+
let store;
|
|
230
|
+
beforeEach(() => {
|
|
231
|
+
store = suite.store();
|
|
232
|
+
});
|
|
233
|
+
describe("fencing", () => {
|
|
234
|
+
it("completes a delivery only with its current lease", async () => {
|
|
235
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
236
|
+
const [stale] = await claim(["handler-a"], at(0));
|
|
237
|
+
const [current] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
238
|
+
expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
|
|
239
|
+
expect(await store.extendLease(stale, at(LEASE_MS * 3), at(LEASE_MS + 1))).toBe(false);
|
|
240
|
+
expect(
|
|
241
|
+
await store.reschedule(stale, at(0), { name: "Error", message: "x" }, at(LEASE_MS + 1))
|
|
242
|
+
).toBe(false);
|
|
243
|
+
expect(await store.bury(stale, { name: "Error", message: "x" }, at(LEASE_MS + 1))).toBe(
|
|
244
|
+
false
|
|
562
245
|
);
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
await
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
246
|
+
expect(await store.complete(current, at(LEASE_MS + 2))).toBe(true);
|
|
247
|
+
if (!keepsCompleted) {
|
|
248
|
+
expect(await store.listDeliveries({})).toEqual([]);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const [record] = await store.listDeliveries({});
|
|
252
|
+
expect(record).toMatchObject({
|
|
253
|
+
status: "completed",
|
|
254
|
+
leaseOwner: null,
|
|
255
|
+
leaseExpiresAt: null
|
|
571
256
|
});
|
|
257
|
+
expect(record.completedAt).toEqual(at(LEASE_MS + 2));
|
|
572
258
|
});
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
const [record] = await store.listDeliveries({});
|
|
579
|
-
expect(record).toMatchObject({
|
|
580
|
-
status: "pending",
|
|
581
|
-
attempts: 0,
|
|
582
|
-
leaseOwner: null,
|
|
583
|
-
leaseExpiresAt: null
|
|
584
|
-
});
|
|
585
|
-
expect(record.availableAt).toEqual(at(500));
|
|
586
|
-
expect(record.updatedAt).toEqual(at(500));
|
|
587
|
-
expect(await store.release(delivery, at(501))).toBe(false);
|
|
588
|
-
const [again] = await claim(["handler-a"], at(500));
|
|
589
|
-
expect(again.attempt).toBe(1);
|
|
590
|
-
});
|
|
259
|
+
it("refuses to complete twice", async () => {
|
|
260
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
261
|
+
const [delivery] = await claim(["handler-a"]);
|
|
262
|
+
expect(await store.complete(delivery, at(1))).toBe(true);
|
|
263
|
+
expect(await store.complete(delivery, at(2))).toBe(false);
|
|
591
264
|
});
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
265
|
+
it("refuses leases of unknown deliveries", async () => {
|
|
266
|
+
const lease = { deliveryId: createUuidV7(), leaseToken: createUuidV7() };
|
|
267
|
+
expect(await store.complete(lease, at(0))).toBe(false);
|
|
268
|
+
expect(await store.extendLease(lease, at(1), at(0))).toBe(false);
|
|
269
|
+
});
|
|
270
|
+
it("extends a held lease", async () => {
|
|
271
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
272
|
+
const [delivery] = await claim(["handler-a"]);
|
|
273
|
+
expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
|
|
274
|
+
expect(await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2")).toEqual([]);
|
|
275
|
+
const [record] = await store.listDeliveries({});
|
|
276
|
+
expect(record.leaseExpiresAt).toEqual(at(LEASE_MS * 2));
|
|
277
|
+
});
|
|
278
|
+
it("completes inside a transaction together with it", async () => {
|
|
279
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
280
|
+
const [delivery] = await claim(["handler-a"]);
|
|
281
|
+
await expect(
|
|
282
|
+
store.transaction(async (transaction) => {
|
|
283
|
+
expect(await store.complete(delivery, at(1), transaction)).toBe(true);
|
|
284
|
+
throw new Error("rollback");
|
|
285
|
+
})
|
|
286
|
+
).rejects.toThrow("rollback");
|
|
287
|
+
expect((await store.listDeliveries({}))[0].status).toBe("processing");
|
|
288
|
+
await store.transaction(async (transaction) => {
|
|
289
|
+
expect(await store.complete(delivery, at(2), transaction)).toBe(true);
|
|
290
|
+
});
|
|
291
|
+
if (keepsCompleted) {
|
|
292
|
+
expect((await store.listDeliveries({}))[0].status).toBe("completed");
|
|
293
|
+
} else {
|
|
608
294
|
expect(await store.listDeliveries({})).toEqual([]);
|
|
609
|
-
|
|
610
|
-
finish.resolve();
|
|
611
|
-
await open;
|
|
612
|
-
expect(await claim(["handler-a"])).toHaveLength(1);
|
|
613
|
-
});
|
|
614
|
-
it("orders same-key events by commit, not by first write", async () => {
|
|
615
|
-
const first = createEnvelope({ key: "race" });
|
|
616
|
-
const second = createEnvelope({ key: "race" });
|
|
617
|
-
const firstWritten = createDeferred();
|
|
618
|
-
const releaseFirst = createDeferred();
|
|
619
|
-
const firstTransaction = store.transaction(async (transaction) => {
|
|
620
|
-
await store.append(
|
|
621
|
-
transaction,
|
|
622
|
-
[{ event: first, deliveries: [createDelivery("handler-a")] }],
|
|
623
|
-
at(0)
|
|
624
|
-
);
|
|
625
|
-
firstWritten.resolve();
|
|
626
|
-
await releaseFirst.promise;
|
|
627
|
-
});
|
|
628
|
-
await firstWritten.promise;
|
|
629
|
-
const secondTransaction = store.transaction(
|
|
630
|
-
(transaction) => store.append(
|
|
631
|
-
transaction,
|
|
632
|
-
[{ event: second, deliveries: [createDelivery("handler-a")] }],
|
|
633
|
-
at(0)
|
|
634
|
-
)
|
|
635
|
-
);
|
|
636
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
637
|
-
releaseFirst.resolve();
|
|
638
|
-
await firstTransaction;
|
|
639
|
-
await secondTransaction;
|
|
640
|
-
const [head] = await claim(["handler-a"]);
|
|
641
|
-
expect(head.event.id).toBe(first.id);
|
|
642
|
-
await store.complete(head, at(1));
|
|
643
|
-
expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
|
|
644
|
-
});
|
|
645
|
-
it("stores an event id written by two concurrent transactions once", async () => {
|
|
646
|
-
const event = createEnvelope({ key: "same" });
|
|
647
|
-
const results = await Promise.all([
|
|
648
|
-
append([{ event, deliveries: [createDelivery("handler-a")] }]),
|
|
649
|
-
append([
|
|
650
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
651
|
-
])
|
|
652
|
-
]);
|
|
653
|
-
const appended = results.flatMap((result) => result.appendedEventIds);
|
|
654
|
-
expect(appended).toEqual([event.id]);
|
|
655
|
-
const records = await store.listDeliveries({});
|
|
656
|
-
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
657
|
-
});
|
|
295
|
+
}
|
|
658
296
|
});
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
event: createEnvelope(),
|
|
668
|
-
deliveries: [createDelivery("handler-a", at(10))]
|
|
669
|
-
}));
|
|
670
|
-
await append(unrelated);
|
|
671
|
-
const claimed = await claim(["handler-a"], at(100), 10);
|
|
672
|
-
expect(claimed).toHaveLength(6);
|
|
673
|
-
expect(claimed.filter((delivery) => delivery.event.key === "hot")).toHaveLength(1);
|
|
674
|
-
});
|
|
675
|
-
it("keeps successors blocked behind an expired, delayed or requeued head", async () => {
|
|
676
|
-
const head = createEnvelope({ key: "k" });
|
|
677
|
-
const next = createEnvelope({ key: "k" });
|
|
678
|
-
await append([{ event: head, deliveries: [createDelivery("handler-a")] }]);
|
|
679
|
-
await append([{ event: next, deliveries: [createDelivery("handler-a")] }]);
|
|
680
|
-
const [claimedHead] = await claim(["handler-a"], at(0));
|
|
681
|
-
const reclaimed = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
682
|
-
expect(reclaimed.map((delivery) => delivery.event.id)).toEqual([head.id]);
|
|
683
|
-
expect(
|
|
684
|
-
await store.reschedule(
|
|
685
|
-
reclaimed[0],
|
|
686
|
-
at(LEASE_MS * 10),
|
|
687
|
-
{ name: "Error", message: "x" },
|
|
688
|
-
at(LEASE_MS)
|
|
689
|
-
)
|
|
690
|
-
).toBe(true);
|
|
691
|
-
expect(await claim(["handler-a"], at(LEASE_MS * 5))).toEqual([]);
|
|
692
|
-
const [delayedHead] = await claim(["handler-a"], at(LEASE_MS * 10));
|
|
693
|
-
await store.bury(delayedHead, { name: "Error", message: "x" }, at(LEASE_MS * 10));
|
|
694
|
-
expect(await store.requeue([delayedHead.deliveryId], at(LEASE_MS * 11))).toBe(1);
|
|
695
|
-
const [requeuedHead] = await claim(["handler-a"], at(LEASE_MS * 11));
|
|
696
|
-
expect(requeuedHead.event.id).toBe(head.id);
|
|
697
|
-
expect(await claim(["handler-a"], at(LEASE_MS * 11))).toEqual([]);
|
|
698
|
-
expect(claimedHead).toBeDefined();
|
|
699
|
-
});
|
|
297
|
+
it("refuses a transactional completion with a lost lease", async () => {
|
|
298
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
299
|
+
const [stale] = await claim(["handler-a"], at(0));
|
|
300
|
+
await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
301
|
+
await store.transaction(async (transaction) => {
|
|
302
|
+
expect(await store.complete(stale, at(LEASE_MS + 1), transaction)).toBe(false);
|
|
303
|
+
});
|
|
304
|
+
expect((await store.listDeliveries({}))[0].status).toBe("processing");
|
|
700
305
|
});
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
const second = createEnvelope({ key: "k" });
|
|
720
|
-
await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
|
|
721
|
-
await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
|
|
722
|
-
const [head] = await claim(["handler-a"], at(0));
|
|
723
|
-
await store.bury(head, { name: "Error", message: "x" }, at(1));
|
|
724
|
-
await claim(["handler-a"], at(2));
|
|
725
|
-
await store.requeue([head.deliveryId], at(3));
|
|
726
|
-
const recovered = await claim(["handler-a"], at(2 + LEASE_MS));
|
|
727
|
-
expect(recovered.map((delivery) => delivery.event.id)).toEqual([first.id]);
|
|
728
|
-
await store.complete(recovered[0], at(3 + LEASE_MS));
|
|
729
|
-
const [retried] = await claim(["handler-a"], at(4 + LEASE_MS));
|
|
730
|
-
expect(retried).toMatchObject({ attempt: 2 });
|
|
731
|
-
expect(retried.event.id).toBe(second.id);
|
|
732
|
-
});
|
|
306
|
+
});
|
|
307
|
+
describe("retries and dead letters", () => {
|
|
308
|
+
it("reschedules a delivery with its error", async () => {
|
|
309
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
310
|
+
const [delivery] = await claim(["handler-a"]);
|
|
311
|
+
const error = { name: "TypeError", message: "boom", stack: "TypeError: boom\n at x" };
|
|
312
|
+
expect(await store.reschedule(delivery, at(1e4), error, at(5))).toBe(true);
|
|
313
|
+
const [record] = await store.listDeliveries({});
|
|
314
|
+
expect(record).toMatchObject({
|
|
315
|
+
status: "pending",
|
|
316
|
+
attempts: 1,
|
|
317
|
+
lastError: error,
|
|
318
|
+
leaseOwner: null
|
|
319
|
+
});
|
|
320
|
+
expect(record.availableAt).toEqual(at(1e4));
|
|
321
|
+
expect(await claim(["handler-a"], at(9999))).toEqual([]);
|
|
322
|
+
const [retry] = await claim(["handler-a"], at(1e4));
|
|
323
|
+
expect(retry.attempt).toBe(2);
|
|
733
324
|
});
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
});
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
325
|
+
it("buries a delivery so it is never claimed again", async () => {
|
|
326
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
327
|
+
const [delivery] = await claim(["handler-a"]);
|
|
328
|
+
const error = { name: "Error", message: "fatal" };
|
|
329
|
+
expect(await store.bury(delivery, error, at(5))).toBe(true);
|
|
330
|
+
expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
|
|
331
|
+
const [record] = await store.listDeliveries({ status: "dead" });
|
|
332
|
+
expect(record).toMatchObject({ status: "dead", lastError: error });
|
|
333
|
+
expect(record.updatedAt).toEqual(at(5));
|
|
334
|
+
});
|
|
335
|
+
it("requeues only dead deliveries, with a fresh attempt budget", async () => {
|
|
336
|
+
const event = createEnvelope();
|
|
337
|
+
await append([
|
|
338
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
339
|
+
]);
|
|
340
|
+
const [dead] = await claim(["handler-a"]);
|
|
341
|
+
await store.bury(dead, { name: "Error", message: "fatal" }, at(1));
|
|
342
|
+
const [pending] = await store.listDeliveries({ handler: "handler-b" });
|
|
343
|
+
const moved = await store.requeue(
|
|
344
|
+
[dead.deliveryId, dead.deliveryId, pending.id, createUuidV7()],
|
|
345
|
+
at(100)
|
|
346
|
+
);
|
|
347
|
+
expect(moved).toBe(1);
|
|
348
|
+
const [record] = await store.listDeliveries({ handler: "handler-a" });
|
|
349
|
+
expect(record).toMatchObject({ status: "pending", attempts: 0 });
|
|
350
|
+
expect(record.availableAt).toEqual(at(100));
|
|
351
|
+
const [again] = await claim(["handler-a"], at(100));
|
|
352
|
+
expect(again.attempt).toBe(1);
|
|
353
|
+
});
|
|
354
|
+
});
|
|
355
|
+
describe("ordering keys", () => {
|
|
356
|
+
async function appendKeyed(key, count, handlers = ["handler-a"]) {
|
|
357
|
+
const events = Array.from({ length: count }, () => createEnvelope({ key }));
|
|
358
|
+
for (const event of events) {
|
|
359
|
+
await append([{ event, deliveries: handlers.map((handler) => createDelivery(handler)) }]);
|
|
360
|
+
}
|
|
361
|
+
return events;
|
|
362
|
+
}
|
|
363
|
+
it("hands out one delivery per handler and key, in publish order", async () => {
|
|
364
|
+
const [first, second, third] = await appendKeyed("order-1", 3);
|
|
365
|
+
const batch1 = await claim(["handler-a"]);
|
|
366
|
+
expect(batch1.map((delivery) => delivery.event.id)).toEqual([first.id]);
|
|
367
|
+
expect(await claim(["handler-a"])).toEqual([]);
|
|
368
|
+
await store.complete(batch1[0], at(1));
|
|
369
|
+
const batch2 = await claim(["handler-a"]);
|
|
370
|
+
expect(batch2.map((delivery) => delivery.event.id)).toEqual([second.id]);
|
|
371
|
+
await store.complete(batch2[0], at(2));
|
|
372
|
+
expect((await claim(["handler-a"])).map((delivery) => delivery.event.id)).toEqual([
|
|
373
|
+
third.id
|
|
374
|
+
]);
|
|
375
|
+
});
|
|
376
|
+
it("keeps later events waiting while the head is retrying", async () => {
|
|
377
|
+
const [first, second] = await appendKeyed("order-1", 2);
|
|
378
|
+
const [head] = await claim(["handler-a"]);
|
|
379
|
+
await store.reschedule(head, at(6e4), { name: "Error", message: "retry" }, at(1));
|
|
380
|
+
expect(await claim(["handler-a"], at(3e4))).toEqual([]);
|
|
381
|
+
const [retried] = await claim(["handler-a"], at(6e4));
|
|
382
|
+
expect(retried.event.id).toBe(first.id);
|
|
383
|
+
await store.complete(retried, at(60001));
|
|
384
|
+
expect((await claim(["handler-a"], at(60002)))[0].event.id).toBe(second.id);
|
|
385
|
+
});
|
|
386
|
+
it("releases the key when the head is dead-lettered", async () => {
|
|
387
|
+
const [, second] = await appendKeyed("order-1", 2);
|
|
388
|
+
const [head] = await claim(["handler-a"]);
|
|
389
|
+
await store.bury(head, { name: "Error", message: "fatal" }, at(1));
|
|
390
|
+
expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
|
|
391
|
+
});
|
|
392
|
+
it("orders each handler independently", async () => {
|
|
393
|
+
const [first, second] = await appendKeyed("order-1", 2, ["handler-a", "handler-b"]);
|
|
394
|
+
const [fromA] = await claim(["handler-a"]);
|
|
395
|
+
await store.complete(fromA, at(1));
|
|
396
|
+
const [nextA] = await claim(["handler-a"]);
|
|
397
|
+
expect(nextA.event.id).toBe(second.id);
|
|
398
|
+
const [fromB] = await claim(["handler-b"]);
|
|
399
|
+
expect(fromB.event.id).toBe(first.id);
|
|
400
|
+
});
|
|
401
|
+
it("runs different keys and unkeyed events in parallel", async () => {
|
|
402
|
+
await appendKeyed("order-1", 2);
|
|
403
|
+
await appendKeyed("order-2", 2);
|
|
404
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
405
|
+
expect(await claim(["handler-a"])).toHaveLength(3);
|
|
406
|
+
});
|
|
407
|
+
it("orders concurrent publishers of the same key by commit", async () => {
|
|
408
|
+
const events = Array.from(
|
|
409
|
+
{ length: 10 },
|
|
410
|
+
(_, index) => createEnvelope({ key: "shared", payload: { index } })
|
|
411
|
+
);
|
|
412
|
+
await Promise.all(
|
|
413
|
+
events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
|
|
414
|
+
);
|
|
415
|
+
const seen = [];
|
|
416
|
+
for (let round = 0; round < 10; round += 1) {
|
|
417
|
+
const batch = await claim(["handler-a"]);
|
|
418
|
+
expect(batch).toHaveLength(1);
|
|
419
|
+
seen.push(batch[0].event.id);
|
|
420
|
+
await store.complete(batch[0], at(round + 1));
|
|
421
|
+
}
|
|
422
|
+
expect(new Set(seen).size).toBe(10);
|
|
423
|
+
});
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
function maintenanceContract(suite) {
|
|
427
|
+
const { append, claim, keepsCompleted } = suite;
|
|
428
|
+
let store;
|
|
429
|
+
beforeEach(() => {
|
|
430
|
+
store = suite.store();
|
|
431
|
+
});
|
|
432
|
+
describe("queries", () => {
|
|
433
|
+
it("filters deliveries and applies the limit in creation order", async () => {
|
|
434
|
+
const first = createEnvelope({ type: "contract.first.v1" });
|
|
435
|
+
const second = createEnvelope({ type: "contract.second.v1" });
|
|
436
|
+
await append([{ event: first, deliveries: [createDelivery("handler-a")] }], at(1));
|
|
437
|
+
await append(
|
|
438
|
+
[
|
|
750
439
|
{
|
|
751
|
-
event:
|
|
752
|
-
deliveries: [createDelivery("a"), createDelivery("b")
|
|
440
|
+
event: second,
|
|
441
|
+
deliveries: [createDelivery("handler-a"), createDelivery("handler-b")]
|
|
753
442
|
}
|
|
754
|
-
]
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
443
|
+
],
|
|
444
|
+
at(2)
|
|
445
|
+
);
|
|
446
|
+
expect(await store.listDeliveries({ eventType: "contract.second.v1" })).toHaveLength(2);
|
|
447
|
+
expect(await store.listDeliveries({ handler: "handler-b" })).toHaveLength(1);
|
|
448
|
+
expect(await store.listDeliveries({ status: "pending", handler: "handler-a" })).toHaveLength(
|
|
449
|
+
2
|
|
450
|
+
);
|
|
451
|
+
expect(await store.listDeliveries({ status: "dead" })).toEqual([]);
|
|
452
|
+
const limited = await store.listDeliveries({ limit: 1 });
|
|
453
|
+
expect(limited).toHaveLength(1);
|
|
454
|
+
expect(limited[0].event.id).toBe(first.id);
|
|
455
|
+
expect(limited[0].createdAt).toEqual(at(1));
|
|
456
|
+
});
|
|
457
|
+
it("reports counts and the oldest due delivery", async () => {
|
|
458
|
+
await append([
|
|
459
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(100))] },
|
|
460
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(200))] },
|
|
461
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-b", at(5e4))] },
|
|
462
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-c", at(0))] }
|
|
463
|
+
]);
|
|
464
|
+
const [processing] = await claim(["handler-c"], at(0));
|
|
465
|
+
const [dead] = await claim(["handler-a"], at(100), 1);
|
|
466
|
+
await store.bury(dead, { name: "Error", message: "x" }, at(101));
|
|
467
|
+
expect(await store.statistics(at(1e3))).toEqual({
|
|
468
|
+
pending: 2,
|
|
469
|
+
processing: 1,
|
|
470
|
+
completed: 0,
|
|
471
|
+
dead: 1,
|
|
472
|
+
oldestDueAt: at(200)
|
|
473
|
+
});
|
|
474
|
+
expect((await store.statistics(at(LEASE_MS))).oldestDueAt).toEqual(at(200));
|
|
475
|
+
await store.complete(processing, at(1));
|
|
476
|
+
expect(await store.statistics(at(0))).toMatchObject({
|
|
477
|
+
completed: keepsCompleted ? 1 : 0,
|
|
478
|
+
processing: 0,
|
|
479
|
+
oldestDueAt: null
|
|
768
480
|
});
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
481
|
+
});
|
|
482
|
+
it("counts an expired lease as due", async () => {
|
|
483
|
+
await append([
|
|
484
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a", at(LEASE_MS * 5))] }
|
|
485
|
+
]);
|
|
486
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
|
|
487
|
+
await claim(["handler-b"], at(0));
|
|
488
|
+
expect((await store.statistics(at(LEASE_MS + 1))).oldestDueAt).toEqual(at(LEASE_MS));
|
|
489
|
+
});
|
|
490
|
+
});
|
|
491
|
+
describe("purge", () => {
|
|
492
|
+
async function appendDoneFailedShared() {
|
|
493
|
+
const done = createEnvelope();
|
|
494
|
+
const failed = createEnvelope();
|
|
495
|
+
const shared = createEnvelope();
|
|
496
|
+
await append([
|
|
497
|
+
{ event: done, deliveries: [createDelivery("handler-a")] },
|
|
498
|
+
{ event: failed, deliveries: [createDelivery("handler-b")] },
|
|
499
|
+
{ event: shared, deliveries: [createDelivery("handler-c"), createDelivery("handler-d")] }
|
|
500
|
+
]);
|
|
501
|
+
const [doneDelivery] = await claim(["handler-a"]);
|
|
502
|
+
await store.complete(doneDelivery, at(10));
|
|
503
|
+
const [failedDelivery] = await claim(["handler-b"]);
|
|
504
|
+
await store.bury(failedDelivery, { name: "Error", message: "x" }, at(10));
|
|
505
|
+
const [sharedDelivery] = await claim(["handler-c"]);
|
|
506
|
+
await store.complete(sharedDelivery, at(10));
|
|
507
|
+
return { shared };
|
|
508
|
+
}
|
|
509
|
+
it.runIf(keepsCompleted)(
|
|
510
|
+
"deletes old completed and dead deliveries and their orphaned events",
|
|
511
|
+
async () => {
|
|
512
|
+
const { shared } = await appendDoneFailedShared();
|
|
513
|
+
expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
|
|
514
|
+
expect(await store.purge({ completedBefore: at(10) })).toEqual({
|
|
780
515
|
deletedDeliveries: 0,
|
|
781
516
|
deletedEvents: 0
|
|
782
517
|
});
|
|
783
|
-
expect(await store.
|
|
784
|
-
|
|
518
|
+
expect(await store.purge({ completedBefore: at(11) })).toEqual({
|
|
519
|
+
deletedDeliveries: 2,
|
|
520
|
+
deletedEvents: 1
|
|
521
|
+
});
|
|
522
|
+
expect(await store.purge({ deadBefore: at(11) })).toEqual({
|
|
523
|
+
deletedDeliveries: 1,
|
|
524
|
+
deletedEvents: 1
|
|
525
|
+
});
|
|
526
|
+
const remaining = await store.listDeliveries({});
|
|
527
|
+
expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
|
|
528
|
+
expect(remaining[0].event.id).toBe(shared.id);
|
|
529
|
+
}
|
|
530
|
+
);
|
|
531
|
+
it.runIf(!keepsCompleted)(
|
|
532
|
+
"deletes old dead deliveries and the events no delivery references any more",
|
|
533
|
+
async () => {
|
|
534
|
+
const { shared } = await appendDoneFailedShared();
|
|
535
|
+
expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
|
|
536
|
+
expect(await store.purge({ completedBefore: at(0) })).toEqual({
|
|
537
|
+
deletedDeliveries: 0,
|
|
538
|
+
deletedEvents: 0
|
|
539
|
+
});
|
|
540
|
+
expect(await store.purge({ completedBefore: at(11) })).toEqual({
|
|
541
|
+
deletedDeliveries: 0,
|
|
542
|
+
deletedEvents: 1
|
|
543
|
+
});
|
|
544
|
+
expect(await store.purge({ deadBefore: at(11) })).toEqual({
|
|
545
|
+
deletedDeliveries: 1,
|
|
546
|
+
deletedEvents: 1
|
|
547
|
+
});
|
|
548
|
+
const remaining = await store.listDeliveries({});
|
|
549
|
+
expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
|
|
550
|
+
expect(remaining[0].event.id).toBe(shared.id);
|
|
551
|
+
}
|
|
552
|
+
);
|
|
553
|
+
it("lets a purged event id be appended again", async () => {
|
|
554
|
+
const event = createEnvelope();
|
|
555
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
556
|
+
const [delivery] = await claim(["handler-a"]);
|
|
557
|
+
await store.complete(delivery, at(1));
|
|
558
|
+
await store.purge({ completedBefore: at(2) });
|
|
559
|
+
const result = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
560
|
+
expect(result.appendedEventIds).toEqual([event.id]);
|
|
785
561
|
});
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
562
|
+
});
|
|
563
|
+
describe("release", () => {
|
|
564
|
+
it("gives a leased delivery back without counting the attempt", async () => {
|
|
565
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
566
|
+
const [delivery] = await claim(["handler-a"], at(0));
|
|
567
|
+
expect(await store.release(delivery, at(500))).toBe(true);
|
|
568
|
+
const [record] = await store.listDeliveries({});
|
|
569
|
+
expect(record).toMatchObject({
|
|
570
|
+
status: "pending",
|
|
571
|
+
attempts: 0,
|
|
572
|
+
leaseOwner: null,
|
|
573
|
+
leaseExpiresAt: null
|
|
574
|
+
});
|
|
575
|
+
expect(record.availableAt).toEqual(at(500));
|
|
576
|
+
expect(record.updatedAt).toEqual(at(500));
|
|
577
|
+
expect(await store.release(delivery, at(501))).toBe(false);
|
|
578
|
+
const [again] = await claim(["handler-a"], at(500));
|
|
579
|
+
expect(again.attempt).toBe(1);
|
|
580
|
+
});
|
|
581
|
+
});
|
|
582
|
+
describe("transactions in flight", () => {
|
|
583
|
+
it("hides uncommitted appends from every reader", async () => {
|
|
584
|
+
const inside = createDeferred();
|
|
585
|
+
const finish = createDeferred();
|
|
586
|
+
const event = createEnvelope();
|
|
587
|
+
const open = store.transaction(async (transaction) => {
|
|
588
|
+
await store.append(
|
|
589
|
+
transaction,
|
|
590
|
+
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
591
|
+
at(0)
|
|
592
|
+
);
|
|
593
|
+
inside.resolve();
|
|
594
|
+
await finish.promise;
|
|
595
|
+
});
|
|
596
|
+
await inside.promise;
|
|
597
|
+
expect(await claim(["handler-a"])).toEqual([]);
|
|
598
|
+
expect(await store.listDeliveries({})).toEqual([]);
|
|
599
|
+
expect((await store.statistics(at(0))).pending).toBe(0);
|
|
600
|
+
finish.resolve();
|
|
601
|
+
await open;
|
|
602
|
+
expect(await claim(["handler-a"])).toHaveLength(1);
|
|
603
|
+
});
|
|
604
|
+
it("orders same-key events by commit, not by first write", async () => {
|
|
605
|
+
const first = createEnvelope({ key: "race" });
|
|
606
|
+
const second = createEnvelope({ key: "race" });
|
|
607
|
+
const firstWritten = createDeferred();
|
|
608
|
+
const releaseFirst = createDeferred();
|
|
609
|
+
const firstTransaction = store.transaction(async (transaction) => {
|
|
610
|
+
await store.append(
|
|
611
|
+
transaction,
|
|
612
|
+
[{ event: first, deliveries: [createDelivery("handler-a")] }],
|
|
613
|
+
at(0)
|
|
614
|
+
);
|
|
615
|
+
firstWritten.resolve();
|
|
616
|
+
await releaseFirst.promise;
|
|
617
|
+
});
|
|
618
|
+
await firstWritten.promise;
|
|
619
|
+
const secondTransaction = store.transaction(
|
|
620
|
+
(transaction) => store.append(
|
|
621
|
+
transaction,
|
|
622
|
+
[{ event: second, deliveries: [createDelivery("handler-a")] }],
|
|
623
|
+
at(0)
|
|
624
|
+
)
|
|
625
|
+
);
|
|
626
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
627
|
+
releaseFirst.resolve();
|
|
628
|
+
await firstTransaction;
|
|
629
|
+
await secondTransaction;
|
|
630
|
+
const [head] = await claim(["handler-a"]);
|
|
631
|
+
expect(head.event.id).toBe(first.id);
|
|
632
|
+
await store.complete(head, at(1));
|
|
633
|
+
expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
|
|
634
|
+
});
|
|
635
|
+
it("stores an event id written by two concurrent transactions once", async () => {
|
|
636
|
+
const event = createEnvelope({ key: "same" });
|
|
637
|
+
const results = await Promise.all([
|
|
638
|
+
append([{ event, deliveries: [createDelivery("handler-a")] }]),
|
|
639
|
+
append([{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }])
|
|
640
|
+
]);
|
|
641
|
+
const appended = results.flatMap((result) => result.appendedEventIds);
|
|
642
|
+
expect(appended).toEqual([event.id]);
|
|
643
|
+
const records = await store.listDeliveries({});
|
|
644
|
+
expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
|
|
645
|
+
});
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
function orderingAndLeasesContract(suite) {
|
|
649
|
+
const { append, claim } = suite;
|
|
650
|
+
let store;
|
|
651
|
+
beforeEach(() => {
|
|
652
|
+
store = suite.store();
|
|
653
|
+
});
|
|
654
|
+
describe("ordering keys under load", () => {
|
|
655
|
+
it("does not let a blocked key starve unrelated deliveries", async () => {
|
|
656
|
+
const blocked = Array.from({ length: 120 }, () => ({
|
|
657
|
+
event: createEnvelope({ key: "hot" }),
|
|
658
|
+
deliveries: [createDelivery("handler-a", at(0))]
|
|
659
|
+
}));
|
|
660
|
+
await append(blocked);
|
|
661
|
+
const unrelated = Array.from({ length: 5 }, () => ({
|
|
662
|
+
event: createEnvelope(),
|
|
663
|
+
deliveries: [createDelivery("handler-a", at(10))]
|
|
664
|
+
}));
|
|
665
|
+
await append(unrelated);
|
|
666
|
+
const claimed = await claim(["handler-a"], at(100), 10);
|
|
667
|
+
expect(claimed).toHaveLength(6);
|
|
668
|
+
expect(claimed.filter((delivery) => delivery.event.key === "hot")).toHaveLength(1);
|
|
669
|
+
});
|
|
670
|
+
it("keeps successors blocked behind an expired, delayed or requeued head", async () => {
|
|
671
|
+
const head = createEnvelope({ key: "k" });
|
|
672
|
+
const next = createEnvelope({ key: "k" });
|
|
673
|
+
await append([{ event: head, deliveries: [createDelivery("handler-a")] }]);
|
|
674
|
+
await append([{ event: next, deliveries: [createDelivery("handler-a")] }]);
|
|
675
|
+
const [claimedHead] = await claim(["handler-a"], at(0));
|
|
676
|
+
const reclaimed = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
677
|
+
expect(reclaimed.map((delivery) => delivery.event.id)).toEqual([head.id]);
|
|
678
|
+
expect(
|
|
679
|
+
await store.reschedule(
|
|
680
|
+
reclaimed[0],
|
|
681
|
+
at(LEASE_MS * 10),
|
|
682
|
+
{ name: "Error", message: "x" },
|
|
683
|
+
at(LEASE_MS)
|
|
684
|
+
)
|
|
685
|
+
).toBe(true);
|
|
686
|
+
expect(await claim(["handler-a"], at(LEASE_MS * 5))).toEqual([]);
|
|
687
|
+
const [delayedHead] = await claim(["handler-a"], at(LEASE_MS * 10));
|
|
688
|
+
await store.bury(delayedHead, { name: "Error", message: "x" }, at(LEASE_MS * 10));
|
|
689
|
+
expect(await store.requeue([delayedHead.deliveryId], at(LEASE_MS * 11))).toBe(1);
|
|
690
|
+
const [requeuedHead] = await claim(["handler-a"], at(LEASE_MS * 11));
|
|
691
|
+
expect(requeuedHead.event.id).toBe(head.id);
|
|
692
|
+
expect(await claim(["handler-a"], at(LEASE_MS * 11))).toEqual([]);
|
|
693
|
+
expect(claimedHead).toBeDefined();
|
|
694
|
+
});
|
|
695
|
+
});
|
|
696
|
+
describe("requeued keyed dead letters", () => {
|
|
697
|
+
it("never runs a requeued dead letter alongside its running successor", async () => {
|
|
698
|
+
const first = createEnvelope({ key: "k" });
|
|
699
|
+
const second = createEnvelope({ key: "k" });
|
|
700
|
+
await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
|
|
701
|
+
await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
|
|
702
|
+
const [head] = await claim(["handler-a"], at(0));
|
|
703
|
+
await store.bury(head, { name: "Error", message: "x" }, at(1));
|
|
704
|
+
const [successor] = await claim(["handler-a"], at(2));
|
|
705
|
+
expect(successor.event.id).toBe(second.id);
|
|
706
|
+
expect(await store.requeue([head.deliveryId], at(3))).toBe(1);
|
|
707
|
+
expect(await claim(["handler-a"], at(4))).toEqual([]);
|
|
708
|
+
await store.complete(successor, at(5));
|
|
709
|
+
const [requeued] = await claim(["handler-a"], at(6));
|
|
710
|
+
expect(requeued.event.id).toBe(first.id);
|
|
711
|
+
});
|
|
712
|
+
it("recovers when the running successor crashes after the requeue", async () => {
|
|
713
|
+
const first = createEnvelope({ key: "k" });
|
|
714
|
+
const second = createEnvelope({ key: "k" });
|
|
715
|
+
await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
|
|
716
|
+
await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
|
|
717
|
+
const [head] = await claim(["handler-a"], at(0));
|
|
718
|
+
await store.bury(head, { name: "Error", message: "x" }, at(1));
|
|
719
|
+
await claim(["handler-a"], at(2));
|
|
720
|
+
await store.requeue([head.deliveryId], at(3));
|
|
721
|
+
const recovered = await claim(["handler-a"], at(2 + LEASE_MS));
|
|
722
|
+
expect(recovered.map((delivery) => delivery.event.id)).toEqual([first.id]);
|
|
723
|
+
await store.complete(recovered[0], at(3 + LEASE_MS));
|
|
724
|
+
const [retried] = await claim(["handler-a"], at(4 + LEASE_MS));
|
|
725
|
+
expect(retried).toMatchObject({ attempt: 2 });
|
|
726
|
+
expect(retried.event.id).toBe(second.id);
|
|
727
|
+
});
|
|
728
|
+
});
|
|
729
|
+
describe("lease semantics", () => {
|
|
730
|
+
it("confirms a repeated renewal with identical values", async () => {
|
|
731
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
732
|
+
const [delivery] = await claim(["handler-a"], at(0));
|
|
733
|
+
expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
|
|
734
|
+
expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
|
|
735
|
+
});
|
|
736
|
+
it("accepts an expired lease that nobody reclaimed", async () => {
|
|
737
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
738
|
+
const [delivery] = await claim(["handler-a"], at(0));
|
|
739
|
+
expect(await store.extendLease(delivery, at(LEASE_MS * 3), at(LEASE_MS * 2))).toBe(true);
|
|
740
|
+
expect(await store.complete(delivery, at(LEASE_MS * 2))).toBe(true);
|
|
741
|
+
});
|
|
742
|
+
it("rejects every lease operation once the delivery left processing", async () => {
|
|
743
|
+
const error = { name: "Error", message: "x" };
|
|
744
|
+
await append([
|
|
745
|
+
{
|
|
746
|
+
event: createEnvelope(),
|
|
747
|
+
deliveries: [createDelivery("a"), createDelivery("b"), createDelivery("c")]
|
|
794
748
|
}
|
|
749
|
+
]);
|
|
750
|
+
const [completed] = await claim(["a"]);
|
|
751
|
+
await store.complete(completed, at(1));
|
|
752
|
+
const [buried] = await claim(["b"]);
|
|
753
|
+
await store.bury(buried, error, at(1));
|
|
754
|
+
const [rescheduled] = await claim(["c"]);
|
|
755
|
+
await store.reschedule(rescheduled, at(100), error, at(1));
|
|
756
|
+
for (const lease of [completed, buried, rescheduled]) {
|
|
757
|
+
expect(await store.extendLease(lease, at(LEASE_MS), at(2))).toBe(false);
|
|
758
|
+
expect(await store.reschedule(lease, at(3), error, at(2))).toBe(false);
|
|
759
|
+
expect(await store.bury(lease, error, at(2))).toBe(false);
|
|
760
|
+
expect(await store.release(lease, at(2))).toBe(false);
|
|
761
|
+
expect(await store.complete(lease, at(2))).toBe(false);
|
|
762
|
+
}
|
|
763
|
+
});
|
|
764
|
+
it("rejects made-up leases on pending deliveries and requeues only dead ones", async () => {
|
|
765
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("a")] }]);
|
|
766
|
+
const [pending] = await store.listDeliveries({});
|
|
767
|
+
const fake = { deliveryId: pending.id, leaseToken: createUuidV7() };
|
|
768
|
+
expect(await store.reschedule(fake, at(1), { name: "Error", message: "x" }, at(1))).toBe(
|
|
769
|
+
false
|
|
795
770
|
);
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
event: createEnvelope(),
|
|
812
|
-
deliveries: [createDelivery("handler-a")]
|
|
813
|
-
}))
|
|
814
|
-
);
|
|
815
|
-
expect(await store.listDeliveries({})).toHaveLength(100);
|
|
816
|
-
});
|
|
817
|
-
it("hands out copies that callers cannot use to change stored data", async () => {
|
|
818
|
-
const event = createEnvelope({ payload: { amount: 1 }, headers: { a: "1" } });
|
|
771
|
+
expect(await store.bury(fake, { name: "Error", message: "x" }, at(1))).toBe(false);
|
|
772
|
+
const [processing] = await claim(["a"]);
|
|
773
|
+
expect(await store.requeue([processing.deliveryId], at(1))).toBe(0);
|
|
774
|
+
expect(await store.purge({ completedBefore: at(1e9), deadBefore: at(1e9) })).toEqual({
|
|
775
|
+
deletedDeliveries: 0,
|
|
776
|
+
deletedEvents: 0
|
|
777
|
+
});
|
|
778
|
+
expect(await store.listDeliveries({})).toHaveLength(1);
|
|
779
|
+
});
|
|
780
|
+
});
|
|
781
|
+
describe("data fidelity", () => {
|
|
782
|
+
it.each([null, 0, 42.5, "text", [1, { two: 2 }], { deep: { list: [] } }, true])(
|
|
783
|
+
"round-trips the payload %j",
|
|
784
|
+
async (payload) => {
|
|
785
|
+
const event = createEnvelope({ payload });
|
|
819
786
|
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
820
787
|
const [claimed] = await claim(["handler-a"]);
|
|
821
|
-
claimed.event.payload.
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
788
|
+
expect(claimed.event.payload).toEqual(payload);
|
|
789
|
+
}
|
|
790
|
+
);
|
|
791
|
+
it("accepts long identifiers and non-UUID event ids", async () => {
|
|
792
|
+
const handler = "h".repeat(255);
|
|
793
|
+
const event = createEnvelope({
|
|
794
|
+
id: `order-42:placed:${"x".repeat(200)}`,
|
|
795
|
+
key: "k".repeat(255),
|
|
796
|
+
type: "t".repeat(255)
|
|
797
|
+
});
|
|
798
|
+
await append([{ event, deliveries: [createDelivery(handler)] }]);
|
|
799
|
+
const [claimed] = await claim([handler]);
|
|
800
|
+
expect(claimed.event).toEqual(event);
|
|
801
|
+
expect(claimed.handler).toBe(handler);
|
|
827
802
|
});
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
])
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
duplicateEventIds: [event.id],
|
|
869
|
-
appendedDeliveryCount: 1
|
|
870
|
-
});
|
|
871
|
-
expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
|
|
872
|
-
expect(await claim(["handler-b"], at(LEASE_MS * 10))).toHaveLength(1);
|
|
873
|
-
const again = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
874
|
-
expect(again.appendedDeliveryCount).toBe(0);
|
|
875
|
-
});
|
|
876
|
-
it("completes many leases at once and reports each one", async () => {
|
|
877
|
-
await append(
|
|
878
|
-
Array.from({ length: 5 }, () => ({
|
|
879
|
-
event: createEnvelope(),
|
|
880
|
-
deliveries: [createDelivery("handler-a")]
|
|
881
|
-
}))
|
|
882
|
-
);
|
|
883
|
-
const claimed = await claim(["handler-a"]);
|
|
884
|
-
expect(claimed).toHaveLength(5);
|
|
885
|
-
expect(await completeMany(store, [], at(1))).toEqual([]);
|
|
886
|
-
expect(await completeMany(store, claimed.slice(0, 3), at(1))).toEqual([true, true, true]);
|
|
887
|
-
expect(await store.statistics(at(2))).toMatchObject({ processing: 2, pending: 0 });
|
|
888
|
-
expect(await completeMany(store, claimed, at(2))).toEqual([
|
|
889
|
-
false,
|
|
890
|
-
false,
|
|
891
|
-
false,
|
|
892
|
-
true,
|
|
893
|
-
true
|
|
803
|
+
it("lists at most 100 deliveries by default", async () => {
|
|
804
|
+
await append(
|
|
805
|
+
Array.from({ length: 101 }, () => ({
|
|
806
|
+
event: createEnvelope(),
|
|
807
|
+
deliveries: [createDelivery("handler-a")]
|
|
808
|
+
}))
|
|
809
|
+
);
|
|
810
|
+
expect(await store.listDeliveries({})).toHaveLength(100);
|
|
811
|
+
});
|
|
812
|
+
it("hands out copies that callers cannot use to change stored data", async () => {
|
|
813
|
+
const event = createEnvelope({ payload: { amount: 1 }, headers: { a: "1" } });
|
|
814
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
815
|
+
const [claimed] = await claim(["handler-a"]);
|
|
816
|
+
claimed.event.payload.amount = 999;
|
|
817
|
+
claimed.event.headers["a"] = "changed";
|
|
818
|
+
const [listed] = await store.listDeliveries({});
|
|
819
|
+
expect(listed.event.payload).toEqual({ amount: 1 });
|
|
820
|
+
expect(listed.event.headers).toEqual({ a: "1" });
|
|
821
|
+
});
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
function completionContract(suite) {
|
|
825
|
+
const { append, claim, completeMany, keepsCompleted } = suite;
|
|
826
|
+
let store;
|
|
827
|
+
beforeEach(() => {
|
|
828
|
+
store = suite.store();
|
|
829
|
+
});
|
|
830
|
+
describe("completion", () => {
|
|
831
|
+
it("removes a completed delivery from every listing", async () => {
|
|
832
|
+
const event = createEnvelope();
|
|
833
|
+
await append([
|
|
834
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
835
|
+
]);
|
|
836
|
+
const [delivery] = await claim(["handler-a"]);
|
|
837
|
+
expect(await store.complete(delivery, at(5))).toBe(true);
|
|
838
|
+
const records = await store.listDeliveries({ eventId: event.id });
|
|
839
|
+
if (keepsCompleted) {
|
|
840
|
+
expect(records.map((record) => [record.handler, record.status]).sort()).toEqual([
|
|
841
|
+
["handler-a", "completed"],
|
|
842
|
+
["handler-b", "pending"]
|
|
894
843
|
]);
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
});
|
|
902
|
-
it("fences every lease of a batch on its own", async () => {
|
|
903
|
-
await append(
|
|
904
|
-
Array.from({ length: 3 }, () => ({
|
|
905
|
-
event: createEnvelope(),
|
|
906
|
-
deliveries: [createDelivery("handler-a")]
|
|
907
|
-
}))
|
|
908
|
-
);
|
|
909
|
-
const stale = await claim(["handler-a"], at(0));
|
|
910
|
-
const current = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
911
|
-
expect(current).toHaveLength(3);
|
|
912
|
-
await append(
|
|
913
|
-
[{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }],
|
|
914
|
-
at(1)
|
|
915
|
-
);
|
|
916
|
-
const [fresh] = await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2");
|
|
917
|
-
expect(
|
|
918
|
-
await completeMany(store, [stale[0], fresh, stale[1], current[2]], at(LEASE_MS + 2))
|
|
919
|
-
).toEqual([false, true, false, true]);
|
|
920
|
-
expect(await store.statistics(at(LEASE_MS + 3))).toMatchObject({ processing: 2 });
|
|
921
|
-
});
|
|
922
|
-
it("does not complete a lease of a delivery that went back to pending", async () => {
|
|
923
|
-
const error = { name: "Error", message: "x" };
|
|
924
|
-
await append([
|
|
925
|
-
{ event: createEnvelope(), deliveries: [createDelivery("a"), createDelivery("b")] }
|
|
844
|
+
const [completed] = await store.listDeliveries({ status: "completed" });
|
|
845
|
+
expect(completed).toMatchObject({ handler: "handler-a", attempts: 1 });
|
|
846
|
+
expect(completed.completedAt).toEqual(at(5));
|
|
847
|
+
} else {
|
|
848
|
+
expect(records.map((record) => [record.handler, record.status])).toEqual([
|
|
849
|
+
["handler-b", "pending"]
|
|
926
850
|
]);
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
851
|
+
expect(await store.listDeliveries({ status: "completed" })).toEqual([]);
|
|
852
|
+
}
|
|
853
|
+
expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
|
|
854
|
+
expect(await store.statistics(at(0))).toMatchObject({
|
|
855
|
+
pending: 1,
|
|
856
|
+
processing: 0,
|
|
857
|
+
completed: keepsCompleted ? 1 : 0
|
|
933
858
|
});
|
|
934
859
|
});
|
|
935
|
-
|
|
860
|
+
it("does not create the delivery of a handler that already completed again", async () => {
|
|
861
|
+
const event = createEnvelope();
|
|
862
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
863
|
+
const [delivery] = await claim(["handler-a"]);
|
|
864
|
+
await store.complete(delivery, at(1));
|
|
865
|
+
const result = await append([
|
|
866
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
867
|
+
]);
|
|
868
|
+
expect(result).toEqual({
|
|
869
|
+
appendedEventIds: [],
|
|
870
|
+
duplicateEventIds: [event.id],
|
|
871
|
+
appendedDeliveryCount: 1
|
|
872
|
+
});
|
|
873
|
+
expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
|
|
874
|
+
expect(await claim(["handler-b"], at(LEASE_MS * 10))).toHaveLength(1);
|
|
875
|
+
const again = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
876
|
+
expect(again.appendedDeliveryCount).toBe(0);
|
|
877
|
+
});
|
|
878
|
+
it("completes many leases at once and reports each one", async () => {
|
|
879
|
+
await append(
|
|
880
|
+
Array.from({ length: 5 }, () => ({
|
|
881
|
+
event: createEnvelope(),
|
|
882
|
+
deliveries: [createDelivery("handler-a")]
|
|
883
|
+
}))
|
|
884
|
+
);
|
|
885
|
+
const claimed = await claim(["handler-a"]);
|
|
886
|
+
expect(claimed).toHaveLength(5);
|
|
887
|
+
expect(await completeMany(store, [], at(1))).toEqual([]);
|
|
888
|
+
expect(await completeMany(store, claimed.slice(0, 3), at(1))).toEqual([true, true, true]);
|
|
889
|
+
expect(await store.statistics(at(2))).toMatchObject({ processing: 2, pending: 0 });
|
|
890
|
+
expect(await completeMany(store, claimed, at(2))).toEqual([false, false, false, true, true]);
|
|
891
|
+
expect(await store.statistics(at(3))).toMatchObject({ processing: 0, pending: 0 });
|
|
892
|
+
});
|
|
893
|
+
it("completes a lease listed twice only once", async () => {
|
|
894
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
895
|
+
const [delivery] = await claim(["handler-a"]);
|
|
896
|
+
expect(await completeMany(store, [delivery, delivery], at(1))).toEqual([true, false]);
|
|
897
|
+
});
|
|
898
|
+
it("fences every lease of a batch on its own", async () => {
|
|
899
|
+
await append(
|
|
900
|
+
Array.from({ length: 3 }, () => ({
|
|
901
|
+
event: createEnvelope(),
|
|
902
|
+
deliveries: [createDelivery("handler-a")]
|
|
903
|
+
}))
|
|
904
|
+
);
|
|
905
|
+
const stale = await claim(["handler-a"], at(0));
|
|
906
|
+
const current = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
907
|
+
expect(current).toHaveLength(3);
|
|
908
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }], at(1));
|
|
909
|
+
const [fresh] = await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2");
|
|
910
|
+
expect(
|
|
911
|
+
await completeMany(store, [stale[0], fresh, stale[1], current[2]], at(LEASE_MS + 2))
|
|
912
|
+
).toEqual([false, true, false, true]);
|
|
913
|
+
expect(await store.statistics(at(LEASE_MS + 3))).toMatchObject({ processing: 2 });
|
|
914
|
+
});
|
|
915
|
+
it("does not complete a lease of a delivery that went back to pending", async () => {
|
|
936
916
|
const error = { name: "Error", message: "x" };
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
917
|
+
await append([
|
|
918
|
+
{ event: createEnvelope(), deliveries: [createDelivery("a"), createDelivery("b")] }
|
|
919
|
+
]);
|
|
920
|
+
const [retrying] = await claim(["a"]);
|
|
921
|
+
const [released] = await claim(["b"]);
|
|
922
|
+
await store.reschedule(retrying, at(1e3), error, at(1));
|
|
923
|
+
await store.release(released, at(1));
|
|
924
|
+
expect(await completeMany(store, [retrying, released], at(2))).toEqual([false, false]);
|
|
925
|
+
expect(await store.statistics(at(2))).toMatchObject({ pending: 2, processing: 0 });
|
|
926
|
+
});
|
|
927
|
+
});
|
|
928
|
+
describe("stale leases of one delivery", () => {
|
|
929
|
+
const error = { name: "Error", message: "x" };
|
|
930
|
+
it("refuses the lease of an attempt that was released when the same owner claims it again", async () => {
|
|
931
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
932
|
+
const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
933
|
+
expect(await store.release(stale, at(1))).toBe(true);
|
|
934
|
+
const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
|
|
935
|
+
expect(current.deliveryId).toBe(stale.deliveryId);
|
|
936
|
+
expect(current.leaseToken).not.toBe(stale.leaseToken);
|
|
937
|
+
expect(await store.complete(stale, at(3))).toBe(false);
|
|
938
|
+
expect(await store.bury(stale, error, at(3))).toBe(false);
|
|
939
|
+
expect(await store.reschedule(stale, at(9), error, at(3))).toBe(false);
|
|
940
|
+
expect(await store.release(stale, at(3))).toBe(false);
|
|
941
|
+
expect(await store.extendLease(stale, at(LEASE_MS * 3), at(3))).toBe(false);
|
|
942
|
+
expect(await store.complete(current, at(4))).toBe(true);
|
|
943
|
+
});
|
|
944
|
+
it("refuses the lease of an attempt that was rescheduled when the same owner claims it again", async () => {
|
|
945
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
946
|
+
const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
947
|
+
expect(await store.reschedule(stale, at(10), error, at(1))).toBe(true);
|
|
948
|
+
const [current] = await claim(["handler-a"], at(10), 100, "worker-1");
|
|
949
|
+
expect(current).toMatchObject({ deliveryId: stale.deliveryId, attempt: 2 });
|
|
950
|
+
expect(await store.complete(stale, at(11))).toBe(false);
|
|
951
|
+
expect(await store.bury(stale, error, at(11))).toBe(false);
|
|
952
|
+
expect(await completeMany(store, [stale], at(11))).toEqual([false]);
|
|
953
|
+
expect(await store.complete(current, at(12))).toBe(true);
|
|
954
|
+
});
|
|
955
|
+
it("matches the results of completeMany to the lease, not to the delivery id", async () => {
|
|
956
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
957
|
+
const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
958
|
+
await store.release(stale, at(1));
|
|
959
|
+
const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
|
|
960
|
+
expect(await completeMany(store, [stale, current], at(3))).toEqual([false, true]);
|
|
961
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
|
|
962
|
+
const [staleB] = await claim(["handler-b"], at(4), 100, "worker-1");
|
|
963
|
+
await store.release(staleB, at(5));
|
|
964
|
+
const [currentB] = await claim(["handler-b"], at(6), 100, "worker-1");
|
|
965
|
+
expect(await completeMany(store, [currentB, staleB], at(7))).toEqual([true, false]);
|
|
966
|
+
});
|
|
967
|
+
});
|
|
968
|
+
describe("retention of the deduplication record", () => {
|
|
969
|
+
const DAY = 864e5;
|
|
970
|
+
it("keeps an event for the retention window after its last delivery completed, however old it is", async () => {
|
|
971
|
+
const event = createEnvelope();
|
|
972
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
973
|
+
const [delivery] = await claim(["handler-a"], at(30 * DAY));
|
|
974
|
+
expect(await store.complete(delivery, at(30 * DAY + 1))).toBe(true);
|
|
975
|
+
expect(await store.purge({ completedBefore: at(29 * DAY) })).toMatchObject({
|
|
976
|
+
deletedEvents: 0
|
|
977
|
+
});
|
|
978
|
+
const again = await append(
|
|
979
|
+
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
980
|
+
at(30 * DAY + 2)
|
|
981
|
+
);
|
|
982
|
+
expect(again.appendedDeliveryCount).toBe(0);
|
|
983
|
+
expect(await store.purge({ completedBefore: at(31 * DAY) })).toMatchObject({
|
|
984
|
+
deletedEvents: 1
|
|
973
985
|
});
|
|
986
|
+
const afterwards = await append(
|
|
987
|
+
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
988
|
+
at(31 * DAY)
|
|
989
|
+
);
|
|
990
|
+
expect(afterwards.appendedEventIds).toEqual([event.id]);
|
|
974
991
|
});
|
|
975
|
-
|
|
976
|
-
const
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
992
|
+
it("keeps the later completion time of an event when completions are recorded out of order", async () => {
|
|
993
|
+
const event = createEnvelope();
|
|
994
|
+
await append([
|
|
995
|
+
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
996
|
+
]);
|
|
997
|
+
const claimed = await claim(["handler-a", "handler-b"]);
|
|
998
|
+
expect(claimed).toHaveLength(2);
|
|
999
|
+
const [first, second] = claimed.sort(
|
|
1000
|
+
(left, right) => left.handler.localeCompare(right.handler)
|
|
1001
|
+
);
|
|
1002
|
+
expect(await store.complete(first, at(200))).toBe(true);
|
|
1003
|
+
expect(await store.complete(second, at(100))).toBe(true);
|
|
1004
|
+
expect(await store.purge({ completedBefore: at(150) })).toMatchObject({ deletedEvents: 0 });
|
|
1005
|
+
const again = await append(
|
|
1006
|
+
[{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
|
|
1007
|
+
at(201)
|
|
1008
|
+
);
|
|
1009
|
+
expect(again.appendedDeliveryCount).toBe(0);
|
|
1010
|
+
});
|
|
1011
|
+
it("remembers every handler of an event appended twice in one transaction", async () => {
|
|
1012
|
+
const event = createEnvelope();
|
|
1013
|
+
const result = await store.transaction(async (transaction) => {
|
|
1014
|
+
await store.append(
|
|
1015
|
+
transaction,
|
|
994
1016
|
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
995
|
-
at(
|
|
1017
|
+
at(0)
|
|
996
1018
|
);
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
await append([
|
|
1002
|
-
{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
|
|
1003
|
-
]);
|
|
1004
|
-
const claimed = await claim(["handler-a", "handler-b"]);
|
|
1005
|
-
expect(claimed).toHaveLength(2);
|
|
1006
|
-
const [first, second] = claimed.sort(
|
|
1007
|
-
(left, right) => left.handler.localeCompare(right.handler)
|
|
1019
|
+
return store.append(
|
|
1020
|
+
transaction,
|
|
1021
|
+
[{ event, deliveries: [createDelivery("handler-b")] }],
|
|
1022
|
+
at(0)
|
|
1008
1023
|
);
|
|
1009
|
-
expect(await store.complete(first, at(200))).toBe(true);
|
|
1010
|
-
expect(await store.complete(second, at(100))).toBe(true);
|
|
1011
|
-
expect(await store.purge({ completedBefore: at(150) })).toMatchObject({ deletedEvents: 0 });
|
|
1012
|
-
const again = await append(
|
|
1013
|
-
[{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
|
|
1014
|
-
at(201)
|
|
1015
|
-
);
|
|
1016
|
-
expect(again.appendedDeliveryCount).toBe(0);
|
|
1017
|
-
});
|
|
1018
|
-
it("remembers every handler of an event appended twice in one transaction", async () => {
|
|
1019
|
-
const event = createEnvelope();
|
|
1020
|
-
const result = await store.transaction(async (transaction) => {
|
|
1021
|
-
await store.append(
|
|
1022
|
-
transaction,
|
|
1023
|
-
[{ event, deliveries: [createDelivery("handler-a")] }],
|
|
1024
|
-
at(0)
|
|
1025
|
-
);
|
|
1026
|
-
return store.append(
|
|
1027
|
-
transaction,
|
|
1028
|
-
[{ event, deliveries: [createDelivery("handler-b")] }],
|
|
1029
|
-
at(0)
|
|
1030
|
-
);
|
|
1031
|
-
});
|
|
1032
|
-
expect(result).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 1 });
|
|
1033
|
-
const claimed = await claim(["handler-a", "handler-b"]);
|
|
1034
|
-
expect(claimed).toHaveLength(2);
|
|
1035
|
-
for (const delivery of claimed) {
|
|
1036
|
-
expect(await store.complete(delivery, at(1))).toBe(true);
|
|
1037
|
-
}
|
|
1038
|
-
const again = await append(
|
|
1039
|
-
[{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
|
|
1040
|
-
at(2)
|
|
1041
|
-
);
|
|
1042
|
-
expect(again).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 0 });
|
|
1043
|
-
});
|
|
1044
|
-
it("does not let the dead-letter cutoff remove events whose deliveries completed", async () => {
|
|
1045
|
-
const event = createEnvelope();
|
|
1046
|
-
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
1047
|
-
const [delivery] = await claim(["handler-a"]);
|
|
1048
|
-
await store.complete(delivery, at(1));
|
|
1049
|
-
expect(await store.purge({ deadBefore: at(100 * DAY) })).toMatchObject({
|
|
1050
|
-
deletedEvents: 0
|
|
1051
|
-
});
|
|
1052
|
-
const again = await append([{ event, deliveries: [createDelivery("handler-a")] }], at(2));
|
|
1053
|
-
expect(again.appendedDeliveryCount).toBe(0);
|
|
1054
1024
|
});
|
|
1025
|
+
expect(result).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 1 });
|
|
1026
|
+
const claimed = await claim(["handler-a", "handler-b"]);
|
|
1027
|
+
expect(claimed).toHaveLength(2);
|
|
1028
|
+
for (const delivery of claimed) {
|
|
1029
|
+
expect(await store.complete(delivery, at(1))).toBe(true);
|
|
1030
|
+
}
|
|
1031
|
+
const again = await append(
|
|
1032
|
+
[{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
|
|
1033
|
+
at(2)
|
|
1034
|
+
);
|
|
1035
|
+
expect(again).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 0 });
|
|
1055
1036
|
});
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
["handler-a"],
|
|
1093
|
-
at(crash * LEASE_MS),
|
|
1094
|
-
100,
|
|
1095
|
-
`worker-${crash}`
|
|
1096
|
-
);
|
|
1097
|
-
expect(delivery.attempt).toBe(crash + 1);
|
|
1098
|
-
}
|
|
1099
|
-
const [last] = await claim(["handler-a"], at(4 * LEASE_MS), 100, "worker-4");
|
|
1100
|
-
expect(last.attempt).toBe(5);
|
|
1101
|
-
expect(
|
|
1102
|
-
await store.bury(last, { name: "Error", message: "poison" }, at(4 * LEASE_MS))
|
|
1103
|
-
).toBe(true);
|
|
1104
|
-
const [dead] = await store.listDeliveries({ status: "dead" });
|
|
1105
|
-
expect(dead).toMatchObject({ attempts: 5, lastError: { message: "poison" } });
|
|
1106
|
-
expect(await claim(["handler-a"], at(100 * LEASE_MS))).toEqual([]);
|
|
1107
|
-
});
|
|
1108
|
-
it("does not count an attempt that was reported as failed twice", async () => {
|
|
1109
|
-
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
1110
|
-
const [delivery] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1111
|
-
await store.reschedule(delivery, at(10), { name: "Error", message: "x" }, at(1));
|
|
1112
|
-
const [retry] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
1113
|
-
expect(retry.attempt).toBe(2);
|
|
1114
|
-
const [afterCrash] = await claim(["handler-a"], at(LEASE_MS * 2), 100, "worker-3");
|
|
1115
|
-
expect(afterCrash.attempt).toBe(3);
|
|
1116
|
-
});
|
|
1117
|
-
it("keeps the order of a key across a takeover", async () => {
|
|
1118
|
-
const [head, next] = await (async () => {
|
|
1119
|
-
const events = [createEnvelope({ key: "k" }), createEnvelope({ key: "k" })];
|
|
1120
|
-
for (const event of events) {
|
|
1121
|
-
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
1122
|
-
}
|
|
1123
|
-
return events;
|
|
1124
|
-
})();
|
|
1125
|
-
const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1126
|
-
expect(stale.event.id).toBe(head.id);
|
|
1127
|
-
const taken = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
1128
|
-
expect(taken.map((delivery) => delivery.event.id)).toEqual([head.id]);
|
|
1037
|
+
it("does not let the dead-letter cutoff remove events whose deliveries completed", async () => {
|
|
1038
|
+
const event = createEnvelope();
|
|
1039
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
1040
|
+
const [delivery] = await claim(["handler-a"]);
|
|
1041
|
+
await store.complete(delivery, at(1));
|
|
1042
|
+
expect(await store.purge({ deadBefore: at(100 * DAY) })).toMatchObject({
|
|
1043
|
+
deletedEvents: 0
|
|
1044
|
+
});
|
|
1045
|
+
const again = await append([{ event, deliveries: [createDelivery("handler-a")] }], at(2));
|
|
1046
|
+
expect(again.appendedDeliveryCount).toBe(0);
|
|
1047
|
+
});
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
function lostLeasesContract(suite) {
|
|
1051
|
+
const { append, claim, completeMany, options } = suite;
|
|
1052
|
+
let store;
|
|
1053
|
+
beforeEach(() => {
|
|
1054
|
+
store = suite.store();
|
|
1055
|
+
});
|
|
1056
|
+
describe("lost leases", () => {
|
|
1057
|
+
it("hands a delivery of a crashed worker to the next claimer with the attempt counted", async () => {
|
|
1058
|
+
await append(
|
|
1059
|
+
Array.from({ length: 4 }, () => ({
|
|
1060
|
+
event: createEnvelope(),
|
|
1061
|
+
deliveries: [createDelivery("handler-a")]
|
|
1062
|
+
}))
|
|
1063
|
+
);
|
|
1064
|
+
const first = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1065
|
+
expect(first.map((delivery) => delivery.attempt)).toEqual([1, 1, 1, 1]);
|
|
1066
|
+
expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
|
|
1067
|
+
const second = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
1068
|
+
expect(second.map((delivery) => delivery.deliveryId).sort()).toEqual(
|
|
1069
|
+
first.map((delivery) => delivery.deliveryId).sort()
|
|
1070
|
+
);
|
|
1071
|
+
expect(second.map((delivery) => delivery.attempt)).toEqual([2, 2, 2, 2]);
|
|
1072
|
+
for (const stale of first) {
|
|
1129
1073
|
expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
deliveries: [createDelivery("handler-a")]
|
|
1139
|
-
}))
|
|
1140
|
-
);
|
|
1141
|
-
const seen = [];
|
|
1142
|
-
for (let round = 0; round < 40; round += 1) {
|
|
1143
|
-
const owners = ["worker-1", "worker-2", "worker-3"];
|
|
1144
|
-
const batches = await Promise.all(
|
|
1145
|
-
owners.map((owner) => claim(["handler-a"], at(round), 4, owner))
|
|
1146
|
-
);
|
|
1147
|
-
for (const batch of batches) {
|
|
1148
|
-
seen.push(...batch.map((delivery) => delivery.deliveryId));
|
|
1149
|
-
expect(await completeMany(store, batch, at(round))).toEqual(batch.map(() => true));
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
expect(seen).toHaveLength(30);
|
|
1153
|
-
expect(new Set(seen).size).toBe(30);
|
|
1154
|
-
});
|
|
1074
|
+
}
|
|
1075
|
+
expect(await completeMany(store, first, at(LEASE_MS + 1))).toEqual([
|
|
1076
|
+
false,
|
|
1077
|
+
false,
|
|
1078
|
+
false,
|
|
1079
|
+
false
|
|
1080
|
+
]);
|
|
1081
|
+
expect(await completeMany(store, second, at(LEASE_MS + 2))).toEqual([true, true, true, true]);
|
|
1155
1082
|
});
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1083
|
+
it("keeps counting attempts of a poison delivery until it is dead-lettered", async () => {
|
|
1084
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
1085
|
+
for (let crash = 0; crash < 4; crash += 1) {
|
|
1086
|
+
const [delivery] = await claim(["handler-a"], at(crash * LEASE_MS), 100, `worker-${crash}`);
|
|
1087
|
+
expect(delivery.attempt).toBe(crash + 1);
|
|
1088
|
+
}
|
|
1089
|
+
const [last] = await claim(["handler-a"], at(4 * LEASE_MS), 100, "worker-4");
|
|
1090
|
+
expect(last.attempt).toBe(5);
|
|
1091
|
+
expect(await store.bury(last, { name: "Error", message: "poison" }, at(4 * LEASE_MS))).toBe(
|
|
1092
|
+
true
|
|
1093
|
+
);
|
|
1094
|
+
const [dead] = await store.listDeliveries({ status: "dead" });
|
|
1095
|
+
expect(dead).toMatchObject({ attempts: 5, lastError: { message: "poison" } });
|
|
1096
|
+
expect(await claim(["handler-a"], at(100 * LEASE_MS))).toEqual([]);
|
|
1097
|
+
});
|
|
1098
|
+
it("does not count an attempt that was reported as failed twice", async () => {
|
|
1099
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
1100
|
+
const [delivery] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1101
|
+
await store.reschedule(delivery, at(10), { name: "Error", message: "x" }, at(1));
|
|
1102
|
+
const [retry] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
1103
|
+
expect(retry.attempt).toBe(2);
|
|
1104
|
+
const [afterCrash] = await claim(["handler-a"], at(LEASE_MS * 2), 100, "worker-3");
|
|
1105
|
+
expect(afterCrash.attempt).toBe(3);
|
|
1106
|
+
});
|
|
1107
|
+
it("keeps the order of a key across a takeover", async () => {
|
|
1108
|
+
const [head, next] = await (async () => {
|
|
1109
|
+
const events = [createEnvelope({ key: "k" }), createEnvelope({ key: "k" })];
|
|
1110
|
+
for (const event of events) {
|
|
1111
|
+
await append([{ event, deliveries: [createDelivery("handler-a")] }]);
|
|
1160
1112
|
}
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1113
|
+
return events;
|
|
1114
|
+
})();
|
|
1115
|
+
const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1116
|
+
expect(stale.event.id).toBe(head.id);
|
|
1117
|
+
const taken = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
|
|
1118
|
+
expect(taken.map((delivery) => delivery.event.id)).toEqual([head.id]);
|
|
1119
|
+
expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
|
|
1120
|
+
expect(await store.complete(taken[0], at(LEASE_MS + 2))).toBe(true);
|
|
1121
|
+
const [following] = await claim(["handler-a"], at(LEASE_MS + 3), 100, "worker-2");
|
|
1122
|
+
expect(following.event.id).toBe(next.id);
|
|
1123
|
+
});
|
|
1124
|
+
it("lets every worker see only deliveries nobody else holds, and none twice", async () => {
|
|
1125
|
+
await append(
|
|
1126
|
+
Array.from({ length: 30 }, () => ({
|
|
1127
|
+
event: createEnvelope(),
|
|
1128
|
+
deliveries: [createDelivery("handler-a")]
|
|
1129
|
+
}))
|
|
1130
|
+
);
|
|
1131
|
+
const seen = [];
|
|
1132
|
+
for (let round = 0; round < 40; round += 1) {
|
|
1133
|
+
const owners = ["worker-1", "worker-2", "worker-3"];
|
|
1134
|
+
const batches = await Promise.all(
|
|
1135
|
+
owners.map((owner) => claim(["handler-a"], at(round), 4, owner))
|
|
1174
1136
|
);
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
if (store.relinquish === void 0) {
|
|
1179
|
-
return context.skip();
|
|
1137
|
+
for (const batch of batches) {
|
|
1138
|
+
seen.push(...batch.map((delivery) => delivery.deliveryId));
|
|
1139
|
+
expect(await completeMany(store, batch, at(round))).toEqual(batch.map(() => true));
|
|
1180
1140
|
}
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
expect(await claim(["handler-a"], at(2), 100, "worker-2")).toEqual([]);
|
|
1185
|
-
expect(await store.complete(running, at(3))).toBe(true);
|
|
1186
|
-
});
|
|
1141
|
+
}
|
|
1142
|
+
expect(seen).toHaveLength(30);
|
|
1143
|
+
expect(new Set(seen).size).toBe(30);
|
|
1187
1144
|
});
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
})
|
|
1229
|
-
|
|
1230
|
-
|
|
1145
|
+
});
|
|
1146
|
+
describe("relinquish", () => {
|
|
1147
|
+
it("hands idle leases to other workers at once and keeps leases with running work", async (context) => {
|
|
1148
|
+
if (store.relinquish === void 0) {
|
|
1149
|
+
return context.skip();
|
|
1150
|
+
}
|
|
1151
|
+
await append([
|
|
1152
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] },
|
|
1153
|
+
{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }
|
|
1154
|
+
]);
|
|
1155
|
+
const claimed = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1156
|
+
expect(claimed).toHaveLength(2);
|
|
1157
|
+
await store.relinquish("worker-1", at(1));
|
|
1158
|
+
expect(await store.complete(claimed[0], at(2))).toBe(true);
|
|
1159
|
+
expect(await store.complete(claimed[1], at(2))).toBe(true);
|
|
1160
|
+
await store.relinquish("worker-1", at(3));
|
|
1161
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }], at(4));
|
|
1162
|
+
expect(await claim(["handler-a"], at(4), 100, "worker-2")).toHaveLength(1);
|
|
1163
|
+
});
|
|
1164
|
+
it("does not give away a delivery that is still running", async (context) => {
|
|
1165
|
+
if (store.relinquish === void 0) {
|
|
1166
|
+
return context.skip();
|
|
1167
|
+
}
|
|
1168
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
1169
|
+
const [running] = await claim(["handler-a"], at(0), 100, "worker-1");
|
|
1170
|
+
await store.relinquish("worker-1", at(1));
|
|
1171
|
+
expect(await claim(["handler-a"], at(2), 100, "worker-2")).toEqual([]);
|
|
1172
|
+
expect(await store.complete(running, at(3))).toBe(true);
|
|
1173
|
+
});
|
|
1174
|
+
});
|
|
1175
|
+
describe.skipIf(options.createPeerStore === void 0)("across store instances", () => {
|
|
1176
|
+
let peer;
|
|
1177
|
+
beforeEach(async () => {
|
|
1178
|
+
peer = await options.createPeerStore(store);
|
|
1179
|
+
});
|
|
1180
|
+
afterEach(async () => {
|
|
1181
|
+
await options.destroyStore?.(peer);
|
|
1182
|
+
});
|
|
1183
|
+
it("never gives one delivery to claimers in different processes", async () => {
|
|
1184
|
+
await append(
|
|
1185
|
+
Array.from({ length: 60 }, (_, index) => ({
|
|
1186
|
+
event: createEnvelope({ key: index % 3 === 0 ? `key-${index % 9}` : null }),
|
|
1187
|
+
deliveries: [createDelivery("handler-a")]
|
|
1188
|
+
}))
|
|
1189
|
+
);
|
|
1190
|
+
const request = (owner) => ({
|
|
1191
|
+
handlers: ["handler-a"],
|
|
1192
|
+
limit: 7,
|
|
1193
|
+
leaseOwner: owner,
|
|
1194
|
+
leaseExpiresAt: at(LEASE_MS),
|
|
1195
|
+
now: at(0)
|
|
1196
|
+
});
|
|
1197
|
+
const batches = await Promise.all([
|
|
1198
|
+
...Array.from({ length: 5 }, (_, index) => store.claim(request(`primary-${index}`))),
|
|
1199
|
+
...Array.from({ length: 5 }, (_, index) => peer.claim(request(`peer-${index}`)))
|
|
1200
|
+
]);
|
|
1201
|
+
const claimed = batches.flat();
|
|
1202
|
+
expect(new Set(claimed.map((delivery) => delivery.deliveryId)).size).toBe(claimed.length);
|
|
1203
|
+
const keys = claimed.map((delivery) => delivery.event.key).filter((key) => key !== null);
|
|
1204
|
+
expect(new Set(keys).size).toBe(keys.length);
|
|
1205
|
+
});
|
|
1206
|
+
it("fences a lease taken over by another process", async () => {
|
|
1207
|
+
await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
|
|
1208
|
+
const [stale] = await claim(["handler-a"], at(0));
|
|
1209
|
+
const [current] = await peer.claim({
|
|
1210
|
+
handlers: ["handler-a"],
|
|
1211
|
+
limit: 1,
|
|
1212
|
+
leaseOwner: "peer",
|
|
1213
|
+
leaseExpiresAt: at(LEASE_MS * 2),
|
|
1214
|
+
now: at(LEASE_MS)
|
|
1215
|
+
});
|
|
1216
|
+
expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
|
|
1217
|
+
expect(await peer.complete(current, at(LEASE_MS + 1))).toBe(true);
|
|
1218
|
+
});
|
|
1219
|
+
it("completes batches in either process and fences the one that lost its lease", async () => {
|
|
1220
|
+
await append(
|
|
1221
|
+
Array.from({ length: 6 }, () => ({
|
|
1222
|
+
event: createEnvelope(),
|
|
1223
|
+
deliveries: [createDelivery("handler-a")]
|
|
1224
|
+
}))
|
|
1225
|
+
);
|
|
1226
|
+
const request = (owner, now) => ({
|
|
1227
|
+
handlers: ["handler-a"],
|
|
1228
|
+
limit: 100,
|
|
1229
|
+
leaseOwner: owner,
|
|
1230
|
+
leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
|
|
1231
|
+
now
|
|
1231
1232
|
});
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
handlers: ["handler-a"],
|
|
1241
|
-
limit: 100,
|
|
1242
|
-
leaseOwner: owner,
|
|
1243
|
-
leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
|
|
1244
|
-
now
|
|
1245
|
-
});
|
|
1246
|
-
const stale = await store.claim(request("primary", at(0)));
|
|
1247
|
-
const current = await peer.claim(request("peer", at(LEASE_MS)));
|
|
1248
|
-
expect(current).toHaveLength(6);
|
|
1249
|
-
expect(await completeMany(store, stale, at(LEASE_MS + 1))).toEqual(stale.map(() => false));
|
|
1250
|
-
expect(await completeMany(peer, current, at(LEASE_MS + 1))).toEqual(
|
|
1251
|
-
current.map(() => true)
|
|
1252
|
-
);
|
|
1253
|
-
expect(await store.statistics(at(LEASE_MS + 2))).toMatchObject({
|
|
1254
|
-
pending: 0,
|
|
1255
|
-
processing: 0
|
|
1256
|
-
});
|
|
1233
|
+
const stale = await store.claim(request("primary", at(0)));
|
|
1234
|
+
const current = await peer.claim(request("peer", at(LEASE_MS)));
|
|
1235
|
+
expect(current).toHaveLength(6);
|
|
1236
|
+
expect(await completeMany(store, stale, at(LEASE_MS + 1))).toEqual(stale.map(() => false));
|
|
1237
|
+
expect(await completeMany(peer, current, at(LEASE_MS + 1))).toEqual(current.map(() => true));
|
|
1238
|
+
expect(await store.statistics(at(LEASE_MS + 2))).toMatchObject({
|
|
1239
|
+
pending: 0,
|
|
1240
|
+
processing: 0
|
|
1257
1241
|
});
|
|
1258
1242
|
});
|
|
1259
1243
|
});
|
|
1260
1244
|
}
|
|
1261
1245
|
|
|
1246
|
+
// src/event-store-contract.ts
|
|
1247
|
+
function eventStoreContract(name, options) {
|
|
1248
|
+
const keepsCompleted = options.keepsCompletedRecords ?? false;
|
|
1249
|
+
describe(`${name} satisfies the EventStore contract`, () => {
|
|
1250
|
+
let store;
|
|
1251
|
+
beforeEach(async () => {
|
|
1252
|
+
store = await options.createStore();
|
|
1253
|
+
});
|
|
1254
|
+
afterEach(async () => {
|
|
1255
|
+
await options.destroyStore?.(store);
|
|
1256
|
+
});
|
|
1257
|
+
async function append(entries, now = at(0)) {
|
|
1258
|
+
return store.transaction((transaction) => store.append(transaction, entries, now));
|
|
1259
|
+
}
|
|
1260
|
+
async function completeMany(target, leases, now) {
|
|
1261
|
+
if (target.completeMany !== void 0) {
|
|
1262
|
+
return target.completeMany(leases, now);
|
|
1263
|
+
}
|
|
1264
|
+
const outcomes = [];
|
|
1265
|
+
for (const lease of leases) {
|
|
1266
|
+
outcomes.push(await target.complete(lease, now));
|
|
1267
|
+
}
|
|
1268
|
+
return outcomes;
|
|
1269
|
+
}
|
|
1270
|
+
async function claim(handlers, now = at(0), limit = 100, owner = "worker-1") {
|
|
1271
|
+
return store.claim({
|
|
1272
|
+
handlers,
|
|
1273
|
+
limit,
|
|
1274
|
+
leaseOwner: owner,
|
|
1275
|
+
leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
|
|
1276
|
+
now
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
const suite = {
|
|
1280
|
+
options,
|
|
1281
|
+
keepsCompleted,
|
|
1282
|
+
store: () => store,
|
|
1283
|
+
append,
|
|
1284
|
+
claim,
|
|
1285
|
+
completeMany
|
|
1286
|
+
};
|
|
1287
|
+
appendAndClaimContract(suite);
|
|
1288
|
+
fencingContract(suite);
|
|
1289
|
+
maintenanceContract(suite);
|
|
1290
|
+
orderingAndLeasesContract(suite);
|
|
1291
|
+
completionContract(suite);
|
|
1292
|
+
lostLeasesContract(suite);
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1262
1296
|
// src/fake-clock.ts
|
|
1263
1297
|
var FakeClock = class {
|
|
1264
1298
|
#now;
|
|
1265
|
-
|
|
1299
|
+
#timers = /* @__PURE__ */ new Set();
|
|
1300
|
+
#sequence = 0;
|
|
1301
|
+
/** Present only with `controlTimers: true`; see {@link Clock.schedule}. */
|
|
1302
|
+
schedule;
|
|
1303
|
+
constructor(start = "2026-01-01T00:00:00.000Z", options = {}) {
|
|
1266
1304
|
this.#now = new Date(start).getTime();
|
|
1305
|
+
if (options.controlTimers === true) {
|
|
1306
|
+
this.schedule = (task) => this.#schedule(task);
|
|
1307
|
+
}
|
|
1267
1308
|
}
|
|
1268
1309
|
now() {
|
|
1269
1310
|
return new Date(this.#now);
|
|
1270
1311
|
}
|
|
1312
|
+
/** Moves the clock forward, running every controlled timer that falls due on the way, in time order. */
|
|
1271
1313
|
advance(milliseconds) {
|
|
1272
|
-
this.#now
|
|
1314
|
+
const target = this.#now + milliseconds;
|
|
1315
|
+
for (let timer = this.#nextDue(target); timer !== void 0; timer = this.#nextDue(target)) {
|
|
1316
|
+
this.#now = Math.max(this.#now, timer.dueAt);
|
|
1317
|
+
if (timer.task.repeat === true) {
|
|
1318
|
+
timer.dueAt += Math.max(1, timer.task.delayMs);
|
|
1319
|
+
} else {
|
|
1320
|
+
this.#timers.delete(timer);
|
|
1321
|
+
}
|
|
1322
|
+
timer.task.run();
|
|
1323
|
+
}
|
|
1324
|
+
this.#now = target;
|
|
1273
1325
|
return this.now();
|
|
1274
1326
|
}
|
|
1275
1327
|
set(instant) {
|
|
1276
1328
|
this.#now = new Date(instant).getTime();
|
|
1277
1329
|
return this.now();
|
|
1278
1330
|
}
|
|
1331
|
+
/** Controlled timers still waiting to run. */
|
|
1332
|
+
get pendingTimers() {
|
|
1333
|
+
return this.#timers.size;
|
|
1334
|
+
}
|
|
1335
|
+
#schedule(task) {
|
|
1336
|
+
const timer = {
|
|
1337
|
+
task,
|
|
1338
|
+
dueAt: this.#now + Math.max(0, task.delayMs),
|
|
1339
|
+
sequence: this.#sequence++
|
|
1340
|
+
};
|
|
1341
|
+
this.#timers.add(timer);
|
|
1342
|
+
return () => {
|
|
1343
|
+
this.#timers.delete(timer);
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
#nextDue(limit) {
|
|
1347
|
+
let next;
|
|
1348
|
+
for (const timer of this.#timers) {
|
|
1349
|
+
if (timer.dueAt <= limit && (next === void 0 || timer.dueAt < next.dueAt || timer.dueAt === next.dueAt && timer.sequence < next.sequence)) {
|
|
1350
|
+
next = timer;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
return next;
|
|
1354
|
+
}
|
|
1279
1355
|
};
|
|
1280
1356
|
|
|
1281
1357
|
// src/in-memory-transport.ts
|
|
@@ -1315,20 +1391,11 @@ var InMemoryTransport = class {
|
|
|
1315
1391
|
* a subscriber that rejects up to `maxRedeliveries` times. Resolves with the
|
|
1316
1392
|
* total number of handler calls.
|
|
1317
1393
|
*/
|
|
1394
|
+
// eslint-disable-next-line relaybus/max-two-params -- public InMemoryTransport.deliver signature
|
|
1318
1395
|
async deliver(destination, envelope, maxRedeliveries = 0) {
|
|
1319
1396
|
let calls = 0;
|
|
1320
1397
|
for (const handler of this.#subscribers.get(destination) ?? []) {
|
|
1321
|
-
|
|
1322
|
-
calls += 1;
|
|
1323
|
-
try {
|
|
1324
|
-
await handler(envelope);
|
|
1325
|
-
break;
|
|
1326
|
-
} catch (error) {
|
|
1327
|
-
if (attempt >= maxRedeliveries) {
|
|
1328
|
-
throw error;
|
|
1329
|
-
}
|
|
1330
|
-
}
|
|
1331
|
-
}
|
|
1398
|
+
calls += await deliverWithRedeliveries(handler, { envelope, destination, maxRedeliveries });
|
|
1332
1399
|
}
|
|
1333
1400
|
return calls;
|
|
1334
1401
|
}
|
|
@@ -1336,6 +1403,18 @@ var InMemoryTransport = class {
|
|
|
1336
1403
|
this.closed = true;
|
|
1337
1404
|
}
|
|
1338
1405
|
};
|
|
1406
|
+
async function deliverWithRedeliveries(handler, delivery) {
|
|
1407
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
1408
|
+
try {
|
|
1409
|
+
await handler(delivery.envelope, { destination: delivery.destination });
|
|
1410
|
+
return attempt + 1;
|
|
1411
|
+
} catch (error) {
|
|
1412
|
+
if (attempt >= delivery.maxRedeliveries) {
|
|
1413
|
+
throw error;
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1339
1418
|
|
|
1340
1419
|
export { FakeClock, InMemoryTransport, at, createDelivery, createEnvelope, eventStoreContract };
|
|
1341
1420
|
//# sourceMappingURL=index.js.map
|