@relaybus/testing 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,7 +1,7 @@
1
1
  'use strict';
2
2
 
3
- var core = require('@relaybus/core');
4
3
  var vitest = require('vitest');
4
+ var core = require('@relaybus/core');
5
5
 
6
6
  // src/event-store-contract.ts
7
7
  function createDeferred() {
@@ -33,1234 +33,1268 @@ function createEnvelope(overrides = {}) {
33
33
  function createDelivery(handler, availableAt = at(0)) {
34
34
  return { id: core.createUuidV7(), handler, availableAt };
35
35
  }
36
- function eventStoreContract(name, options) {
37
- const keepsCompleted = options.keepsCompletedRecords ?? false;
38
- vitest.describe(`${name} satisfies the EventStore contract`, () => {
39
- let store;
40
- vitest.beforeEach(async () => {
41
- store = await options.createStore();
36
+
37
+ // src/contract/append-and-claim.ts
38
+ function appendAndClaimContract(suite) {
39
+ const { append, claim } = suite;
40
+ let store;
41
+ vitest.beforeEach(() => {
42
+ store = suite.store();
43
+ });
44
+ vitest.describe("setup", () => {
45
+ vitest.it("can run more than once", async () => {
46
+ await store.setup?.();
47
+ await store.setup?.();
48
+ vitest.expect(await store.statistics(at(0))).toMatchObject({ pending: 0 });
42
49
  });
43
- vitest.afterEach(async () => {
44
- await options.destroyStore?.(store);
50
+ });
51
+ vitest.describe("append", () => {
52
+ vitest.it("stores events and deliveries when the transaction commits", async () => {
53
+ const event = createEnvelope();
54
+ const result = await append([
55
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
56
+ ]);
57
+ vitest.expect(result).toEqual({
58
+ appendedEventIds: [event.id],
59
+ duplicateEventIds: [],
60
+ appendedDeliveryCount: 2
61
+ });
62
+ const records = await store.listDeliveries({ eventId: event.id });
63
+ vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
64
+ vitest.expect(records.every((record) => record.status === "pending" && record.attempts === 0)).toBe(
65
+ true
66
+ );
45
67
  });
46
- async function append(entries, now = at(0)) {
47
- return store.transaction((transaction) => store.append(transaction, entries, now));
48
- }
49
- async function completeMany(target, leases, now) {
50
- if (target.completeMany !== void 0) {
51
- return target.completeMany(leases, now);
52
- }
53
- const outcomes = [];
54
- for (const lease of leases) {
55
- outcomes.push(await target.complete(lease, now));
56
- }
57
- return outcomes;
58
- }
59
- async function claim(handlers, now = at(0), limit = 100, owner = "worker-1") {
60
- return store.claim({
61
- handlers,
62
- limit,
63
- leaseOwner: owner,
64
- leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
65
- now
66
- });
67
- }
68
- vitest.describe("setup", () => {
69
- vitest.it("can run more than once", async () => {
70
- await store.setup?.();
71
- await store.setup?.();
72
- vitest.expect(await store.statistics(at(0))).toMatchObject({ pending: 0 });
73
- });
68
+ vitest.it("stores nothing when the transaction rolls back", async () => {
69
+ const event = createEnvelope();
70
+ const failure = new Error("rollback");
71
+ await vitest.expect(
72
+ store.transaction(async (transaction) => {
73
+ await store.append(
74
+ transaction,
75
+ [{ event, deliveries: [createDelivery("handler-a")] }],
76
+ at(0)
77
+ );
78
+ throw failure;
79
+ })
80
+ ).rejects.toBe(failure);
81
+ vitest.expect(await store.listDeliveries({})).toEqual([]);
82
+ vitest.expect(await claim(["handler-a"])).toEqual([]);
74
83
  });
75
- vitest.describe("append", () => {
76
- vitest.it("stores events and deliveries when the transaction commits", async () => {
77
- const event = createEnvelope();
78
- const result = await append([
79
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
80
- ]);
81
- vitest.expect(result).toEqual({
82
- appendedEventIds: [event.id],
83
- duplicateEventIds: [],
84
- appendedDeliveryCount: 2
85
- });
86
- const records = await store.listDeliveries({ eventId: event.id });
87
- vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
88
- vitest.expect(
89
- records.every((record) => record.status === "pending" && record.attempts === 0)
90
- ).toBe(true);
91
- });
92
- vitest.it("stores nothing when the transaction rolls back", async () => {
93
- const event = createEnvelope();
94
- const failure = new Error("rollback");
95
- await vitest.expect(
96
- store.transaction(async (transaction) => {
97
- await store.append(
98
- transaction,
99
- [{ event, deliveries: [createDelivery("handler-a")] }],
100
- at(0)
101
- );
102
- throw failure;
103
- })
104
- ).rejects.toBe(failure);
105
- vitest.expect(await store.listDeliveries({})).toEqual([]);
106
- vitest.expect(await claim(["handler-a"])).toEqual([]);
107
- });
108
- vitest.it("returns the value of the transaction work", async () => {
109
- await vitest.expect(store.transaction(async () => "result")).resolves.toBe("result");
110
- });
111
- vitest.it("round-trips the envelope exactly as JSON would", async () => {
112
- const event = createEnvelope({
113
- key: "order-42",
114
- payload: { text: "\xE7\u011F\u0131\xF6\u015F\xFC \u{1F680}", nested: { list: [1, "two", null, true] }, amount: 12.5 },
115
- headers: { traceparent: "00-abc-def-01", tenant: "acme" },
116
- correlationId: core.createUuidV7(),
117
- causationId: core.createUuidV7(),
118
- occurredAt: /* @__PURE__ */ new Date("2026-03-04T05:06:07.089Z")
119
- });
120
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
121
- const [claimed] = await claim(["handler-a"]);
122
- vitest.expect(claimed.event).toEqual(event);
123
- vitest.expect(claimed.event.occurredAt).toBeInstanceOf(Date);
124
- });
125
- vitest.it("skips an event id that already exists but adds its missing deliveries", async () => {
126
- const event = createEnvelope();
127
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
128
- const result = await append([
129
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
130
- ]);
131
- vitest.expect(result).toEqual({
132
- appendedEventIds: [],
133
- duplicateEventIds: [event.id],
134
- appendedDeliveryCount: 1
135
- });
136
- const records = await store.listDeliveries({ eventId: event.id });
137
- vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
138
- });
139
- vitest.it("deduplicates repeated events and handlers inside one call", async () => {
140
- const event = createEnvelope();
141
- const result = await append([
142
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-a")] },
143
- { event, deliveries: [createDelivery("handler-a")] }
144
- ]);
145
- vitest.expect(result.appendedEventIds).toEqual([event.id]);
146
- vitest.expect(result.duplicateEventIds).toEqual([event.id]);
147
- vitest.expect(result.appendedDeliveryCount).toBe(1);
148
- vitest.expect(await store.listDeliveries({})).toHaveLength(1);
149
- });
150
- vitest.it("keeps events written by concurrent transactions apart", async () => {
151
- const events = Array.from({ length: 20 }, () => createEnvelope());
152
- await Promise.all(
153
- events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
154
- );
155
- vitest.expect(await store.listDeliveries({ limit: 1e3 })).toHaveLength(20);
156
- });
84
+ vitest.it("returns the value of the transaction work", async () => {
85
+ await vitest.expect(store.transaction(async () => "result")).resolves.toBe("result");
157
86
  });
158
- vitest.describe("claim", () => {
159
- vitest.it("leases due deliveries of the requested handlers and counts the attempt", async () => {
160
- const event = createEnvelope();
161
- await append([
162
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
163
- ]);
164
- const claimed = await claim(["handler-a"], at(1e3));
165
- vitest.expect(claimed).toHaveLength(1);
166
- vitest.expect(claimed[0]).toMatchObject({
167
- handler: "handler-a",
168
- attempt: 1,
169
- leaseExpiresAt: at(1e3 + LEASE_MS)
170
- });
171
- vitest.expect(claimed[0].leaseToken).toEqual(vitest.expect.any(String));
172
- const [record] = await store.listDeliveries({ handler: "handler-a" });
173
- vitest.expect(record).toMatchObject({ status: "processing", attempts: 1, leaseOwner: "worker-1" });
174
- vitest.expect(record.leaseExpiresAt).toEqual(at(1e3 + LEASE_MS));
175
- });
176
- vitest.it("ignores deliveries that are not due yet", async () => {
177
- await append([
178
- { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(5e3))] }
179
- ]);
180
- vitest.expect(await claim(["handler-a"], at(4999))).toEqual([]);
181
- vitest.expect(await claim(["handler-a"], at(5e3))).toHaveLength(1);
182
- });
183
- vitest.it("returns nothing for an empty handler list", async () => {
184
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
185
- vitest.expect(await claim([])).toEqual([]);
186
- });
187
- vitest.it("respects the limit and claims the earliest due first", async () => {
188
- const late = createEnvelope();
189
- const early = createEnvelope();
190
- const middle = createEnvelope();
191
- await append([
192
- { event: late, deliveries: [createDelivery("handler-a", at(300))] },
193
- { event: early, deliveries: [createDelivery("handler-a", at(100))] },
194
- { event: middle, deliveries: [createDelivery("handler-a", at(200))] }
195
- ]);
196
- const first = await claim(["handler-a"], at(1e3), 2);
197
- vitest.expect(first.map((delivery) => delivery.event.id)).toEqual([early.id, middle.id]);
198
- const second = await claim(["handler-a"], at(1e3), 2);
199
- vitest.expect(second.map((delivery) => delivery.event.id)).toEqual([late.id]);
200
- });
201
- vitest.it("breaks ties between equally due deliveries by delivery id", async () => {
202
- const ids = [core.createUuidV7(), core.createUuidV7(), core.createUuidV7()].sort();
203
- await append([
204
- {
205
- event: createEnvelope(),
206
- deliveries: [{ id: ids[2], handler: "handler-a", availableAt: at(0) }]
207
- },
208
- {
209
- event: createEnvelope(),
210
- deliveries: [{ id: ids[0], handler: "handler-a", availableAt: at(0) }]
211
- },
212
- {
213
- event: createEnvelope(),
214
- deliveries: [{ id: ids[1], handler: "handler-a", availableAt: at(0) }]
215
- }
216
- ]);
217
- const claimed = await claim(["handler-a"], at(0), 3);
218
- vitest.expect(claimed.map((delivery) => delivery.deliveryId)).toEqual(ids);
219
- });
220
- vitest.it("does not hand out a leased delivery until its lease expires", async () => {
221
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
222
- const [first] = await claim(["handler-a"], at(0));
223
- vitest.expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
224
- const [second] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
225
- vitest.expect(second).toMatchObject({ deliveryId: first.deliveryId, attempt: 2 });
226
- vitest.expect(second.leaseToken).not.toBe(first.leaseToken);
227
- });
228
- vitest.it("never gives the same delivery to two concurrent claimers", async () => {
229
- const entries = Array.from({ length: 40 }, () => ({
230
- event: createEnvelope(),
231
- deliveries: [createDelivery("handler-a")]
232
- }));
233
- await append(entries);
234
- const batches = await Promise.all(
235
- Array.from({ length: 8 }, async (_, index) => {
236
- const own = [];
237
- for (; ; ) {
238
- const batch = await claim(["handler-a"], at(0), 10, `worker-${index}`);
239
- if (batch.length === 0) {
240
- return own;
241
- }
242
- own.push(...batch);
243
- }
244
- })
245
- );
246
- const leftovers = await claim(["handler-a"], at(0), 100, "worker-late");
247
- const claimedIds = [...batches.flat(), ...leftovers].map((delivery) => delivery.deliveryId);
248
- vitest.expect(new Set(claimedIds).size).toBe(claimedIds.length);
249
- vitest.expect(claimedIds).toHaveLength(40);
250
- });
87
+ vitest.it("round-trips the envelope exactly as JSON would", async () => {
88
+ const event = createEnvelope({
89
+ key: "order-42",
90
+ payload: { text: "\xE7\u011F\u0131\xF6\u015F\xFC \u{1F680}", nested: { list: [1, "two", null, true] }, amount: 12.5 },
91
+ headers: { traceparent: "00-abc-def-01", tenant: "acme" },
92
+ correlationId: core.createUuidV7(),
93
+ causationId: core.createUuidV7(),
94
+ occurredAt: /* @__PURE__ */ new Date("2026-03-04T05:06:07.089Z")
95
+ });
96
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
97
+ const [claimed] = await claim(["handler-a"]);
98
+ vitest.expect(claimed.event).toEqual(event);
99
+ vitest.expect(claimed.event.occurredAt).toBeInstanceOf(Date);
251
100
  });
252
- vitest.describe("fencing", () => {
253
- vitest.it("completes a delivery only with its current lease", async () => {
254
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
255
- const [stale] = await claim(["handler-a"], at(0));
256
- const [current] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
257
- vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
258
- vitest.expect(await store.extendLease(stale, at(LEASE_MS * 3), at(LEASE_MS + 1))).toBe(false);
259
- vitest.expect(
260
- await store.reschedule(stale, at(0), { name: "Error", message: "x" }, at(LEASE_MS + 1))
261
- ).toBe(false);
262
- vitest.expect(await store.bury(stale, { name: "Error", message: "x" }, at(LEASE_MS + 1))).toBe(
263
- false
264
- );
265
- vitest.expect(await store.complete(current, at(LEASE_MS + 2))).toBe(true);
266
- if (!keepsCompleted) {
267
- vitest.expect(await store.listDeliveries({})).toEqual([]);
268
- return;
269
- }
270
- const [record] = await store.listDeliveries({});
271
- vitest.expect(record).toMatchObject({
272
- status: "completed",
273
- leaseOwner: null,
274
- leaseExpiresAt: null
275
- });
276
- vitest.expect(record.completedAt).toEqual(at(LEASE_MS + 2));
277
- });
278
- vitest.it("refuses to complete twice", async () => {
279
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
280
- const [delivery] = await claim(["handler-a"]);
281
- vitest.expect(await store.complete(delivery, at(1))).toBe(true);
282
- vitest.expect(await store.complete(delivery, at(2))).toBe(false);
283
- });
284
- vitest.it("refuses leases of unknown deliveries", async () => {
285
- const lease = { deliveryId: core.createUuidV7(), leaseToken: core.createUuidV7() };
286
- vitest.expect(await store.complete(lease, at(0))).toBe(false);
287
- vitest.expect(await store.extendLease(lease, at(1), at(0))).toBe(false);
288
- });
289
- vitest.it("extends a held lease", async () => {
290
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
291
- const [delivery] = await claim(["handler-a"]);
292
- vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
293
- vitest.expect(await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2")).toEqual([]);
294
- const [record] = await store.listDeliveries({});
295
- vitest.expect(record.leaseExpiresAt).toEqual(at(LEASE_MS * 2));
296
- });
297
- vitest.it("completes inside a transaction together with it", async () => {
298
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
299
- const [delivery] = await claim(["handler-a"]);
300
- await vitest.expect(
301
- store.transaction(async (transaction) => {
302
- vitest.expect(await store.complete(delivery, at(1), transaction)).toBe(true);
303
- throw new Error("rollback");
304
- })
305
- ).rejects.toThrow("rollback");
306
- vitest.expect((await store.listDeliveries({}))[0].status).toBe("processing");
307
- await store.transaction(async (transaction) => {
308
- vitest.expect(await store.complete(delivery, at(2), transaction)).toBe(true);
309
- });
310
- if (keepsCompleted) {
311
- vitest.expect((await store.listDeliveries({}))[0].status).toBe("completed");
312
- } else {
313
- vitest.expect(await store.listDeliveries({})).toEqual([]);
314
- }
315
- });
316
- vitest.it("refuses a transactional completion with a lost lease", async () => {
317
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
318
- const [stale] = await claim(["handler-a"], at(0));
319
- await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
320
- await store.transaction(async (transaction) => {
321
- vitest.expect(await store.complete(stale, at(LEASE_MS + 1), transaction)).toBe(false);
322
- });
323
- vitest.expect((await store.listDeliveries({}))[0].status).toBe("processing");
324
- });
101
+ vitest.it("skips an event id that already exists but adds its missing deliveries", async () => {
102
+ const event = createEnvelope();
103
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
104
+ const result = await append([
105
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
106
+ ]);
107
+ vitest.expect(result).toEqual({
108
+ appendedEventIds: [],
109
+ duplicateEventIds: [event.id],
110
+ appendedDeliveryCount: 1
111
+ });
112
+ const records = await store.listDeliveries({ eventId: event.id });
113
+ vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
325
114
  });
326
- vitest.describe("retries and dead letters", () => {
327
- vitest.it("reschedules a delivery with its error", async () => {
328
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
329
- const [delivery] = await claim(["handler-a"]);
330
- const error = { name: "TypeError", message: "boom", stack: "TypeError: boom\n at x" };
331
- vitest.expect(await store.reschedule(delivery, at(1e4), error, at(5))).toBe(true);
332
- const [record] = await store.listDeliveries({});
333
- vitest.expect(record).toMatchObject({
334
- status: "pending",
335
- attempts: 1,
336
- lastError: error,
337
- leaseOwner: null
338
- });
339
- vitest.expect(record.availableAt).toEqual(at(1e4));
340
- vitest.expect(await claim(["handler-a"], at(9999))).toEqual([]);
341
- const [retry] = await claim(["handler-a"], at(1e4));
342
- vitest.expect(retry.attempt).toBe(2);
343
- });
344
- vitest.it("buries a delivery so it is never claimed again", async () => {
345
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
346
- const [delivery] = await claim(["handler-a"]);
347
- const error = { name: "Error", message: "fatal" };
348
- vitest.expect(await store.bury(delivery, error, at(5))).toBe(true);
349
- vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
350
- const [record] = await store.listDeliveries({ status: "dead" });
351
- vitest.expect(record).toMatchObject({ status: "dead", lastError: error });
352
- vitest.expect(record.updatedAt).toEqual(at(5));
353
- });
354
- vitest.it("requeues only dead deliveries, with a fresh attempt budget", async () => {
355
- const event = createEnvelope();
356
- await append([
357
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
358
- ]);
359
- const [dead] = await claim(["handler-a"]);
360
- await store.bury(dead, { name: "Error", message: "fatal" }, at(1));
361
- const [pending] = await store.listDeliveries({ handler: "handler-b" });
362
- const moved = await store.requeue(
363
- [dead.deliveryId, dead.deliveryId, pending.id, core.createUuidV7()],
364
- at(100)
365
- );
366
- vitest.expect(moved).toBe(1);
367
- const [record] = await store.listDeliveries({ handler: "handler-a" });
368
- vitest.expect(record).toMatchObject({ status: "pending", attempts: 0 });
369
- vitest.expect(record.availableAt).toEqual(at(100));
370
- const [again] = await claim(["handler-a"], at(100));
371
- vitest.expect(again.attempt).toBe(1);
372
- });
115
+ vitest.it("deduplicates repeated events and handlers inside one call", async () => {
116
+ const event = createEnvelope();
117
+ const result = await append([
118
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-a")] },
119
+ { event, deliveries: [createDelivery("handler-a")] }
120
+ ]);
121
+ vitest.expect(result.appendedEventIds).toEqual([event.id]);
122
+ vitest.expect(result.duplicateEventIds).toEqual([event.id]);
123
+ vitest.expect(result.appendedDeliveryCount).toBe(1);
124
+ vitest.expect(await store.listDeliveries({})).toHaveLength(1);
373
125
  });
374
- vitest.describe("ordering keys", () => {
375
- async function appendKeyed(key, count, handlers = ["handler-a"]) {
376
- const events = Array.from({ length: count }, () => createEnvelope({ key }));
377
- for (const event of events) {
378
- await append([{ event, deliveries: handlers.map((handler) => createDelivery(handler)) }]);
379
- }
380
- return events;
381
- }
382
- vitest.it("hands out one delivery per handler and key, in publish order", async () => {
383
- const [first, second, third] = await appendKeyed("order-1", 3);
384
- const batch1 = await claim(["handler-a"]);
385
- vitest.expect(batch1.map((delivery) => delivery.event.id)).toEqual([first.id]);
386
- vitest.expect(await claim(["handler-a"])).toEqual([]);
387
- await store.complete(batch1[0], at(1));
388
- const batch2 = await claim(["handler-a"]);
389
- vitest.expect(batch2.map((delivery) => delivery.event.id)).toEqual([second.id]);
390
- await store.complete(batch2[0], at(2));
391
- vitest.expect((await claim(["handler-a"])).map((delivery) => delivery.event.id)).toEqual([
392
- third.id
393
- ]);
394
- });
395
- vitest.it("keeps later events waiting while the head is retrying", async () => {
396
- const [first, second] = await appendKeyed("order-1", 2);
397
- const [head] = await claim(["handler-a"]);
398
- await store.reschedule(head, at(6e4), { name: "Error", message: "retry" }, at(1));
399
- vitest.expect(await claim(["handler-a"], at(3e4))).toEqual([]);
400
- const [retried] = await claim(["handler-a"], at(6e4));
401
- vitest.expect(retried.event.id).toBe(first.id);
402
- await store.complete(retried, at(60001));
403
- vitest.expect((await claim(["handler-a"], at(60002)))[0].event.id).toBe(second.id);
404
- });
405
- vitest.it("releases the key when the head is dead-lettered", async () => {
406
- const [, second] = await appendKeyed("order-1", 2);
407
- const [head] = await claim(["handler-a"]);
408
- await store.bury(head, { name: "Error", message: "fatal" }, at(1));
409
- vitest.expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
410
- });
411
- vitest.it("orders each handler independently", async () => {
412
- const [first, second] = await appendKeyed("order-1", 2, ["handler-a", "handler-b"]);
413
- const [fromA] = await claim(["handler-a"]);
414
- await store.complete(fromA, at(1));
415
- const [nextA] = await claim(["handler-a"]);
416
- vitest.expect(nextA.event.id).toBe(second.id);
417
- const [fromB] = await claim(["handler-b"]);
418
- vitest.expect(fromB.event.id).toBe(first.id);
419
- });
420
- vitest.it("runs different keys and unkeyed events in parallel", async () => {
421
- await appendKeyed("order-1", 2);
422
- await appendKeyed("order-2", 2);
423
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
424
- vitest.expect(await claim(["handler-a"])).toHaveLength(3);
425
- });
426
- vitest.it("orders concurrent publishers of the same key by commit", async () => {
427
- const events = Array.from(
428
- { length: 10 },
429
- (_, index) => createEnvelope({ key: "shared", payload: { index } })
430
- );
431
- await Promise.all(
432
- events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
433
- );
434
- const seen = [];
435
- for (let round = 0; round < 10; round += 1) {
436
- const batch = await claim(["handler-a"]);
437
- vitest.expect(batch).toHaveLength(1);
438
- seen.push(batch[0].event.id);
439
- await store.complete(batch[0], at(round + 1));
440
- }
441
- vitest.expect(new Set(seen).size).toBe(10);
442
- });
126
+ vitest.it("keeps events written by concurrent transactions apart", async () => {
127
+ const events = Array.from({ length: 20 }, () => createEnvelope());
128
+ await Promise.all(
129
+ events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
130
+ );
131
+ vitest.expect(await store.listDeliveries({ limit: 1e3 })).toHaveLength(20);
443
132
  });
444
- vitest.describe("queries", () => {
445
- vitest.it("filters deliveries and applies the limit in creation order", async () => {
446
- const first = createEnvelope({ type: "contract.first.v1" });
447
- const second = createEnvelope({ type: "contract.second.v1" });
448
- await append([{ event: first, deliveries: [createDelivery("handler-a")] }], at(1));
449
- await append(
450
- [
451
- {
452
- event: second,
453
- deliveries: [createDelivery("handler-a"), createDelivery("handler-b")]
454
- }
455
- ],
456
- at(2)
457
- );
458
- vitest.expect(await store.listDeliveries({ eventType: "contract.second.v1" })).toHaveLength(2);
459
- vitest.expect(await store.listDeliveries({ handler: "handler-b" })).toHaveLength(1);
460
- vitest.expect(
461
- await store.listDeliveries({ status: "pending", handler: "handler-a" })
462
- ).toHaveLength(2);
463
- vitest.expect(await store.listDeliveries({ status: "dead" })).toEqual([]);
464
- const limited = await store.listDeliveries({ limit: 1 });
465
- vitest.expect(limited).toHaveLength(1);
466
- vitest.expect(limited[0].event.id).toBe(first.id);
467
- vitest.expect(limited[0].createdAt).toEqual(at(1));
468
- });
469
- vitest.it("reports counts and the oldest due delivery", async () => {
470
- await append([
471
- { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(100))] },
472
- { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(200))] },
473
- { event: createEnvelope(), deliveries: [createDelivery("handler-b", at(5e4))] },
474
- { event: createEnvelope(), deliveries: [createDelivery("handler-c", at(0))] }
475
- ]);
476
- const [processing] = await claim(["handler-c"], at(0));
477
- const [dead] = await claim(["handler-a"], at(100), 1);
478
- await store.bury(dead, { name: "Error", message: "x" }, at(101));
479
- vitest.expect(await store.statistics(at(1e3))).toEqual({
480
- pending: 2,
481
- processing: 1,
482
- completed: 0,
483
- dead: 1,
484
- oldestDueAt: at(200)
485
- });
486
- vitest.expect((await store.statistics(at(LEASE_MS))).oldestDueAt).toEqual(at(200));
487
- await store.complete(processing, at(1));
488
- vitest.expect(await store.statistics(at(0))).toMatchObject({
489
- completed: keepsCompleted ? 1 : 0,
490
- processing: 0,
491
- oldestDueAt: null
492
- });
493
- });
494
- vitest.it("counts an expired lease as due", async () => {
495
- await append([
496
- { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(LEASE_MS * 5))] }
497
- ]);
498
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
499
- await claim(["handler-b"], at(0));
500
- vitest.expect((await store.statistics(at(LEASE_MS + 1))).oldestDueAt).toEqual(at(LEASE_MS));
501
- });
133
+ });
134
+ vitest.describe("claim", () => {
135
+ vitest.it("leases due deliveries of the requested handlers and counts the attempt", async () => {
136
+ const event = createEnvelope();
137
+ await append([
138
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
139
+ ]);
140
+ const claimed = await claim(["handler-a"], at(1e3));
141
+ vitest.expect(claimed).toHaveLength(1);
142
+ vitest.expect(claimed[0]).toMatchObject({
143
+ handler: "handler-a",
144
+ attempt: 1,
145
+ leaseExpiresAt: at(1e3 + LEASE_MS)
146
+ });
147
+ vitest.expect(claimed[0].leaseToken).toEqual(vitest.expect.any(String));
148
+ const [record] = await store.listDeliveries({ handler: "handler-a" });
149
+ vitest.expect(record).toMatchObject({ status: "processing", attempts: 1, leaseOwner: "worker-1" });
150
+ vitest.expect(record.leaseExpiresAt).toEqual(at(1e3 + LEASE_MS));
502
151
  });
503
- vitest.describe("purge", () => {
504
- async function appendDoneFailedShared() {
505
- const done = createEnvelope();
506
- const failed = createEnvelope();
507
- const shared = createEnvelope();
508
- await append([
509
- { event: done, deliveries: [createDelivery("handler-a")] },
510
- { event: failed, deliveries: [createDelivery("handler-b")] },
511
- { event: shared, deliveries: [createDelivery("handler-c"), createDelivery("handler-d")] }
512
- ]);
513
- const [doneDelivery] = await claim(["handler-a"]);
514
- await store.complete(doneDelivery, at(10));
515
- const [failedDelivery] = await claim(["handler-b"]);
516
- await store.bury(failedDelivery, { name: "Error", message: "x" }, at(10));
517
- const [sharedDelivery] = await claim(["handler-c"]);
518
- await store.complete(sharedDelivery, at(10));
519
- return { shared };
520
- }
521
- vitest.it.runIf(keepsCompleted)(
522
- "deletes old completed and dead deliveries and their orphaned events",
523
- async () => {
524
- const { shared } = await appendDoneFailedShared();
525
- vitest.expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
526
- vitest.expect(await store.purge({ completedBefore: at(10) })).toEqual({
527
- deletedDeliveries: 0,
528
- deletedEvents: 0
529
- });
530
- vitest.expect(await store.purge({ completedBefore: at(11) })).toEqual({
531
- deletedDeliveries: 2,
532
- deletedEvents: 1
533
- });
534
- vitest.expect(await store.purge({ deadBefore: at(11) })).toEqual({
535
- deletedDeliveries: 1,
536
- deletedEvents: 1
537
- });
538
- const remaining = await store.listDeliveries({});
539
- vitest.expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
540
- vitest.expect(remaining[0].event.id).toBe(shared.id);
152
+ vitest.it("ignores deliveries that are not due yet", async () => {
153
+ await append([
154
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(5e3))] }
155
+ ]);
156
+ vitest.expect(await claim(["handler-a"], at(4999))).toEqual([]);
157
+ vitest.expect(await claim(["handler-a"], at(5e3))).toHaveLength(1);
158
+ });
159
+ vitest.it("returns nothing for an empty handler list", async () => {
160
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
161
+ vitest.expect(await claim([])).toEqual([]);
162
+ });
163
+ vitest.it("respects the limit and claims the earliest due first", async () => {
164
+ const late = createEnvelope();
165
+ const early = createEnvelope();
166
+ const middle = createEnvelope();
167
+ await append([
168
+ { event: late, deliveries: [createDelivery("handler-a", at(300))] },
169
+ { event: early, deliveries: [createDelivery("handler-a", at(100))] },
170
+ { event: middle, deliveries: [createDelivery("handler-a", at(200))] }
171
+ ]);
172
+ const first = await claim(["handler-a"], at(1e3), 2);
173
+ vitest.expect(first.map((delivery) => delivery.event.id)).toEqual([early.id, middle.id]);
174
+ const second = await claim(["handler-a"], at(1e3), 2);
175
+ vitest.expect(second.map((delivery) => delivery.event.id)).toEqual([late.id]);
176
+ });
177
+ vitest.it("breaks ties between equally due deliveries by delivery id", async () => {
178
+ const ids = [core.createUuidV7(), core.createUuidV7(), core.createUuidV7()].sort();
179
+ await append([
180
+ {
181
+ event: createEnvelope(),
182
+ deliveries: [{ id: ids[2], handler: "handler-a", availableAt: at(0) }]
183
+ },
184
+ {
185
+ event: createEnvelope(),
186
+ deliveries: [{ id: ids[0], handler: "handler-a", availableAt: at(0) }]
187
+ },
188
+ {
189
+ event: createEnvelope(),
190
+ deliveries: [{ id: ids[1], handler: "handler-a", availableAt: at(0) }]
541
191
  }
192
+ ]);
193
+ const claimed = await claim(["handler-a"], at(0), 3);
194
+ vitest.expect(claimed.map((delivery) => delivery.deliveryId)).toEqual(ids);
195
+ });
196
+ vitest.it("does not hand out a leased delivery until its lease expires", async () => {
197
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
198
+ const [first] = await claim(["handler-a"], at(0));
199
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
200
+ const [second] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
201
+ vitest.expect(second).toMatchObject({ deliveryId: first.deliveryId, attempt: 2 });
202
+ vitest.expect(second.leaseToken).not.toBe(first.leaseToken);
203
+ });
204
+ vitest.it("never gives the same delivery to two concurrent claimers", async () => {
205
+ const entries = Array.from({ length: 40 }, () => ({
206
+ event: createEnvelope(),
207
+ deliveries: [createDelivery("handler-a")]
208
+ }));
209
+ await append(entries);
210
+ const batches = await Promise.all(
211
+ Array.from({ length: 8 }, async (_, index) => {
212
+ const own = [];
213
+ for (; ; ) {
214
+ const batch = await claim(["handler-a"], at(0), 10, `worker-${index}`);
215
+ if (batch.length === 0) {
216
+ return own;
217
+ }
218
+ own.push(...batch);
219
+ }
220
+ })
542
221
  );
543
- vitest.it.runIf(!keepsCompleted)(
544
- "deletes old dead deliveries and the events no delivery references any more",
545
- async () => {
546
- const { shared } = await appendDoneFailedShared();
547
- vitest.expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
548
- vitest.expect(await store.purge({ completedBefore: at(0) })).toEqual({
549
- deletedDeliveries: 0,
550
- deletedEvents: 0
551
- });
552
- vitest.expect(await store.purge({ completedBefore: at(11) })).toEqual({
553
- deletedDeliveries: 0,
554
- deletedEvents: 1
555
- });
556
- vitest.expect(await store.purge({ deadBefore: at(11) })).toEqual({
557
- deletedDeliveries: 1,
558
- deletedEvents: 1
559
- });
560
- const remaining = await store.listDeliveries({});
561
- vitest.expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
562
- vitest.expect(remaining[0].event.id).toBe(shared.id);
563
- }
222
+ const leftovers = await claim(["handler-a"], at(0), 100, "worker-late");
223
+ const claimedIds = [...batches.flat(), ...leftovers].map((delivery) => delivery.deliveryId);
224
+ vitest.expect(new Set(claimedIds).size).toBe(claimedIds.length);
225
+ vitest.expect(claimedIds).toHaveLength(40);
226
+ });
227
+ });
228
+ }
229
+ function fencingContract(suite) {
230
+ const { append, claim, keepsCompleted } = suite;
231
+ let store;
232
+ vitest.beforeEach(() => {
233
+ store = suite.store();
234
+ });
235
+ vitest.describe("fencing", () => {
236
+ vitest.it("completes a delivery only with its current lease", async () => {
237
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
238
+ const [stale] = await claim(["handler-a"], at(0));
239
+ const [current] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
240
+ vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
241
+ vitest.expect(await store.extendLease(stale, at(LEASE_MS * 3), at(LEASE_MS + 1))).toBe(false);
242
+ vitest.expect(
243
+ await store.reschedule(stale, at(0), { name: "Error", message: "x" }, at(LEASE_MS + 1))
244
+ ).toBe(false);
245
+ vitest.expect(await store.bury(stale, { name: "Error", message: "x" }, at(LEASE_MS + 1))).toBe(
246
+ false
564
247
  );
565
- vitest.it("lets a purged event id be appended again", async () => {
566
- const event = createEnvelope();
567
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
568
- const [delivery] = await claim(["handler-a"]);
569
- await store.complete(delivery, at(1));
570
- await store.purge({ completedBefore: at(2) });
571
- const result = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
572
- vitest.expect(result.appendedEventIds).toEqual([event.id]);
248
+ vitest.expect(await store.complete(current, at(LEASE_MS + 2))).toBe(true);
249
+ if (!keepsCompleted) {
250
+ vitest.expect(await store.listDeliveries({})).toEqual([]);
251
+ return;
252
+ }
253
+ const [record] = await store.listDeliveries({});
254
+ vitest.expect(record).toMatchObject({
255
+ status: "completed",
256
+ leaseOwner: null,
257
+ leaseExpiresAt: null
573
258
  });
259
+ vitest.expect(record.completedAt).toEqual(at(LEASE_MS + 2));
574
260
  });
575
- vitest.describe("release", () => {
576
- vitest.it("gives a leased delivery back without counting the attempt", async () => {
577
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
578
- const [delivery] = await claim(["handler-a"], at(0));
579
- vitest.expect(await store.release(delivery, at(500))).toBe(true);
580
- const [record] = await store.listDeliveries({});
581
- vitest.expect(record).toMatchObject({
582
- status: "pending",
583
- attempts: 0,
584
- leaseOwner: null,
585
- leaseExpiresAt: null
586
- });
587
- vitest.expect(record.availableAt).toEqual(at(500));
588
- vitest.expect(record.updatedAt).toEqual(at(500));
589
- vitest.expect(await store.release(delivery, at(501))).toBe(false);
590
- const [again] = await claim(["handler-a"], at(500));
591
- vitest.expect(again.attempt).toBe(1);
592
- });
261
+ vitest.it("refuses to complete twice", async () => {
262
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
263
+ const [delivery] = await claim(["handler-a"]);
264
+ vitest.expect(await store.complete(delivery, at(1))).toBe(true);
265
+ vitest.expect(await store.complete(delivery, at(2))).toBe(false);
593
266
  });
594
- vitest.describe("transactions in flight", () => {
595
- vitest.it("hides uncommitted appends from every reader", async () => {
596
- const inside = createDeferred();
597
- const finish = createDeferred();
598
- const event = createEnvelope();
599
- const open = store.transaction(async (transaction) => {
600
- await store.append(
601
- transaction,
602
- [{ event, deliveries: [createDelivery("handler-a")] }],
603
- at(0)
604
- );
605
- inside.resolve();
606
- await finish.promise;
607
- });
608
- await inside.promise;
609
- vitest.expect(await claim(["handler-a"])).toEqual([]);
267
+ vitest.it("refuses leases of unknown deliveries", async () => {
268
+ const lease = { deliveryId: core.createUuidV7(), leaseToken: core.createUuidV7() };
269
+ vitest.expect(await store.complete(lease, at(0))).toBe(false);
270
+ vitest.expect(await store.extendLease(lease, at(1), at(0))).toBe(false);
271
+ });
272
+ vitest.it("extends a held lease", async () => {
273
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
274
+ const [delivery] = await claim(["handler-a"]);
275
+ vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
276
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2")).toEqual([]);
277
+ const [record] = await store.listDeliveries({});
278
+ vitest.expect(record.leaseExpiresAt).toEqual(at(LEASE_MS * 2));
279
+ });
280
+ vitest.it("completes inside a transaction together with it", async () => {
281
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
282
+ const [delivery] = await claim(["handler-a"]);
283
+ await vitest.expect(
284
+ store.transaction(async (transaction) => {
285
+ vitest.expect(await store.complete(delivery, at(1), transaction)).toBe(true);
286
+ throw new Error("rollback");
287
+ })
288
+ ).rejects.toThrow("rollback");
289
+ vitest.expect((await store.listDeliveries({}))[0].status).toBe("processing");
290
+ await store.transaction(async (transaction) => {
291
+ vitest.expect(await store.complete(delivery, at(2), transaction)).toBe(true);
292
+ });
293
+ if (keepsCompleted) {
294
+ vitest.expect((await store.listDeliveries({}))[0].status).toBe("completed");
295
+ } else {
610
296
  vitest.expect(await store.listDeliveries({})).toEqual([]);
611
- vitest.expect((await store.statistics(at(0))).pending).toBe(0);
612
- finish.resolve();
613
- await open;
614
- vitest.expect(await claim(["handler-a"])).toHaveLength(1);
615
- });
616
- vitest.it("orders same-key events by commit, not by first write", async () => {
617
- const first = createEnvelope({ key: "race" });
618
- const second = createEnvelope({ key: "race" });
619
- const firstWritten = createDeferred();
620
- const releaseFirst = createDeferred();
621
- const firstTransaction = store.transaction(async (transaction) => {
622
- await store.append(
623
- transaction,
624
- [{ event: first, deliveries: [createDelivery("handler-a")] }],
625
- at(0)
626
- );
627
- firstWritten.resolve();
628
- await releaseFirst.promise;
629
- });
630
- await firstWritten.promise;
631
- const secondTransaction = store.transaction(
632
- (transaction) => store.append(
633
- transaction,
634
- [{ event: second, deliveries: [createDelivery("handler-a")] }],
635
- at(0)
636
- )
637
- );
638
- await new Promise((resolve) => setTimeout(resolve, 50));
639
- releaseFirst.resolve();
640
- await firstTransaction;
641
- await secondTransaction;
642
- const [head] = await claim(["handler-a"]);
643
- vitest.expect(head.event.id).toBe(first.id);
644
- await store.complete(head, at(1));
645
- vitest.expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
646
- });
647
- vitest.it("stores an event id written by two concurrent transactions once", async () => {
648
- const event = createEnvelope({ key: "same" });
649
- const results = await Promise.all([
650
- append([{ event, deliveries: [createDelivery("handler-a")] }]),
651
- append([
652
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
653
- ])
654
- ]);
655
- const appended = results.flatMap((result) => result.appendedEventIds);
656
- vitest.expect(appended).toEqual([event.id]);
657
- const records = await store.listDeliveries({});
658
- vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
659
- });
297
+ }
660
298
  });
661
- vitest.describe("ordering keys under load", () => {
662
- vitest.it("does not let a blocked key starve unrelated deliveries", async () => {
663
- const blocked = Array.from({ length: 120 }, () => ({
664
- event: createEnvelope({ key: "hot" }),
665
- deliveries: [createDelivery("handler-a", at(0))]
666
- }));
667
- await append(blocked);
668
- const unrelated = Array.from({ length: 5 }, () => ({
669
- event: createEnvelope(),
670
- deliveries: [createDelivery("handler-a", at(10))]
671
- }));
672
- await append(unrelated);
673
- const claimed = await claim(["handler-a"], at(100), 10);
674
- vitest.expect(claimed).toHaveLength(6);
675
- vitest.expect(claimed.filter((delivery) => delivery.event.key === "hot")).toHaveLength(1);
676
- });
677
- vitest.it("keeps successors blocked behind an expired, delayed or requeued head", async () => {
678
- const head = createEnvelope({ key: "k" });
679
- const next = createEnvelope({ key: "k" });
680
- await append([{ event: head, deliveries: [createDelivery("handler-a")] }]);
681
- await append([{ event: next, deliveries: [createDelivery("handler-a")] }]);
682
- const [claimedHead] = await claim(["handler-a"], at(0));
683
- const reclaimed = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
684
- vitest.expect(reclaimed.map((delivery) => delivery.event.id)).toEqual([head.id]);
685
- vitest.expect(
686
- await store.reschedule(
687
- reclaimed[0],
688
- at(LEASE_MS * 10),
689
- { name: "Error", message: "x" },
690
- at(LEASE_MS)
691
- )
692
- ).toBe(true);
693
- vitest.expect(await claim(["handler-a"], at(LEASE_MS * 5))).toEqual([]);
694
- const [delayedHead] = await claim(["handler-a"], at(LEASE_MS * 10));
695
- await store.bury(delayedHead, { name: "Error", message: "x" }, at(LEASE_MS * 10));
696
- vitest.expect(await store.requeue([delayedHead.deliveryId], at(LEASE_MS * 11))).toBe(1);
697
- const [requeuedHead] = await claim(["handler-a"], at(LEASE_MS * 11));
698
- vitest.expect(requeuedHead.event.id).toBe(head.id);
699
- vitest.expect(await claim(["handler-a"], at(LEASE_MS * 11))).toEqual([]);
700
- vitest.expect(claimedHead).toBeDefined();
701
- });
299
+ vitest.it("refuses a transactional completion with a lost lease", async () => {
300
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
301
+ const [stale] = await claim(["handler-a"], at(0));
302
+ await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
303
+ await store.transaction(async (transaction) => {
304
+ vitest.expect(await store.complete(stale, at(LEASE_MS + 1), transaction)).toBe(false);
305
+ });
306
+ vitest.expect((await store.listDeliveries({}))[0].status).toBe("processing");
702
307
  });
703
- vitest.describe("requeued keyed dead letters", () => {
704
- vitest.it("never runs a requeued dead letter alongside its running successor", async () => {
705
- const first = createEnvelope({ key: "k" });
706
- const second = createEnvelope({ key: "k" });
707
- await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
708
- await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
709
- const [head] = await claim(["handler-a"], at(0));
710
- await store.bury(head, { name: "Error", message: "x" }, at(1));
711
- const [successor] = await claim(["handler-a"], at(2));
712
- vitest.expect(successor.event.id).toBe(second.id);
713
- vitest.expect(await store.requeue([head.deliveryId], at(3))).toBe(1);
714
- vitest.expect(await claim(["handler-a"], at(4))).toEqual([]);
715
- await store.complete(successor, at(5));
716
- const [requeued] = await claim(["handler-a"], at(6));
717
- vitest.expect(requeued.event.id).toBe(first.id);
718
- });
719
- vitest.it("recovers when the running successor crashes after the requeue", async () => {
720
- const first = createEnvelope({ key: "k" });
721
- const second = createEnvelope({ key: "k" });
722
- await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
723
- await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
724
- const [head] = await claim(["handler-a"], at(0));
725
- await store.bury(head, { name: "Error", message: "x" }, at(1));
726
- await claim(["handler-a"], at(2));
727
- await store.requeue([head.deliveryId], at(3));
728
- const recovered = await claim(["handler-a"], at(2 + LEASE_MS));
729
- vitest.expect(recovered.map((delivery) => delivery.event.id)).toEqual([first.id]);
730
- await store.complete(recovered[0], at(3 + LEASE_MS));
731
- const [retried] = await claim(["handler-a"], at(4 + LEASE_MS));
732
- vitest.expect(retried).toMatchObject({ attempt: 2 });
733
- vitest.expect(retried.event.id).toBe(second.id);
734
- });
308
+ });
309
+ vitest.describe("retries and dead letters", () => {
310
+ vitest.it("reschedules a delivery with its error", async () => {
311
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
312
+ const [delivery] = await claim(["handler-a"]);
313
+ const error = { name: "TypeError", message: "boom", stack: "TypeError: boom\n at x" };
314
+ vitest.expect(await store.reschedule(delivery, at(1e4), error, at(5))).toBe(true);
315
+ const [record] = await store.listDeliveries({});
316
+ vitest.expect(record).toMatchObject({
317
+ status: "pending",
318
+ attempts: 1,
319
+ lastError: error,
320
+ leaseOwner: null
321
+ });
322
+ vitest.expect(record.availableAt).toEqual(at(1e4));
323
+ vitest.expect(await claim(["handler-a"], at(9999))).toEqual([]);
324
+ const [retry] = await claim(["handler-a"], at(1e4));
325
+ vitest.expect(retry.attempt).toBe(2);
735
326
  });
736
- vitest.describe("lease semantics", () => {
737
- vitest.it("confirms a repeated renewal with identical values", async () => {
738
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
739
- const [delivery] = await claim(["handler-a"], at(0));
740
- vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
741
- vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
742
- });
743
- vitest.it("accepts an expired lease that nobody reclaimed", async () => {
744
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
745
- const [delivery] = await claim(["handler-a"], at(0));
746
- vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 3), at(LEASE_MS * 2))).toBe(true);
747
- vitest.expect(await store.complete(delivery, at(LEASE_MS * 2))).toBe(true);
748
- });
749
- vitest.it("rejects every lease operation once the delivery left processing", async () => {
750
- const error = { name: "Error", message: "x" };
751
- await append([
327
+ vitest.it("buries a delivery so it is never claimed again", async () => {
328
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
329
+ const [delivery] = await claim(["handler-a"]);
330
+ const error = { name: "Error", message: "fatal" };
331
+ vitest.expect(await store.bury(delivery, error, at(5))).toBe(true);
332
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
333
+ const [record] = await store.listDeliveries({ status: "dead" });
334
+ vitest.expect(record).toMatchObject({ status: "dead", lastError: error });
335
+ vitest.expect(record.updatedAt).toEqual(at(5));
336
+ });
337
+ vitest.it("requeues only dead deliveries, with a fresh attempt budget", async () => {
338
+ const event = createEnvelope();
339
+ await append([
340
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
341
+ ]);
342
+ const [dead] = await claim(["handler-a"]);
343
+ await store.bury(dead, { name: "Error", message: "fatal" }, at(1));
344
+ const [pending] = await store.listDeliveries({ handler: "handler-b" });
345
+ const moved = await store.requeue(
346
+ [dead.deliveryId, dead.deliveryId, pending.id, core.createUuidV7()],
347
+ at(100)
348
+ );
349
+ vitest.expect(moved).toBe(1);
350
+ const [record] = await store.listDeliveries({ handler: "handler-a" });
351
+ vitest.expect(record).toMatchObject({ status: "pending", attempts: 0 });
352
+ vitest.expect(record.availableAt).toEqual(at(100));
353
+ const [again] = await claim(["handler-a"], at(100));
354
+ vitest.expect(again.attempt).toBe(1);
355
+ });
356
+ });
357
+ vitest.describe("ordering keys", () => {
358
+ async function appendKeyed(key, count, handlers = ["handler-a"]) {
359
+ const events = Array.from({ length: count }, () => createEnvelope({ key }));
360
+ for (const event of events) {
361
+ await append([{ event, deliveries: handlers.map((handler) => createDelivery(handler)) }]);
362
+ }
363
+ return events;
364
+ }
365
+ vitest.it("hands out one delivery per handler and key, in publish order", async () => {
366
+ const [first, second, third] = await appendKeyed("order-1", 3);
367
+ const batch1 = await claim(["handler-a"]);
368
+ vitest.expect(batch1.map((delivery) => delivery.event.id)).toEqual([first.id]);
369
+ vitest.expect(await claim(["handler-a"])).toEqual([]);
370
+ await store.complete(batch1[0], at(1));
371
+ const batch2 = await claim(["handler-a"]);
372
+ vitest.expect(batch2.map((delivery) => delivery.event.id)).toEqual([second.id]);
373
+ await store.complete(batch2[0], at(2));
374
+ vitest.expect((await claim(["handler-a"])).map((delivery) => delivery.event.id)).toEqual([
375
+ third.id
376
+ ]);
377
+ });
378
+ vitest.it("keeps later events waiting while the head is retrying", async () => {
379
+ const [first, second] = await appendKeyed("order-1", 2);
380
+ const [head] = await claim(["handler-a"]);
381
+ await store.reschedule(head, at(6e4), { name: "Error", message: "retry" }, at(1));
382
+ vitest.expect(await claim(["handler-a"], at(3e4))).toEqual([]);
383
+ const [retried] = await claim(["handler-a"], at(6e4));
384
+ vitest.expect(retried.event.id).toBe(first.id);
385
+ await store.complete(retried, at(60001));
386
+ vitest.expect((await claim(["handler-a"], at(60002)))[0].event.id).toBe(second.id);
387
+ });
388
+ vitest.it("releases the key when the head is dead-lettered", async () => {
389
+ const [, second] = await appendKeyed("order-1", 2);
390
+ const [head] = await claim(["handler-a"]);
391
+ await store.bury(head, { name: "Error", message: "fatal" }, at(1));
392
+ vitest.expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
393
+ });
394
+ vitest.it("orders each handler independently", async () => {
395
+ const [first, second] = await appendKeyed("order-1", 2, ["handler-a", "handler-b"]);
396
+ const [fromA] = await claim(["handler-a"]);
397
+ await store.complete(fromA, at(1));
398
+ const [nextA] = await claim(["handler-a"]);
399
+ vitest.expect(nextA.event.id).toBe(second.id);
400
+ const [fromB] = await claim(["handler-b"]);
401
+ vitest.expect(fromB.event.id).toBe(first.id);
402
+ });
403
+ vitest.it("runs different keys and unkeyed events in parallel", async () => {
404
+ await appendKeyed("order-1", 2);
405
+ await appendKeyed("order-2", 2);
406
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
407
+ vitest.expect(await claim(["handler-a"])).toHaveLength(3);
408
+ });
409
+ vitest.it("orders concurrent publishers of the same key by commit", async () => {
410
+ const events = Array.from(
411
+ { length: 10 },
412
+ (_, index) => createEnvelope({ key: "shared", payload: { index } })
413
+ );
414
+ await Promise.all(
415
+ events.map((event) => append([{ event, deliveries: [createDelivery("handler-a")] }]))
416
+ );
417
+ const seen = [];
418
+ for (let round = 0; round < 10; round += 1) {
419
+ const batch = await claim(["handler-a"]);
420
+ vitest.expect(batch).toHaveLength(1);
421
+ seen.push(batch[0].event.id);
422
+ await store.complete(batch[0], at(round + 1));
423
+ }
424
+ vitest.expect(new Set(seen).size).toBe(10);
425
+ });
426
+ });
427
+ }
428
+ function maintenanceContract(suite) {
429
+ const { append, claim, keepsCompleted } = suite;
430
+ let store;
431
+ vitest.beforeEach(() => {
432
+ store = suite.store();
433
+ });
434
+ vitest.describe("queries", () => {
435
+ vitest.it("filters deliveries and applies the limit in creation order", async () => {
436
+ const first = createEnvelope({ type: "contract.first.v1" });
437
+ const second = createEnvelope({ type: "contract.second.v1" });
438
+ await append([{ event: first, deliveries: [createDelivery("handler-a")] }], at(1));
439
+ await append(
440
+ [
752
441
  {
753
- event: createEnvelope(),
754
- deliveries: [createDelivery("a"), createDelivery("b"), createDelivery("c")]
442
+ event: second,
443
+ deliveries: [createDelivery("handler-a"), createDelivery("handler-b")]
755
444
  }
756
- ]);
757
- const [completed] = await claim(["a"]);
758
- await store.complete(completed, at(1));
759
- const [buried] = await claim(["b"]);
760
- await store.bury(buried, error, at(1));
761
- const [rescheduled] = await claim(["c"]);
762
- await store.reschedule(rescheduled, at(100), error, at(1));
763
- for (const lease of [completed, buried, rescheduled]) {
764
- vitest.expect(await store.extendLease(lease, at(LEASE_MS), at(2))).toBe(false);
765
- vitest.expect(await store.reschedule(lease, at(3), error, at(2))).toBe(false);
766
- vitest.expect(await store.bury(lease, error, at(2))).toBe(false);
767
- vitest.expect(await store.release(lease, at(2))).toBe(false);
768
- vitest.expect(await store.complete(lease, at(2))).toBe(false);
769
- }
445
+ ],
446
+ at(2)
447
+ );
448
+ vitest.expect(await store.listDeliveries({ eventType: "contract.second.v1" })).toHaveLength(2);
449
+ vitest.expect(await store.listDeliveries({ handler: "handler-b" })).toHaveLength(1);
450
+ vitest.expect(await store.listDeliveries({ status: "pending", handler: "handler-a" })).toHaveLength(
451
+ 2
452
+ );
453
+ vitest.expect(await store.listDeliveries({ status: "dead" })).toEqual([]);
454
+ const limited = await store.listDeliveries({ limit: 1 });
455
+ vitest.expect(limited).toHaveLength(1);
456
+ vitest.expect(limited[0].event.id).toBe(first.id);
457
+ vitest.expect(limited[0].createdAt).toEqual(at(1));
458
+ });
459
+ vitest.it("reports counts and the oldest due delivery", async () => {
460
+ await append([
461
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(100))] },
462
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(200))] },
463
+ { event: createEnvelope(), deliveries: [createDelivery("handler-b", at(5e4))] },
464
+ { event: createEnvelope(), deliveries: [createDelivery("handler-c", at(0))] }
465
+ ]);
466
+ const [processing] = await claim(["handler-c"], at(0));
467
+ const [dead] = await claim(["handler-a"], at(100), 1);
468
+ await store.bury(dead, { name: "Error", message: "x" }, at(101));
469
+ vitest.expect(await store.statistics(at(1e3))).toEqual({
470
+ pending: 2,
471
+ processing: 1,
472
+ completed: 0,
473
+ dead: 1,
474
+ oldestDueAt: at(200)
475
+ });
476
+ vitest.expect((await store.statistics(at(LEASE_MS))).oldestDueAt).toEqual(at(200));
477
+ await store.complete(processing, at(1));
478
+ vitest.expect(await store.statistics(at(0))).toMatchObject({
479
+ completed: keepsCompleted ? 1 : 0,
480
+ processing: 0,
481
+ oldestDueAt: null
770
482
  });
771
- vitest.it("rejects made-up leases on pending deliveries and requeues only dead ones", async () => {
772
- await append([{ event: createEnvelope(), deliveries: [createDelivery("a")] }]);
773
- const [pending] = await store.listDeliveries({});
774
- const fake = { deliveryId: pending.id, leaseToken: core.createUuidV7() };
775
- vitest.expect(await store.reschedule(fake, at(1), { name: "Error", message: "x" }, at(1))).toBe(
776
- false
777
- );
778
- vitest.expect(await store.bury(fake, { name: "Error", message: "x" }, at(1))).toBe(false);
779
- const [processing] = await claim(["a"]);
780
- vitest.expect(await store.requeue([processing.deliveryId], at(1))).toBe(0);
781
- vitest.expect(await store.purge({ completedBefore: at(1e9), deadBefore: at(1e9) })).toEqual({
483
+ });
484
+ vitest.it("counts an expired lease as due", async () => {
485
+ await append([
486
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a", at(LEASE_MS * 5))] }
487
+ ]);
488
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
489
+ await claim(["handler-b"], at(0));
490
+ vitest.expect((await store.statistics(at(LEASE_MS + 1))).oldestDueAt).toEqual(at(LEASE_MS));
491
+ });
492
+ });
493
+ vitest.describe("purge", () => {
494
+ async function appendDoneFailedShared() {
495
+ const done = createEnvelope();
496
+ const failed = createEnvelope();
497
+ const shared = createEnvelope();
498
+ await append([
499
+ { event: done, deliveries: [createDelivery("handler-a")] },
500
+ { event: failed, deliveries: [createDelivery("handler-b")] },
501
+ { event: shared, deliveries: [createDelivery("handler-c"), createDelivery("handler-d")] }
502
+ ]);
503
+ const [doneDelivery] = await claim(["handler-a"]);
504
+ await store.complete(doneDelivery, at(10));
505
+ const [failedDelivery] = await claim(["handler-b"]);
506
+ await store.bury(failedDelivery, { name: "Error", message: "x" }, at(10));
507
+ const [sharedDelivery] = await claim(["handler-c"]);
508
+ await store.complete(sharedDelivery, at(10));
509
+ return { shared };
510
+ }
511
+ vitest.it.runIf(keepsCompleted)(
512
+ "deletes old completed and dead deliveries and their orphaned events",
513
+ async () => {
514
+ const { shared } = await appendDoneFailedShared();
515
+ vitest.expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
516
+ vitest.expect(await store.purge({ completedBefore: at(10) })).toEqual({
782
517
  deletedDeliveries: 0,
783
518
  deletedEvents: 0
784
519
  });
785
- vitest.expect(await store.listDeliveries({})).toHaveLength(1);
786
- });
520
+ vitest.expect(await store.purge({ completedBefore: at(11) })).toEqual({
521
+ deletedDeliveries: 2,
522
+ deletedEvents: 1
523
+ });
524
+ vitest.expect(await store.purge({ deadBefore: at(11) })).toEqual({
525
+ deletedDeliveries: 1,
526
+ deletedEvents: 1
527
+ });
528
+ const remaining = await store.listDeliveries({});
529
+ vitest.expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
530
+ vitest.expect(remaining[0].event.id).toBe(shared.id);
531
+ }
532
+ );
533
+ vitest.it.runIf(!keepsCompleted)(
534
+ "deletes old dead deliveries and the events no delivery references any more",
535
+ async () => {
536
+ const { shared } = await appendDoneFailedShared();
537
+ vitest.expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
538
+ vitest.expect(await store.purge({ completedBefore: at(0) })).toEqual({
539
+ deletedDeliveries: 0,
540
+ deletedEvents: 0
541
+ });
542
+ vitest.expect(await store.purge({ completedBefore: at(11) })).toEqual({
543
+ deletedDeliveries: 0,
544
+ deletedEvents: 1
545
+ });
546
+ vitest.expect(await store.purge({ deadBefore: at(11) })).toEqual({
547
+ deletedDeliveries: 1,
548
+ deletedEvents: 1
549
+ });
550
+ const remaining = await store.listDeliveries({});
551
+ vitest.expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
552
+ vitest.expect(remaining[0].event.id).toBe(shared.id);
553
+ }
554
+ );
555
+ vitest.it("lets a purged event id be appended again", async () => {
556
+ const event = createEnvelope();
557
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
558
+ const [delivery] = await claim(["handler-a"]);
559
+ await store.complete(delivery, at(1));
560
+ await store.purge({ completedBefore: at(2) });
561
+ const result = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
562
+ vitest.expect(result.appendedEventIds).toEqual([event.id]);
787
563
  });
788
- vitest.describe("data fidelity", () => {
789
- vitest.it.each([null, 0, 42.5, "text", [1, { two: 2 }], { deep: { list: [] } }, true])(
790
- "round-trips the payload %j",
791
- async (payload) => {
792
- const event = createEnvelope({ payload });
793
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
794
- const [claimed] = await claim(["handler-a"]);
795
- vitest.expect(claimed.event.payload).toEqual(payload);
564
+ });
565
+ vitest.describe("release", () => {
566
+ vitest.it("gives a leased delivery back without counting the attempt", async () => {
567
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
568
+ const [delivery] = await claim(["handler-a"], at(0));
569
+ vitest.expect(await store.release(delivery, at(500))).toBe(true);
570
+ const [record] = await store.listDeliveries({});
571
+ vitest.expect(record).toMatchObject({
572
+ status: "pending",
573
+ attempts: 0,
574
+ leaseOwner: null,
575
+ leaseExpiresAt: null
576
+ });
577
+ vitest.expect(record.availableAt).toEqual(at(500));
578
+ vitest.expect(record.updatedAt).toEqual(at(500));
579
+ vitest.expect(await store.release(delivery, at(501))).toBe(false);
580
+ const [again] = await claim(["handler-a"], at(500));
581
+ vitest.expect(again.attempt).toBe(1);
582
+ });
583
+ });
584
+ vitest.describe("transactions in flight", () => {
585
+ vitest.it("hides uncommitted appends from every reader", async () => {
586
+ const inside = createDeferred();
587
+ const finish = createDeferred();
588
+ const event = createEnvelope();
589
+ const open = store.transaction(async (transaction) => {
590
+ await store.append(
591
+ transaction,
592
+ [{ event, deliveries: [createDelivery("handler-a")] }],
593
+ at(0)
594
+ );
595
+ inside.resolve();
596
+ await finish.promise;
597
+ });
598
+ await inside.promise;
599
+ vitest.expect(await claim(["handler-a"])).toEqual([]);
600
+ vitest.expect(await store.listDeliveries({})).toEqual([]);
601
+ vitest.expect((await store.statistics(at(0))).pending).toBe(0);
602
+ finish.resolve();
603
+ await open;
604
+ vitest.expect(await claim(["handler-a"])).toHaveLength(1);
605
+ });
606
+ vitest.it("orders same-key events by commit, not by first write", async () => {
607
+ const first = createEnvelope({ key: "race" });
608
+ const second = createEnvelope({ key: "race" });
609
+ const firstWritten = createDeferred();
610
+ const releaseFirst = createDeferred();
611
+ const firstTransaction = store.transaction(async (transaction) => {
612
+ await store.append(
613
+ transaction,
614
+ [{ event: first, deliveries: [createDelivery("handler-a")] }],
615
+ at(0)
616
+ );
617
+ firstWritten.resolve();
618
+ await releaseFirst.promise;
619
+ });
620
+ await firstWritten.promise;
621
+ const secondTransaction = store.transaction(
622
+ (transaction) => store.append(
623
+ transaction,
624
+ [{ event: second, deliveries: [createDelivery("handler-a")] }],
625
+ at(0)
626
+ )
627
+ );
628
+ await new Promise((resolve) => setTimeout(resolve, 50));
629
+ releaseFirst.resolve();
630
+ await firstTransaction;
631
+ await secondTransaction;
632
+ const [head] = await claim(["handler-a"]);
633
+ vitest.expect(head.event.id).toBe(first.id);
634
+ await store.complete(head, at(1));
635
+ vitest.expect((await claim(["handler-a"]))[0].event.id).toBe(second.id);
636
+ });
637
+ vitest.it("stores an event id written by two concurrent transactions once", async () => {
638
+ const event = createEnvelope({ key: "same" });
639
+ const results = await Promise.all([
640
+ append([{ event, deliveries: [createDelivery("handler-a")] }]),
641
+ append([{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }])
642
+ ]);
643
+ const appended = results.flatMap((result) => result.appendedEventIds);
644
+ vitest.expect(appended).toEqual([event.id]);
645
+ const records = await store.listDeliveries({});
646
+ vitest.expect(records.map((record) => record.handler).sort()).toEqual(["handler-a", "handler-b"]);
647
+ });
648
+ });
649
+ }
650
+ function orderingAndLeasesContract(suite) {
651
+ const { append, claim } = suite;
652
+ let store;
653
+ vitest.beforeEach(() => {
654
+ store = suite.store();
655
+ });
656
+ vitest.describe("ordering keys under load", () => {
657
+ vitest.it("does not let a blocked key starve unrelated deliveries", async () => {
658
+ const blocked = Array.from({ length: 120 }, () => ({
659
+ event: createEnvelope({ key: "hot" }),
660
+ deliveries: [createDelivery("handler-a", at(0))]
661
+ }));
662
+ await append(blocked);
663
+ const unrelated = Array.from({ length: 5 }, () => ({
664
+ event: createEnvelope(),
665
+ deliveries: [createDelivery("handler-a", at(10))]
666
+ }));
667
+ await append(unrelated);
668
+ const claimed = await claim(["handler-a"], at(100), 10);
669
+ vitest.expect(claimed).toHaveLength(6);
670
+ vitest.expect(claimed.filter((delivery) => delivery.event.key === "hot")).toHaveLength(1);
671
+ });
672
+ vitest.it("keeps successors blocked behind an expired, delayed or requeued head", async () => {
673
+ const head = createEnvelope({ key: "k" });
674
+ const next = createEnvelope({ key: "k" });
675
+ await append([{ event: head, deliveries: [createDelivery("handler-a")] }]);
676
+ await append([{ event: next, deliveries: [createDelivery("handler-a")] }]);
677
+ const [claimedHead] = await claim(["handler-a"], at(0));
678
+ const reclaimed = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
679
+ vitest.expect(reclaimed.map((delivery) => delivery.event.id)).toEqual([head.id]);
680
+ vitest.expect(
681
+ await store.reschedule(
682
+ reclaimed[0],
683
+ at(LEASE_MS * 10),
684
+ { name: "Error", message: "x" },
685
+ at(LEASE_MS)
686
+ )
687
+ ).toBe(true);
688
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS * 5))).toEqual([]);
689
+ const [delayedHead] = await claim(["handler-a"], at(LEASE_MS * 10));
690
+ await store.bury(delayedHead, { name: "Error", message: "x" }, at(LEASE_MS * 10));
691
+ vitest.expect(await store.requeue([delayedHead.deliveryId], at(LEASE_MS * 11))).toBe(1);
692
+ const [requeuedHead] = await claim(["handler-a"], at(LEASE_MS * 11));
693
+ vitest.expect(requeuedHead.event.id).toBe(head.id);
694
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS * 11))).toEqual([]);
695
+ vitest.expect(claimedHead).toBeDefined();
696
+ });
697
+ });
698
+ vitest.describe("requeued keyed dead letters", () => {
699
+ vitest.it("never runs a requeued dead letter alongside its running successor", async () => {
700
+ const first = createEnvelope({ key: "k" });
701
+ const second = createEnvelope({ key: "k" });
702
+ await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
703
+ await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
704
+ const [head] = await claim(["handler-a"], at(0));
705
+ await store.bury(head, { name: "Error", message: "x" }, at(1));
706
+ const [successor] = await claim(["handler-a"], at(2));
707
+ vitest.expect(successor.event.id).toBe(second.id);
708
+ vitest.expect(await store.requeue([head.deliveryId], at(3))).toBe(1);
709
+ vitest.expect(await claim(["handler-a"], at(4))).toEqual([]);
710
+ await store.complete(successor, at(5));
711
+ const [requeued] = await claim(["handler-a"], at(6));
712
+ vitest.expect(requeued.event.id).toBe(first.id);
713
+ });
714
+ vitest.it("recovers when the running successor crashes after the requeue", async () => {
715
+ const first = createEnvelope({ key: "k" });
716
+ const second = createEnvelope({ key: "k" });
717
+ await append([{ event: first, deliveries: [createDelivery("handler-a")] }]);
718
+ await append([{ event: second, deliveries: [createDelivery("handler-a")] }]);
719
+ const [head] = await claim(["handler-a"], at(0));
720
+ await store.bury(head, { name: "Error", message: "x" }, at(1));
721
+ await claim(["handler-a"], at(2));
722
+ await store.requeue([head.deliveryId], at(3));
723
+ const recovered = await claim(["handler-a"], at(2 + LEASE_MS));
724
+ vitest.expect(recovered.map((delivery) => delivery.event.id)).toEqual([first.id]);
725
+ await store.complete(recovered[0], at(3 + LEASE_MS));
726
+ const [retried] = await claim(["handler-a"], at(4 + LEASE_MS));
727
+ vitest.expect(retried).toMatchObject({ attempt: 2 });
728
+ vitest.expect(retried.event.id).toBe(second.id);
729
+ });
730
+ });
731
+ vitest.describe("lease semantics", () => {
732
+ vitest.it("confirms a repeated renewal with identical values", async () => {
733
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
734
+ const [delivery] = await claim(["handler-a"], at(0));
735
+ vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
736
+ vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 2), at(10))).toBe(true);
737
+ });
738
+ vitest.it("accepts an expired lease that nobody reclaimed", async () => {
739
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
740
+ const [delivery] = await claim(["handler-a"], at(0));
741
+ vitest.expect(await store.extendLease(delivery, at(LEASE_MS * 3), at(LEASE_MS * 2))).toBe(true);
742
+ vitest.expect(await store.complete(delivery, at(LEASE_MS * 2))).toBe(true);
743
+ });
744
+ vitest.it("rejects every lease operation once the delivery left processing", async () => {
745
+ const error = { name: "Error", message: "x" };
746
+ await append([
747
+ {
748
+ event: createEnvelope(),
749
+ deliveries: [createDelivery("a"), createDelivery("b"), createDelivery("c")]
796
750
  }
751
+ ]);
752
+ const [completed] = await claim(["a"]);
753
+ await store.complete(completed, at(1));
754
+ const [buried] = await claim(["b"]);
755
+ await store.bury(buried, error, at(1));
756
+ const [rescheduled] = await claim(["c"]);
757
+ await store.reschedule(rescheduled, at(100), error, at(1));
758
+ for (const lease of [completed, buried, rescheduled]) {
759
+ vitest.expect(await store.extendLease(lease, at(LEASE_MS), at(2))).toBe(false);
760
+ vitest.expect(await store.reschedule(lease, at(3), error, at(2))).toBe(false);
761
+ vitest.expect(await store.bury(lease, error, at(2))).toBe(false);
762
+ vitest.expect(await store.release(lease, at(2))).toBe(false);
763
+ vitest.expect(await store.complete(lease, at(2))).toBe(false);
764
+ }
765
+ });
766
+ vitest.it("rejects made-up leases on pending deliveries and requeues only dead ones", async () => {
767
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("a")] }]);
768
+ const [pending] = await store.listDeliveries({});
769
+ const fake = { deliveryId: pending.id, leaseToken: core.createUuidV7() };
770
+ vitest.expect(await store.reschedule(fake, at(1), { name: "Error", message: "x" }, at(1))).toBe(
771
+ false
797
772
  );
798
- vitest.it("accepts long identifiers and non-UUID event ids", async () => {
799
- const handler = "h".repeat(255);
800
- const event = createEnvelope({
801
- id: `order-42:placed:${"x".repeat(200)}`,
802
- key: "k".repeat(255),
803
- type: "t".repeat(255)
804
- });
805
- await append([{ event, deliveries: [createDelivery(handler)] }]);
806
- const [claimed] = await claim([handler]);
807
- vitest.expect(claimed.event).toEqual(event);
808
- vitest.expect(claimed.handler).toBe(handler);
809
- });
810
- vitest.it("lists at most 100 deliveries by default", async () => {
811
- await append(
812
- Array.from({ length: 101 }, () => ({
813
- event: createEnvelope(),
814
- deliveries: [createDelivery("handler-a")]
815
- }))
816
- );
817
- vitest.expect(await store.listDeliveries({})).toHaveLength(100);
818
- });
819
- vitest.it("hands out copies that callers cannot use to change stored data", async () => {
820
- const event = createEnvelope({ payload: { amount: 1 }, headers: { a: "1" } });
773
+ vitest.expect(await store.bury(fake, { name: "Error", message: "x" }, at(1))).toBe(false);
774
+ const [processing] = await claim(["a"]);
775
+ vitest.expect(await store.requeue([processing.deliveryId], at(1))).toBe(0);
776
+ vitest.expect(await store.purge({ completedBefore: at(1e9), deadBefore: at(1e9) })).toEqual({
777
+ deletedDeliveries: 0,
778
+ deletedEvents: 0
779
+ });
780
+ vitest.expect(await store.listDeliveries({})).toHaveLength(1);
781
+ });
782
+ });
783
+ vitest.describe("data fidelity", () => {
784
+ vitest.it.each([null, 0, 42.5, "text", [1, { two: 2 }], { deep: { list: [] } }, true])(
785
+ "round-trips the payload %j",
786
+ async (payload) => {
787
+ const event = createEnvelope({ payload });
821
788
  await append([{ event, deliveries: [createDelivery("handler-a")] }]);
822
789
  const [claimed] = await claim(["handler-a"]);
823
- claimed.event.payload.amount = 999;
824
- claimed.event.headers["a"] = "changed";
825
- const [listed] = await store.listDeliveries({});
826
- vitest.expect(listed.event.payload).toEqual({ amount: 1 });
827
- vitest.expect(listed.event.headers).toEqual({ a: "1" });
828
- });
790
+ vitest.expect(claimed.event.payload).toEqual(payload);
791
+ }
792
+ );
793
+ vitest.it("accepts long identifiers and non-UUID event ids", async () => {
794
+ const handler = "h".repeat(255);
795
+ const event = createEnvelope({
796
+ id: `order-42:placed:${"x".repeat(200)}`,
797
+ key: "k".repeat(255),
798
+ type: "t".repeat(255)
799
+ });
800
+ await append([{ event, deliveries: [createDelivery(handler)] }]);
801
+ const [claimed] = await claim([handler]);
802
+ vitest.expect(claimed.event).toEqual(event);
803
+ vitest.expect(claimed.handler).toBe(handler);
829
804
  });
830
- vitest.describe("completion", () => {
831
- vitest.it("removes a completed delivery from every listing", async () => {
832
- const event = createEnvelope();
833
- await append([
834
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
835
- ]);
836
- const [delivery] = await claim(["handler-a"]);
837
- vitest.expect(await store.complete(delivery, at(5))).toBe(true);
838
- const records = await store.listDeliveries({ eventId: event.id });
839
- if (keepsCompleted) {
840
- vitest.expect(records.map((record) => [record.handler, record.status]).sort()).toEqual([
841
- ["handler-a", "completed"],
842
- ["handler-b", "pending"]
843
- ]);
844
- const [completed] = await store.listDeliveries({ status: "completed" });
845
- vitest.expect(completed).toMatchObject({ handler: "handler-a", attempts: 1 });
846
- vitest.expect(completed.completedAt).toEqual(at(5));
847
- } else {
848
- vitest.expect(records.map((record) => [record.handler, record.status])).toEqual([
849
- ["handler-b", "pending"]
850
- ]);
851
- vitest.expect(await store.listDeliveries({ status: "completed" })).toEqual([]);
852
- }
853
- vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
854
- vitest.expect(await store.statistics(at(0))).toMatchObject({
855
- pending: 1,
856
- processing: 0,
857
- completed: keepsCompleted ? 1 : 0
858
- });
859
- });
860
- vitest.it("does not create the delivery of a handler that already completed again", async () => {
861
- const event = createEnvelope();
862
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
863
- const [delivery] = await claim(["handler-a"]);
864
- await store.complete(delivery, at(1));
865
- const result = await append([
866
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
867
- ]);
868
- vitest.expect(result).toEqual({
869
- appendedEventIds: [],
870
- duplicateEventIds: [event.id],
871
- appendedDeliveryCount: 1
872
- });
873
- vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
874
- vitest.expect(await claim(["handler-b"], at(LEASE_MS * 10))).toHaveLength(1);
875
- const again = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
876
- vitest.expect(again.appendedDeliveryCount).toBe(0);
877
- });
878
- vitest.it("completes many leases at once and reports each one", async () => {
879
- await append(
880
- Array.from({ length: 5 }, () => ({
881
- event: createEnvelope(),
882
- deliveries: [createDelivery("handler-a")]
883
- }))
884
- );
885
- const claimed = await claim(["handler-a"]);
886
- vitest.expect(claimed).toHaveLength(5);
887
- vitest.expect(await completeMany(store, [], at(1))).toEqual([]);
888
- vitest.expect(await completeMany(store, claimed.slice(0, 3), at(1))).toEqual([true, true, true]);
889
- vitest.expect(await store.statistics(at(2))).toMatchObject({ processing: 2, pending: 0 });
890
- vitest.expect(await completeMany(store, claimed, at(2))).toEqual([
891
- false,
892
- false,
893
- false,
894
- true,
895
- true
805
+ vitest.it("lists at most 100 deliveries by default", async () => {
806
+ await append(
807
+ Array.from({ length: 101 }, () => ({
808
+ event: createEnvelope(),
809
+ deliveries: [createDelivery("handler-a")]
810
+ }))
811
+ );
812
+ vitest.expect(await store.listDeliveries({})).toHaveLength(100);
813
+ });
814
+ vitest.it("hands out copies that callers cannot use to change stored data", async () => {
815
+ const event = createEnvelope({ payload: { amount: 1 }, headers: { a: "1" } });
816
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
817
+ const [claimed] = await claim(["handler-a"]);
818
+ claimed.event.payload.amount = 999;
819
+ claimed.event.headers["a"] = "changed";
820
+ const [listed] = await store.listDeliveries({});
821
+ vitest.expect(listed.event.payload).toEqual({ amount: 1 });
822
+ vitest.expect(listed.event.headers).toEqual({ a: "1" });
823
+ });
824
+ });
825
+ }
826
+ function completionContract(suite) {
827
+ const { append, claim, completeMany, keepsCompleted } = suite;
828
+ let store;
829
+ vitest.beforeEach(() => {
830
+ store = suite.store();
831
+ });
832
+ vitest.describe("completion", () => {
833
+ vitest.it("removes a completed delivery from every listing", async () => {
834
+ const event = createEnvelope();
835
+ await append([
836
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
837
+ ]);
838
+ const [delivery] = await claim(["handler-a"]);
839
+ vitest.expect(await store.complete(delivery, at(5))).toBe(true);
840
+ const records = await store.listDeliveries({ eventId: event.id });
841
+ if (keepsCompleted) {
842
+ vitest.expect(records.map((record) => [record.handler, record.status]).sort()).toEqual([
843
+ ["handler-a", "completed"],
844
+ ["handler-b", "pending"]
896
845
  ]);
897
- vitest.expect(await store.statistics(at(3))).toMatchObject({ processing: 0, pending: 0 });
898
- });
899
- vitest.it("completes a lease listed twice only once", async () => {
900
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
901
- const [delivery] = await claim(["handler-a"]);
902
- vitest.expect(await completeMany(store, [delivery, delivery], at(1))).toEqual([true, false]);
903
- });
904
- vitest.it("fences every lease of a batch on its own", async () => {
905
- await append(
906
- Array.from({ length: 3 }, () => ({
907
- event: createEnvelope(),
908
- deliveries: [createDelivery("handler-a")]
909
- }))
910
- );
911
- const stale = await claim(["handler-a"], at(0));
912
- const current = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
913
- vitest.expect(current).toHaveLength(3);
914
- await append(
915
- [{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }],
916
- at(1)
917
- );
918
- const [fresh] = await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2");
919
- vitest.expect(
920
- await completeMany(store, [stale[0], fresh, stale[1], current[2]], at(LEASE_MS + 2))
921
- ).toEqual([false, true, false, true]);
922
- vitest.expect(await store.statistics(at(LEASE_MS + 3))).toMatchObject({ processing: 2 });
923
- });
924
- vitest.it("does not complete a lease of a delivery that went back to pending", async () => {
925
- const error = { name: "Error", message: "x" };
926
- await append([
927
- { event: createEnvelope(), deliveries: [createDelivery("a"), createDelivery("b")] }
846
+ const [completed] = await store.listDeliveries({ status: "completed" });
847
+ vitest.expect(completed).toMatchObject({ handler: "handler-a", attempts: 1 });
848
+ vitest.expect(completed.completedAt).toEqual(at(5));
849
+ } else {
850
+ vitest.expect(records.map((record) => [record.handler, record.status])).toEqual([
851
+ ["handler-b", "pending"]
928
852
  ]);
929
- const [retrying] = await claim(["a"]);
930
- const [released] = await claim(["b"]);
931
- await store.reschedule(retrying, at(1e3), error, at(1));
932
- await store.release(released, at(1));
933
- vitest.expect(await completeMany(store, [retrying, released], at(2))).toEqual([false, false]);
934
- vitest.expect(await store.statistics(at(2))).toMatchObject({ pending: 2, processing: 0 });
853
+ vitest.expect(await store.listDeliveries({ status: "completed" })).toEqual([]);
854
+ }
855
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
856
+ vitest.expect(await store.statistics(at(0))).toMatchObject({
857
+ pending: 1,
858
+ processing: 0,
859
+ completed: keepsCompleted ? 1 : 0
935
860
  });
936
861
  });
937
- vitest.describe("stale leases of one delivery", () => {
862
+ vitest.it("does not create the delivery of a handler that already completed again", async () => {
863
+ const event = createEnvelope();
864
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
865
+ const [delivery] = await claim(["handler-a"]);
866
+ await store.complete(delivery, at(1));
867
+ const result = await append([
868
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
869
+ ]);
870
+ vitest.expect(result).toEqual({
871
+ appendedEventIds: [],
872
+ duplicateEventIds: [event.id],
873
+ appendedDeliveryCount: 1
874
+ });
875
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
876
+ vitest.expect(await claim(["handler-b"], at(LEASE_MS * 10))).toHaveLength(1);
877
+ const again = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
878
+ vitest.expect(again.appendedDeliveryCount).toBe(0);
879
+ });
880
+ vitest.it("completes many leases at once and reports each one", async () => {
881
+ await append(
882
+ Array.from({ length: 5 }, () => ({
883
+ event: createEnvelope(),
884
+ deliveries: [createDelivery("handler-a")]
885
+ }))
886
+ );
887
+ const claimed = await claim(["handler-a"]);
888
+ vitest.expect(claimed).toHaveLength(5);
889
+ vitest.expect(await completeMany(store, [], at(1))).toEqual([]);
890
+ vitest.expect(await completeMany(store, claimed.slice(0, 3), at(1))).toEqual([true, true, true]);
891
+ vitest.expect(await store.statistics(at(2))).toMatchObject({ processing: 2, pending: 0 });
892
+ vitest.expect(await completeMany(store, claimed, at(2))).toEqual([false, false, false, true, true]);
893
+ vitest.expect(await store.statistics(at(3))).toMatchObject({ processing: 0, pending: 0 });
894
+ });
895
+ vitest.it("completes a lease listed twice only once", async () => {
896
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
897
+ const [delivery] = await claim(["handler-a"]);
898
+ vitest.expect(await completeMany(store, [delivery, delivery], at(1))).toEqual([true, false]);
899
+ });
900
+ vitest.it("fences every lease of a batch on its own", async () => {
901
+ await append(
902
+ Array.from({ length: 3 }, () => ({
903
+ event: createEnvelope(),
904
+ deliveries: [createDelivery("handler-a")]
905
+ }))
906
+ );
907
+ const stale = await claim(["handler-a"], at(0));
908
+ const current = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
909
+ vitest.expect(current).toHaveLength(3);
910
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }], at(1));
911
+ const [fresh] = await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2");
912
+ vitest.expect(
913
+ await completeMany(store, [stale[0], fresh, stale[1], current[2]], at(LEASE_MS + 2))
914
+ ).toEqual([false, true, false, true]);
915
+ vitest.expect(await store.statistics(at(LEASE_MS + 3))).toMatchObject({ processing: 2 });
916
+ });
917
+ vitest.it("does not complete a lease of a delivery that went back to pending", async () => {
938
918
  const error = { name: "Error", message: "x" };
939
- vitest.it("refuses the lease of an attempt that was released when the same owner claims it again", async () => {
940
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
941
- const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
942
- vitest.expect(await store.release(stale, at(1))).toBe(true);
943
- const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
944
- vitest.expect(current.deliveryId).toBe(stale.deliveryId);
945
- vitest.expect(current.leaseToken).not.toBe(stale.leaseToken);
946
- vitest.expect(await store.complete(stale, at(3))).toBe(false);
947
- vitest.expect(await store.bury(stale, error, at(3))).toBe(false);
948
- vitest.expect(await store.reschedule(stale, at(9), error, at(3))).toBe(false);
949
- vitest.expect(await store.release(stale, at(3))).toBe(false);
950
- vitest.expect(await store.extendLease(stale, at(LEASE_MS * 3), at(3))).toBe(false);
951
- vitest.expect(await store.complete(current, at(4))).toBe(true);
952
- });
953
- vitest.it("refuses the lease of an attempt that was rescheduled when the same owner claims it again", async () => {
954
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
955
- const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
956
- vitest.expect(await store.reschedule(stale, at(10), error, at(1))).toBe(true);
957
- const [current] = await claim(["handler-a"], at(10), 100, "worker-1");
958
- vitest.expect(current).toMatchObject({ deliveryId: stale.deliveryId, attempt: 2 });
959
- vitest.expect(await store.complete(stale, at(11))).toBe(false);
960
- vitest.expect(await store.bury(stale, error, at(11))).toBe(false);
961
- vitest.expect(await completeMany(store, [stale], at(11))).toEqual([false]);
962
- vitest.expect(await store.complete(current, at(12))).toBe(true);
963
- });
964
- vitest.it("matches the results of completeMany to the lease, not to the delivery id", async () => {
965
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
966
- const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
967
- await store.release(stale, at(1));
968
- const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
969
- vitest.expect(await completeMany(store, [stale, current], at(3))).toEqual([false, true]);
970
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
971
- const [staleB] = await claim(["handler-b"], at(4), 100, "worker-1");
972
- await store.release(staleB, at(5));
973
- const [currentB] = await claim(["handler-b"], at(6), 100, "worker-1");
974
- vitest.expect(await completeMany(store, [currentB, staleB], at(7))).toEqual([true, false]);
919
+ await append([
920
+ { event: createEnvelope(), deliveries: [createDelivery("a"), createDelivery("b")] }
921
+ ]);
922
+ const [retrying] = await claim(["a"]);
923
+ const [released] = await claim(["b"]);
924
+ await store.reschedule(retrying, at(1e3), error, at(1));
925
+ await store.release(released, at(1));
926
+ vitest.expect(await completeMany(store, [retrying, released], at(2))).toEqual([false, false]);
927
+ vitest.expect(await store.statistics(at(2))).toMatchObject({ pending: 2, processing: 0 });
928
+ });
929
+ });
930
+ vitest.describe("stale leases of one delivery", () => {
931
+ const error = { name: "Error", message: "x" };
932
+ vitest.it("refuses the lease of an attempt that was released when the same owner claims it again", async () => {
933
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
934
+ const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
935
+ vitest.expect(await store.release(stale, at(1))).toBe(true);
936
+ const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
937
+ vitest.expect(current.deliveryId).toBe(stale.deliveryId);
938
+ vitest.expect(current.leaseToken).not.toBe(stale.leaseToken);
939
+ vitest.expect(await store.complete(stale, at(3))).toBe(false);
940
+ vitest.expect(await store.bury(stale, error, at(3))).toBe(false);
941
+ vitest.expect(await store.reschedule(stale, at(9), error, at(3))).toBe(false);
942
+ vitest.expect(await store.release(stale, at(3))).toBe(false);
943
+ vitest.expect(await store.extendLease(stale, at(LEASE_MS * 3), at(3))).toBe(false);
944
+ vitest.expect(await store.complete(current, at(4))).toBe(true);
945
+ });
946
+ vitest.it("refuses the lease of an attempt that was rescheduled when the same owner claims it again", async () => {
947
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
948
+ const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
949
+ vitest.expect(await store.reschedule(stale, at(10), error, at(1))).toBe(true);
950
+ const [current] = await claim(["handler-a"], at(10), 100, "worker-1");
951
+ vitest.expect(current).toMatchObject({ deliveryId: stale.deliveryId, attempt: 2 });
952
+ vitest.expect(await store.complete(stale, at(11))).toBe(false);
953
+ vitest.expect(await store.bury(stale, error, at(11))).toBe(false);
954
+ vitest.expect(await completeMany(store, [stale], at(11))).toEqual([false]);
955
+ vitest.expect(await store.complete(current, at(12))).toBe(true);
956
+ });
957
+ vitest.it("matches the results of completeMany to the lease, not to the delivery id", async () => {
958
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
959
+ const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
960
+ await store.release(stale, at(1));
961
+ const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
962
+ vitest.expect(await completeMany(store, [stale, current], at(3))).toEqual([false, true]);
963
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
964
+ const [staleB] = await claim(["handler-b"], at(4), 100, "worker-1");
965
+ await store.release(staleB, at(5));
966
+ const [currentB] = await claim(["handler-b"], at(6), 100, "worker-1");
967
+ vitest.expect(await completeMany(store, [currentB, staleB], at(7))).toEqual([true, false]);
968
+ });
969
+ });
970
+ vitest.describe("retention of the deduplication record", () => {
971
+ const DAY = 864e5;
972
+ vitest.it("keeps an event for the retention window after its last delivery completed, however old it is", async () => {
973
+ const event = createEnvelope();
974
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
975
+ const [delivery] = await claim(["handler-a"], at(30 * DAY));
976
+ vitest.expect(await store.complete(delivery, at(30 * DAY + 1))).toBe(true);
977
+ vitest.expect(await store.purge({ completedBefore: at(29 * DAY) })).toMatchObject({
978
+ deletedEvents: 0
979
+ });
980
+ const again = await append(
981
+ [{ event, deliveries: [createDelivery("handler-a")] }],
982
+ at(30 * DAY + 2)
983
+ );
984
+ vitest.expect(again.appendedDeliveryCount).toBe(0);
985
+ vitest.expect(await store.purge({ completedBefore: at(31 * DAY) })).toMatchObject({
986
+ deletedEvents: 1
975
987
  });
988
+ const afterwards = await append(
989
+ [{ event, deliveries: [createDelivery("handler-a")] }],
990
+ at(31 * DAY)
991
+ );
992
+ vitest.expect(afterwards.appendedEventIds).toEqual([event.id]);
976
993
  });
977
- vitest.describe("retention of the deduplication record", () => {
978
- const DAY = 864e5;
979
- vitest.it("keeps an event for the retention window after its last delivery completed, however old it is", async () => {
980
- const event = createEnvelope();
981
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
982
- const [delivery] = await claim(["handler-a"], at(30 * DAY));
983
- vitest.expect(await store.complete(delivery, at(30 * DAY + 1))).toBe(true);
984
- vitest.expect(await store.purge({ completedBefore: at(29 * DAY) })).toMatchObject({
985
- deletedEvents: 0
986
- });
987
- const again = await append(
988
- [{ event, deliveries: [createDelivery("handler-a")] }],
989
- at(30 * DAY + 2)
990
- );
991
- vitest.expect(again.appendedDeliveryCount).toBe(0);
992
- vitest.expect(await store.purge({ completedBefore: at(31 * DAY) })).toMatchObject({
993
- deletedEvents: 1
994
- });
995
- const afterwards = await append(
994
+ vitest.it("keeps the later completion time of an event when completions are recorded out of order", async () => {
995
+ const event = createEnvelope();
996
+ await append([
997
+ { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
998
+ ]);
999
+ const claimed = await claim(["handler-a", "handler-b"]);
1000
+ vitest.expect(claimed).toHaveLength(2);
1001
+ const [first, second] = claimed.sort(
1002
+ (left, right) => left.handler.localeCompare(right.handler)
1003
+ );
1004
+ vitest.expect(await store.complete(first, at(200))).toBe(true);
1005
+ vitest.expect(await store.complete(second, at(100))).toBe(true);
1006
+ vitest.expect(await store.purge({ completedBefore: at(150) })).toMatchObject({ deletedEvents: 0 });
1007
+ const again = await append(
1008
+ [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
1009
+ at(201)
1010
+ );
1011
+ vitest.expect(again.appendedDeliveryCount).toBe(0);
1012
+ });
1013
+ vitest.it("remembers every handler of an event appended twice in one transaction", async () => {
1014
+ const event = createEnvelope();
1015
+ const result = await store.transaction(async (transaction) => {
1016
+ await store.append(
1017
+ transaction,
996
1018
  [{ event, deliveries: [createDelivery("handler-a")] }],
997
- at(31 * DAY)
1019
+ at(0)
998
1020
  );
999
- vitest.expect(afterwards.appendedEventIds).toEqual([event.id]);
1000
- });
1001
- vitest.it("keeps the later completion time of an event when completions are recorded out of order", async () => {
1002
- const event = createEnvelope();
1003
- await append([
1004
- { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
1005
- ]);
1006
- const claimed = await claim(["handler-a", "handler-b"]);
1007
- vitest.expect(claimed).toHaveLength(2);
1008
- const [first, second] = claimed.sort(
1009
- (left, right) => left.handler.localeCompare(right.handler)
1010
- );
1011
- vitest.expect(await store.complete(first, at(200))).toBe(true);
1012
- vitest.expect(await store.complete(second, at(100))).toBe(true);
1013
- vitest.expect(await store.purge({ completedBefore: at(150) })).toMatchObject({ deletedEvents: 0 });
1014
- const again = await append(
1015
- [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
1016
- at(201)
1021
+ return store.append(
1022
+ transaction,
1023
+ [{ event, deliveries: [createDelivery("handler-b")] }],
1024
+ at(0)
1017
1025
  );
1018
- vitest.expect(again.appendedDeliveryCount).toBe(0);
1019
- });
1020
- vitest.it("remembers every handler of an event appended twice in one transaction", async () => {
1021
- const event = createEnvelope();
1022
- const result = await store.transaction(async (transaction) => {
1023
- await store.append(
1024
- transaction,
1025
- [{ event, deliveries: [createDelivery("handler-a")] }],
1026
- at(0)
1027
- );
1028
- return store.append(
1029
- transaction,
1030
- [{ event, deliveries: [createDelivery("handler-b")] }],
1031
- at(0)
1032
- );
1033
- });
1034
- vitest.expect(result).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 1 });
1035
- const claimed = await claim(["handler-a", "handler-b"]);
1036
- vitest.expect(claimed).toHaveLength(2);
1037
- for (const delivery of claimed) {
1038
- vitest.expect(await store.complete(delivery, at(1))).toBe(true);
1039
- }
1040
- const again = await append(
1041
- [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
1042
- at(2)
1043
- );
1044
- vitest.expect(again).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 0 });
1045
- });
1046
- vitest.it("does not let the dead-letter cutoff remove events whose deliveries completed", async () => {
1047
- const event = createEnvelope();
1048
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
1049
- const [delivery] = await claim(["handler-a"]);
1050
- await store.complete(delivery, at(1));
1051
- vitest.expect(await store.purge({ deadBefore: at(100 * DAY) })).toMatchObject({
1052
- deletedEvents: 0
1053
- });
1054
- const again = await append([{ event, deliveries: [createDelivery("handler-a")] }], at(2));
1055
- vitest.expect(again.appendedDeliveryCount).toBe(0);
1056
1026
  });
1027
+ vitest.expect(result).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 1 });
1028
+ const claimed = await claim(["handler-a", "handler-b"]);
1029
+ vitest.expect(claimed).toHaveLength(2);
1030
+ for (const delivery of claimed) {
1031
+ vitest.expect(await store.complete(delivery, at(1))).toBe(true);
1032
+ }
1033
+ const again = await append(
1034
+ [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
1035
+ at(2)
1036
+ );
1037
+ vitest.expect(again).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 0 });
1057
1038
  });
1058
- vitest.describe("lost leases", () => {
1059
- vitest.it("hands a delivery of a crashed worker to the next claimer with the attempt counted", async () => {
1060
- await append(
1061
- Array.from({ length: 4 }, () => ({
1062
- event: createEnvelope(),
1063
- deliveries: [createDelivery("handler-a")]
1064
- }))
1065
- );
1066
- const first = await claim(["handler-a"], at(0), 100, "worker-1");
1067
- vitest.expect(first.map((delivery) => delivery.attempt)).toEqual([1, 1, 1, 1]);
1068
- vitest.expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
1069
- const second = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1070
- vitest.expect(second.map((delivery) => delivery.deliveryId).sort()).toEqual(
1071
- first.map((delivery) => delivery.deliveryId).sort()
1072
- );
1073
- vitest.expect(second.map((delivery) => delivery.attempt)).toEqual([2, 2, 2, 2]);
1074
- for (const stale of first) {
1075
- vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
1076
- }
1077
- vitest.expect(await completeMany(store, first, at(LEASE_MS + 1))).toEqual([
1078
- false,
1079
- false,
1080
- false,
1081
- false
1082
- ]);
1083
- vitest.expect(await completeMany(store, second, at(LEASE_MS + 2))).toEqual([
1084
- true,
1085
- true,
1086
- true,
1087
- true
1088
- ]);
1089
- });
1090
- vitest.it("keeps counting attempts of a poison delivery until it is dead-lettered", async () => {
1091
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1092
- for (let crash = 0; crash < 4; crash += 1) {
1093
- const [delivery] = await claim(
1094
- ["handler-a"],
1095
- at(crash * LEASE_MS),
1096
- 100,
1097
- `worker-${crash}`
1098
- );
1099
- vitest.expect(delivery.attempt).toBe(crash + 1);
1100
- }
1101
- const [last] = await claim(["handler-a"], at(4 * LEASE_MS), 100, "worker-4");
1102
- vitest.expect(last.attempt).toBe(5);
1103
- vitest.expect(
1104
- await store.bury(last, { name: "Error", message: "poison" }, at(4 * LEASE_MS))
1105
- ).toBe(true);
1106
- const [dead] = await store.listDeliveries({ status: "dead" });
1107
- vitest.expect(dead).toMatchObject({ attempts: 5, lastError: { message: "poison" } });
1108
- vitest.expect(await claim(["handler-a"], at(100 * LEASE_MS))).toEqual([]);
1109
- });
1110
- vitest.it("does not count an attempt that was reported as failed twice", async () => {
1111
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1112
- const [delivery] = await claim(["handler-a"], at(0), 100, "worker-1");
1113
- await store.reschedule(delivery, at(10), { name: "Error", message: "x" }, at(1));
1114
- const [retry] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1115
- vitest.expect(retry.attempt).toBe(2);
1116
- const [afterCrash] = await claim(["handler-a"], at(LEASE_MS * 2), 100, "worker-3");
1117
- vitest.expect(afterCrash.attempt).toBe(3);
1118
- });
1119
- vitest.it("keeps the order of a key across a takeover", async () => {
1120
- const [head, next] = await (async () => {
1121
- const events = [createEnvelope({ key: "k" }), createEnvelope({ key: "k" })];
1122
- for (const event of events) {
1123
- await append([{ event, deliveries: [createDelivery("handler-a")] }]);
1124
- }
1125
- return events;
1126
- })();
1127
- const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
1128
- vitest.expect(stale.event.id).toBe(head.id);
1129
- const taken = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1130
- vitest.expect(taken.map((delivery) => delivery.event.id)).toEqual([head.id]);
1039
+ vitest.it("does not let the dead-letter cutoff remove events whose deliveries completed", async () => {
1040
+ const event = createEnvelope();
1041
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
1042
+ const [delivery] = await claim(["handler-a"]);
1043
+ await store.complete(delivery, at(1));
1044
+ vitest.expect(await store.purge({ deadBefore: at(100 * DAY) })).toMatchObject({
1045
+ deletedEvents: 0
1046
+ });
1047
+ const again = await append([{ event, deliveries: [createDelivery("handler-a")] }], at(2));
1048
+ vitest.expect(again.appendedDeliveryCount).toBe(0);
1049
+ });
1050
+ });
1051
+ }
1052
+ function lostLeasesContract(suite) {
1053
+ const { append, claim, completeMany, options } = suite;
1054
+ let store;
1055
+ vitest.beforeEach(() => {
1056
+ store = suite.store();
1057
+ });
1058
+ vitest.describe("lost leases", () => {
1059
+ vitest.it("hands a delivery of a crashed worker to the next claimer with the attempt counted", async () => {
1060
+ await append(
1061
+ Array.from({ length: 4 }, () => ({
1062
+ event: createEnvelope(),
1063
+ deliveries: [createDelivery("handler-a")]
1064
+ }))
1065
+ );
1066
+ const first = await claim(["handler-a"], at(0), 100, "worker-1");
1067
+ vitest.expect(first.map((delivery) => delivery.attempt)).toEqual([1, 1, 1, 1]);
1068
+ vitest.expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
1069
+ const second = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1070
+ vitest.expect(second.map((delivery) => delivery.deliveryId).sort()).toEqual(
1071
+ first.map((delivery) => delivery.deliveryId).sort()
1072
+ );
1073
+ vitest.expect(second.map((delivery) => delivery.attempt)).toEqual([2, 2, 2, 2]);
1074
+ for (const stale of first) {
1131
1075
  vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
1132
- vitest.expect(await store.complete(taken[0], at(LEASE_MS + 2))).toBe(true);
1133
- const [following] = await claim(["handler-a"], at(LEASE_MS + 3), 100, "worker-2");
1134
- vitest.expect(following.event.id).toBe(next.id);
1135
- });
1136
- vitest.it("lets every worker see only deliveries nobody else holds, and none twice", async () => {
1137
- await append(
1138
- Array.from({ length: 30 }, () => ({
1139
- event: createEnvelope(),
1140
- deliveries: [createDelivery("handler-a")]
1141
- }))
1142
- );
1143
- const seen = [];
1144
- for (let round = 0; round < 40; round += 1) {
1145
- const owners = ["worker-1", "worker-2", "worker-3"];
1146
- const batches = await Promise.all(
1147
- owners.map((owner) => claim(["handler-a"], at(round), 4, owner))
1148
- );
1149
- for (const batch of batches) {
1150
- seen.push(...batch.map((delivery) => delivery.deliveryId));
1151
- vitest.expect(await completeMany(store, batch, at(round))).toEqual(batch.map(() => true));
1152
- }
1153
- }
1154
- vitest.expect(seen).toHaveLength(30);
1155
- vitest.expect(new Set(seen).size).toBe(30);
1156
- });
1076
+ }
1077
+ vitest.expect(await completeMany(store, first, at(LEASE_MS + 1))).toEqual([
1078
+ false,
1079
+ false,
1080
+ false,
1081
+ false
1082
+ ]);
1083
+ vitest.expect(await completeMany(store, second, at(LEASE_MS + 2))).toEqual([true, true, true, true]);
1084
+ });
1085
+ vitest.it("keeps counting attempts of a poison delivery until it is dead-lettered", async () => {
1086
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1087
+ for (let crash = 0; crash < 4; crash += 1) {
1088
+ const [delivery] = await claim(["handler-a"], at(crash * LEASE_MS), 100, `worker-${crash}`);
1089
+ vitest.expect(delivery.attempt).toBe(crash + 1);
1090
+ }
1091
+ const [last] = await claim(["handler-a"], at(4 * LEASE_MS), 100, "worker-4");
1092
+ vitest.expect(last.attempt).toBe(5);
1093
+ vitest.expect(await store.bury(last, { name: "Error", message: "poison" }, at(4 * LEASE_MS))).toBe(
1094
+ true
1095
+ );
1096
+ const [dead] = await store.listDeliveries({ status: "dead" });
1097
+ vitest.expect(dead).toMatchObject({ attempts: 5, lastError: { message: "poison" } });
1098
+ vitest.expect(await claim(["handler-a"], at(100 * LEASE_MS))).toEqual([]);
1099
+ });
1100
+ vitest.it("does not count an attempt that was reported as failed twice", async () => {
1101
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1102
+ const [delivery] = await claim(["handler-a"], at(0), 100, "worker-1");
1103
+ await store.reschedule(delivery, at(10), { name: "Error", message: "x" }, at(1));
1104
+ const [retry] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1105
+ vitest.expect(retry.attempt).toBe(2);
1106
+ const [afterCrash] = await claim(["handler-a"], at(LEASE_MS * 2), 100, "worker-3");
1107
+ vitest.expect(afterCrash.attempt).toBe(3);
1157
1108
  });
1158
- vitest.describe("relinquish", () => {
1159
- vitest.it("hands idle leases to other workers at once and keeps leases with running work", async (context) => {
1160
- if (store.relinquish === void 0) {
1161
- return context.skip();
1109
+ vitest.it("keeps the order of a key across a takeover", async () => {
1110
+ const [head, next] = await (async () => {
1111
+ const events = [createEnvelope({ key: "k" }), createEnvelope({ key: "k" })];
1112
+ for (const event of events) {
1113
+ await append([{ event, deliveries: [createDelivery("handler-a")] }]);
1162
1114
  }
1163
- await append([
1164
- { event: createEnvelope(), deliveries: [createDelivery("handler-a")] },
1165
- { event: createEnvelope(), deliveries: [createDelivery("handler-a")] }
1166
- ]);
1167
- const claimed = await claim(["handler-a"], at(0), 100, "worker-1");
1168
- vitest.expect(claimed).toHaveLength(2);
1169
- await store.relinquish("worker-1", at(1));
1170
- vitest.expect(await store.complete(claimed[0], at(2))).toBe(true);
1171
- vitest.expect(await store.complete(claimed[1], at(2))).toBe(true);
1172
- await store.relinquish("worker-1", at(3));
1173
- await append(
1174
- [{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }],
1175
- at(4)
1115
+ return events;
1116
+ })();
1117
+ const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
1118
+ vitest.expect(stale.event.id).toBe(head.id);
1119
+ const taken = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
1120
+ vitest.expect(taken.map((delivery) => delivery.event.id)).toEqual([head.id]);
1121
+ vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
1122
+ vitest.expect(await store.complete(taken[0], at(LEASE_MS + 2))).toBe(true);
1123
+ const [following] = await claim(["handler-a"], at(LEASE_MS + 3), 100, "worker-2");
1124
+ vitest.expect(following.event.id).toBe(next.id);
1125
+ });
1126
+ vitest.it("lets every worker see only deliveries nobody else holds, and none twice", async () => {
1127
+ await append(
1128
+ Array.from({ length: 30 }, () => ({
1129
+ event: createEnvelope(),
1130
+ deliveries: [createDelivery("handler-a")]
1131
+ }))
1132
+ );
1133
+ const seen = [];
1134
+ for (let round = 0; round < 40; round += 1) {
1135
+ const owners = ["worker-1", "worker-2", "worker-3"];
1136
+ const batches = await Promise.all(
1137
+ owners.map((owner) => claim(["handler-a"], at(round), 4, owner))
1176
1138
  );
1177
- vitest.expect(await claim(["handler-a"], at(4), 100, "worker-2")).toHaveLength(1);
1178
- });
1179
- vitest.it("does not give away a delivery that is still running", async (context) => {
1180
- if (store.relinquish === void 0) {
1181
- return context.skip();
1139
+ for (const batch of batches) {
1140
+ seen.push(...batch.map((delivery) => delivery.deliveryId));
1141
+ vitest.expect(await completeMany(store, batch, at(round))).toEqual(batch.map(() => true));
1182
1142
  }
1183
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1184
- const [running] = await claim(["handler-a"], at(0), 100, "worker-1");
1185
- await store.relinquish("worker-1", at(1));
1186
- vitest.expect(await claim(["handler-a"], at(2), 100, "worker-2")).toEqual([]);
1187
- vitest.expect(await store.complete(running, at(3))).toBe(true);
1188
- });
1143
+ }
1144
+ vitest.expect(seen).toHaveLength(30);
1145
+ vitest.expect(new Set(seen).size).toBe(30);
1189
1146
  });
1190
- vitest.describe.skipIf(options.createPeerStore === void 0)("across store instances", () => {
1191
- let peer;
1192
- vitest.beforeEach(async () => {
1193
- peer = await options.createPeerStore(store);
1194
- });
1195
- vitest.afterEach(async () => {
1196
- await options.destroyStore?.(peer);
1197
- });
1198
- vitest.it("never gives one delivery to claimers in different processes", async () => {
1199
- await append(
1200
- Array.from({ length: 60 }, (_, index) => ({
1201
- event: createEnvelope({ key: index % 3 === 0 ? `key-${index % 9}` : null }),
1202
- deliveries: [createDelivery("handler-a")]
1203
- }))
1204
- );
1205
- const request = (owner) => ({
1206
- handlers: ["handler-a"],
1207
- limit: 7,
1208
- leaseOwner: owner,
1209
- leaseExpiresAt: at(LEASE_MS),
1210
- now: at(0)
1211
- });
1212
- const batches = await Promise.all([
1213
- ...Array.from({ length: 5 }, (_, index) => store.claim(request(`primary-${index}`))),
1214
- ...Array.from({ length: 5 }, (_, index) => peer.claim(request(`peer-${index}`)))
1215
- ]);
1216
- const claimed = batches.flat();
1217
- vitest.expect(new Set(claimed.map((delivery) => delivery.deliveryId)).size).toBe(claimed.length);
1218
- const keys = claimed.map((delivery) => delivery.event.key).filter((key) => key !== null);
1219
- vitest.expect(new Set(keys).size).toBe(keys.length);
1220
- });
1221
- vitest.it("fences a lease taken over by another process", async () => {
1222
- await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1223
- const [stale] = await claim(["handler-a"], at(0));
1224
- const [current] = await peer.claim({
1225
- handlers: ["handler-a"],
1226
- limit: 1,
1227
- leaseOwner: "peer",
1228
- leaseExpiresAt: at(LEASE_MS * 2),
1229
- now: at(LEASE_MS)
1230
- });
1231
- vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
1232
- vitest.expect(await peer.complete(current, at(LEASE_MS + 1))).toBe(true);
1147
+ });
1148
+ vitest.describe("relinquish", () => {
1149
+ vitest.it("hands idle leases to other workers at once and keeps leases with running work", async (context) => {
1150
+ if (store.relinquish === void 0) {
1151
+ return context.skip();
1152
+ }
1153
+ await append([
1154
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a")] },
1155
+ { event: createEnvelope(), deliveries: [createDelivery("handler-a")] }
1156
+ ]);
1157
+ const claimed = await claim(["handler-a"], at(0), 100, "worker-1");
1158
+ vitest.expect(claimed).toHaveLength(2);
1159
+ await store.relinquish("worker-1", at(1));
1160
+ vitest.expect(await store.complete(claimed[0], at(2))).toBe(true);
1161
+ vitest.expect(await store.complete(claimed[1], at(2))).toBe(true);
1162
+ await store.relinquish("worker-1", at(3));
1163
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }], at(4));
1164
+ vitest.expect(await claim(["handler-a"], at(4), 100, "worker-2")).toHaveLength(1);
1165
+ });
1166
+ vitest.it("does not give away a delivery that is still running", async (context) => {
1167
+ if (store.relinquish === void 0) {
1168
+ return context.skip();
1169
+ }
1170
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1171
+ const [running] = await claim(["handler-a"], at(0), 100, "worker-1");
1172
+ await store.relinquish("worker-1", at(1));
1173
+ vitest.expect(await claim(["handler-a"], at(2), 100, "worker-2")).toEqual([]);
1174
+ vitest.expect(await store.complete(running, at(3))).toBe(true);
1175
+ });
1176
+ });
1177
+ vitest.describe.skipIf(options.createPeerStore === void 0)("across store instances", () => {
1178
+ let peer;
1179
+ vitest.beforeEach(async () => {
1180
+ peer = await options.createPeerStore(store);
1181
+ });
1182
+ vitest.afterEach(async () => {
1183
+ await options.destroyStore?.(peer);
1184
+ });
1185
+ vitest.it("never gives one delivery to claimers in different processes", async () => {
1186
+ await append(
1187
+ Array.from({ length: 60 }, (_, index) => ({
1188
+ event: createEnvelope({ key: index % 3 === 0 ? `key-${index % 9}` : null }),
1189
+ deliveries: [createDelivery("handler-a")]
1190
+ }))
1191
+ );
1192
+ const request = (owner) => ({
1193
+ handlers: ["handler-a"],
1194
+ limit: 7,
1195
+ leaseOwner: owner,
1196
+ leaseExpiresAt: at(LEASE_MS),
1197
+ now: at(0)
1198
+ });
1199
+ const batches = await Promise.all([
1200
+ ...Array.from({ length: 5 }, (_, index) => store.claim(request(`primary-${index}`))),
1201
+ ...Array.from({ length: 5 }, (_, index) => peer.claim(request(`peer-${index}`)))
1202
+ ]);
1203
+ const claimed = batches.flat();
1204
+ vitest.expect(new Set(claimed.map((delivery) => delivery.deliveryId)).size).toBe(claimed.length);
1205
+ const keys = claimed.map((delivery) => delivery.event.key).filter((key) => key !== null);
1206
+ vitest.expect(new Set(keys).size).toBe(keys.length);
1207
+ });
1208
+ vitest.it("fences a lease taken over by another process", async () => {
1209
+ await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
1210
+ const [stale] = await claim(["handler-a"], at(0));
1211
+ const [current] = await peer.claim({
1212
+ handlers: ["handler-a"],
1213
+ limit: 1,
1214
+ leaseOwner: "peer",
1215
+ leaseExpiresAt: at(LEASE_MS * 2),
1216
+ now: at(LEASE_MS)
1217
+ });
1218
+ vitest.expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
1219
+ vitest.expect(await peer.complete(current, at(LEASE_MS + 1))).toBe(true);
1220
+ });
1221
+ vitest.it("completes batches in either process and fences the one that lost its lease", async () => {
1222
+ await append(
1223
+ Array.from({ length: 6 }, () => ({
1224
+ event: createEnvelope(),
1225
+ deliveries: [createDelivery("handler-a")]
1226
+ }))
1227
+ );
1228
+ const request = (owner, now) => ({
1229
+ handlers: ["handler-a"],
1230
+ limit: 100,
1231
+ leaseOwner: owner,
1232
+ leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
1233
+ now
1233
1234
  });
1234
- vitest.it("completes batches in either process and fences the one that lost its lease", async () => {
1235
- await append(
1236
- Array.from({ length: 6 }, () => ({
1237
- event: createEnvelope(),
1238
- deliveries: [createDelivery("handler-a")]
1239
- }))
1240
- );
1241
- const request = (owner, now) => ({
1242
- handlers: ["handler-a"],
1243
- limit: 100,
1244
- leaseOwner: owner,
1245
- leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
1246
- now
1247
- });
1248
- const stale = await store.claim(request("primary", at(0)));
1249
- const current = await peer.claim(request("peer", at(LEASE_MS)));
1250
- vitest.expect(current).toHaveLength(6);
1251
- vitest.expect(await completeMany(store, stale, at(LEASE_MS + 1))).toEqual(stale.map(() => false));
1252
- vitest.expect(await completeMany(peer, current, at(LEASE_MS + 1))).toEqual(
1253
- current.map(() => true)
1254
- );
1255
- vitest.expect(await store.statistics(at(LEASE_MS + 2))).toMatchObject({
1256
- pending: 0,
1257
- processing: 0
1258
- });
1235
+ const stale = await store.claim(request("primary", at(0)));
1236
+ const current = await peer.claim(request("peer", at(LEASE_MS)));
1237
+ vitest.expect(current).toHaveLength(6);
1238
+ vitest.expect(await completeMany(store, stale, at(LEASE_MS + 1))).toEqual(stale.map(() => false));
1239
+ vitest.expect(await completeMany(peer, current, at(LEASE_MS + 1))).toEqual(current.map(() => true));
1240
+ vitest.expect(await store.statistics(at(LEASE_MS + 2))).toMatchObject({
1241
+ pending: 0,
1242
+ processing: 0
1259
1243
  });
1260
1244
  });
1261
1245
  });
1262
1246
  }
1263
1247
 
1248
+ // src/event-store-contract.ts
1249
+ function eventStoreContract(name, options) {
1250
+ const keepsCompleted = options.keepsCompletedRecords ?? false;
1251
+ vitest.describe(`${name} satisfies the EventStore contract`, () => {
1252
+ let store;
1253
+ vitest.beforeEach(async () => {
1254
+ store = await options.createStore();
1255
+ });
1256
+ vitest.afterEach(async () => {
1257
+ await options.destroyStore?.(store);
1258
+ });
1259
+ async function append(entries, now = at(0)) {
1260
+ return store.transaction((transaction) => store.append(transaction, entries, now));
1261
+ }
1262
+ async function completeMany(target, leases, now) {
1263
+ if (target.completeMany !== void 0) {
1264
+ return target.completeMany(leases, now);
1265
+ }
1266
+ const outcomes = [];
1267
+ for (const lease of leases) {
1268
+ outcomes.push(await target.complete(lease, now));
1269
+ }
1270
+ return outcomes;
1271
+ }
1272
+ async function claim(handlers, now = at(0), limit = 100, owner = "worker-1") {
1273
+ return store.claim({
1274
+ handlers,
1275
+ limit,
1276
+ leaseOwner: owner,
1277
+ leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
1278
+ now
1279
+ });
1280
+ }
1281
+ const suite = {
1282
+ options,
1283
+ keepsCompleted,
1284
+ store: () => store,
1285
+ append,
1286
+ claim,
1287
+ completeMany
1288
+ };
1289
+ appendAndClaimContract(suite);
1290
+ fencingContract(suite);
1291
+ maintenanceContract(suite);
1292
+ orderingAndLeasesContract(suite);
1293
+ completionContract(suite);
1294
+ lostLeasesContract(suite);
1295
+ });
1296
+ }
1297
+
1264
1298
  // src/fake-clock.ts
1265
1299
  var FakeClock = class {
1266
1300
  #now;