@relaybus/testing 0.0.0 → 0.2.0

package/dist/index.js

@@ -32,6 +32,7 @@ function createDelivery(handler, availableAt = at(0)) {
   return { id: createUuidV7(), handler, availableAt };
 }
 function eventStoreContract(name, options) {
+  const keepsCompleted = options.keepsCompletedRecords ?? false;
   describe(`${name} satisfies the EventStore contract`, () => {
     let store;
     beforeEach(async () => {
@@ -43,6 +44,16 @@ function eventStoreContract(name, options) {
     async function append(entries, now = at(0)) {
       return store.transaction((transaction) => store.append(transaction, entries, now));
     }
+    async function completeMany(target, leases, now) {
+      if (target.completeMany !== void 0) {
+        return target.completeMany(leases, now);
+      }
+      const outcomes = [];
+      for (const lease of leases) {
+        outcomes.push(await target.complete(lease, now));
+      }
+      return outcomes;
+    }
     async function claim(handlers, now = at(0), limit = 100, owner = "worker-1") {
       return store.claim({
         handlers,
@@ -219,12 +230,19 @@ function eventStoreContract(name, options) {
         }));
         await append(entries);
         const batches = await Promise.all(
-          Array.from(
-            { length: 8 },
-            (_, index) => claim(["handler-a"], at(0), 10, `worker-${index}`)
-          )
+          Array.from({ length: 8 }, async (_, index) => {
+            const own = [];
+            for (; ; ) {
+              const batch = await claim(["handler-a"], at(0), 10, `worker-${index}`);
+              if (batch.length === 0) {
+                return own;
+              }
+              own.push(...batch);
+            }
+          })
         );
-        const claimedIds = batches.flat().map((delivery) => delivery.deliveryId);
+        const leftovers = await claim(["handler-a"], at(0), 100, "worker-late");
+        const claimedIds = [...batches.flat(), ...leftovers].map((delivery) => delivery.deliveryId);
         expect(new Set(claimedIds).size).toBe(claimedIds.length);
         expect(claimedIds).toHaveLength(40);
       });
@@ -243,6 +261,10 @@ function eventStoreContract(name, options) {
           false
         );
         expect(await store.complete(current, at(LEASE_MS + 2))).toBe(true);
+        if (!keepsCompleted) {
+          expect(await store.listDeliveries({})).toEqual([]);
+          return;
+        }
         const [record] = await store.listDeliveries({});
         expect(record).toMatchObject({
           status: "completed",
@@ -283,7 +305,11 @@ function eventStoreContract(name, options) {
         await store.transaction(async (transaction) => {
           expect(await store.complete(delivery, at(2), transaction)).toBe(true);
         });
-        expect((await store.listDeliveries({}))[0].status).toBe("completed");
+        if (keepsCompleted) {
+          expect((await store.listDeliveries({}))[0].status).toBe("completed");
+        } else {
+          expect(await store.listDeliveries({})).toEqual([]);
+        }
       });
       it("refuses a transactional completion with a lost lease", async () => {
         await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
@@ -457,7 +483,11 @@ function eventStoreContract(name, options) {
         });
         expect((await store.statistics(at(LEASE_MS))).oldestDueAt).toEqual(at(200));
         await store.complete(processing, at(1));
-        expect(await store.statistics(at(0))).toMatchObject({ completed: 1, oldestDueAt: null });
+        expect(await store.statistics(at(0))).toMatchObject({
+          completed: keepsCompleted ? 1 : 0,
+          processing: 0,
+          oldestDueAt: null
+        });
       });
       it("counts an expired lease as due", async () => {
         await append([
@@ -469,7 +499,7 @@ function eventStoreContract(name, options) {
       });
     });
     describe("purge", () => {
-      it("deletes old completed and dead deliveries and their orphaned events", async () => {
+      async function appendDoneFailedShared() {
         const done = createEnvelope();
         const failed = createEnvelope();
         const shared = createEnvelope();
@@ -484,23 +514,52 @@ function eventStoreContract(name, options) {
         await store.bury(failedDelivery, { name: "Error", message: "x" }, at(10));
         const [sharedDelivery] = await claim(["handler-c"]);
         await store.complete(sharedDelivery, at(10));
-        expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
-        expect(await store.purge({ completedBefore: at(10) })).toEqual({
-          deletedDeliveries: 0,
-          deletedEvents: 0
-        });
-        expect(await store.purge({ completedBefore: at(11) })).toEqual({
-          deletedDeliveries: 2,
-          deletedEvents: 1
-        });
-        expect(await store.purge({ deadBefore: at(11) })).toEqual({
-          deletedDeliveries: 1,
-          deletedEvents: 1
-        });
-        const remaining = await store.listDeliveries({});
-        expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
-        expect(remaining[0].event.id).toBe(shared.id);
-      });
+        return { shared };
+      }
+      it.runIf(keepsCompleted)(
+        "deletes old completed and dead deliveries and their orphaned events",
+        async () => {
+          const { shared } = await appendDoneFailedShared();
+          expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
+          expect(await store.purge({ completedBefore: at(10) })).toEqual({
+            deletedDeliveries: 0,
+            deletedEvents: 0
+          });
+          expect(await store.purge({ completedBefore: at(11) })).toEqual({
+            deletedDeliveries: 2,
+            deletedEvents: 1
+          });
+          expect(await store.purge({ deadBefore: at(11) })).toEqual({
+            deletedDeliveries: 1,
+            deletedEvents: 1
+          });
+          const remaining = await store.listDeliveries({});
+          expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
+          expect(remaining[0].event.id).toBe(shared.id);
+        }
+      );
+      it.runIf(!keepsCompleted)(
+        "deletes old dead deliveries and the events no delivery references any more",
+        async () => {
+          const { shared } = await appendDoneFailedShared();
+          expect(await store.purge({})).toEqual({ deletedDeliveries: 0, deletedEvents: 0 });
+          expect(await store.purge({ completedBefore: at(0) })).toEqual({
+            deletedDeliveries: 0,
+            deletedEvents: 0
+          });
+          expect(await store.purge({ completedBefore: at(11) })).toEqual({
+            deletedDeliveries: 0,
+            deletedEvents: 1
+          });
+          expect(await store.purge({ deadBefore: at(11) })).toEqual({
+            deletedDeliveries: 1,
+            deletedEvents: 1
+          });
+          const remaining = await store.listDeliveries({});
+          expect(remaining.map((record) => record.handler)).toEqual(["handler-d"]);
+          expect(remaining[0].event.id).toBe(shared.id);
+        }
+      );
       it("lets a purged event id be appended again", async () => {
         const event = createEnvelope();
         await append([{ event, deliveries: [createDelivery("handler-a")] }]);
@@ -766,6 +825,366 @@ function eventStoreContract(name, options) {
         expect(listed.event.headers).toEqual({ a: "1" });
       });
     });
+    describe("completion", () => {
+      it("removes a completed delivery from every listing", async () => {
+        const event = createEnvelope();
+        await append([
+          { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
+        ]);
+        const [delivery] = await claim(["handler-a"]);
+        expect(await store.complete(delivery, at(5))).toBe(true);
+        const records = await store.listDeliveries({ eventId: event.id });
+        if (keepsCompleted) {
+          expect(records.map((record) => [record.handler, record.status]).sort()).toEqual([
+            ["handler-a", "completed"],
+            ["handler-b", "pending"]
+          ]);
+          const [completed] = await store.listDeliveries({ status: "completed" });
+          expect(completed).toMatchObject({ handler: "handler-a", attempts: 1 });
+          expect(completed.completedAt).toEqual(at(5));
+        } else {
+          expect(records.map((record) => [record.handler, record.status])).toEqual([
+            ["handler-b", "pending"]
+          ]);
+          expect(await store.listDeliveries({ status: "completed" })).toEqual([]);
+        }
+        expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
+        expect(await store.statistics(at(0))).toMatchObject({
+          pending: 1,
+          processing: 0,
+          completed: keepsCompleted ? 1 : 0
+        });
+      });
+      it("does not create the delivery of a handler that already completed again", async () => {
+        const event = createEnvelope();
+        await append([{ event, deliveries: [createDelivery("handler-a")] }]);
+        const [delivery] = await claim(["handler-a"]);
+        await store.complete(delivery, at(1));
+        const result = await append([
+          { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
+        ]);
+        expect(result).toEqual({
+          appendedEventIds: [],
+          duplicateEventIds: [event.id],
+          appendedDeliveryCount: 1
+        });
+        expect(await claim(["handler-a"], at(LEASE_MS * 10))).toEqual([]);
+        expect(await claim(["handler-b"], at(LEASE_MS * 10))).toHaveLength(1);
+        const again = await append([{ event, deliveries: [createDelivery("handler-a")] }]);
+        expect(again.appendedDeliveryCount).toBe(0);
+      });
+      it("completes many leases at once and reports each one", async () => {
+        await append(
+          Array.from({ length: 5 }, () => ({
+            event: createEnvelope(),
+            deliveries: [createDelivery("handler-a")]
+          }))
+        );
+        const claimed = await claim(["handler-a"]);
+        expect(claimed).toHaveLength(5);
+        expect(await completeMany(store, [], at(1))).toEqual([]);
+        expect(await completeMany(store, claimed.slice(0, 3), at(1))).toEqual([true, true, true]);
+        expect(await store.statistics(at(2))).toMatchObject({ processing: 2, pending: 0 });
+        expect(await completeMany(store, claimed, at(2))).toEqual([
+          false,
+          false,
+          false,
+          true,
+          true
+        ]);
+        expect(await store.statistics(at(3))).toMatchObject({ processing: 0, pending: 0 });
+      });
+      it("completes a lease listed twice only once", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [delivery] = await claim(["handler-a"]);
+        expect(await completeMany(store, [delivery, delivery], at(1))).toEqual([true, false]);
+      });
+      it("fences every lease of a batch on its own", async () => {
+        await append(
+          Array.from({ length: 3 }, () => ({
+            event: createEnvelope(),
+            deliveries: [createDelivery("handler-a")]
+          }))
+        );
+        const stale = await claim(["handler-a"], at(0));
+        const current = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
+        expect(current).toHaveLength(3);
+        await append(
+          [{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }],
+          at(1)
+        );
+        const [fresh] = await claim(["handler-a"], at(LEASE_MS + 1), 100, "worker-2");
+        expect(
+          await completeMany(store, [stale[0], fresh, stale[1], current[2]], at(LEASE_MS + 2))
+        ).toEqual([false, true, false, true]);
+        expect(await store.statistics(at(LEASE_MS + 3))).toMatchObject({ processing: 2 });
+      });
+      it("does not complete a lease of a delivery that went back to pending", async () => {
+        const error = { name: "Error", message: "x" };
+        await append([
+          { event: createEnvelope(), deliveries: [createDelivery("a"), createDelivery("b")] }
+        ]);
+        const [retrying] = await claim(["a"]);
+        const [released] = await claim(["b"]);
+        await store.reschedule(retrying, at(1e3), error, at(1));
+        await store.release(released, at(1));
+        expect(await completeMany(store, [retrying, released], at(2))).toEqual([false, false]);
+        expect(await store.statistics(at(2))).toMatchObject({ pending: 2, processing: 0 });
+      });
+    });
+    describe("stale leases of one delivery", () => {
+      const error = { name: "Error", message: "x" };
+      it("refuses the lease of an attempt that was released when the same owner claims it again", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
+        expect(await store.release(stale, at(1))).toBe(true);
+        const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
+        expect(current.deliveryId).toBe(stale.deliveryId);
+        expect(current.leaseToken).not.toBe(stale.leaseToken);
+        expect(await store.complete(stale, at(3))).toBe(false);
+        expect(await store.bury(stale, error, at(3))).toBe(false);
+        expect(await store.reschedule(stale, at(9), error, at(3))).toBe(false);
+        expect(await store.release(stale, at(3))).toBe(false);
+        expect(await store.extendLease(stale, at(LEASE_MS * 3), at(3))).toBe(false);
+        expect(await store.complete(current, at(4))).toBe(true);
+      });
+      it("refuses the lease of an attempt that was rescheduled when the same owner claims it again", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
+        expect(await store.reschedule(stale, at(10), error, at(1))).toBe(true);
+        const [current] = await claim(["handler-a"], at(10), 100, "worker-1");
+        expect(current).toMatchObject({ deliveryId: stale.deliveryId, attempt: 2 });
+        expect(await store.complete(stale, at(11))).toBe(false);
+        expect(await store.bury(stale, error, at(11))).toBe(false);
+        expect(await completeMany(store, [stale], at(11))).toEqual([false]);
+        expect(await store.complete(current, at(12))).toBe(true);
+      });
+      it("matches the results of completeMany to the lease, not to the delivery id", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
+        await store.release(stale, at(1));
+        const [current] = await claim(["handler-a"], at(2), 100, "worker-1");
+        expect(await completeMany(store, [stale, current], at(3))).toEqual([false, true]);
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-b")] }]);
+        const [staleB] = await claim(["handler-b"], at(4), 100, "worker-1");
+        await store.release(staleB, at(5));
+        const [currentB] = await claim(["handler-b"], at(6), 100, "worker-1");
+        expect(await completeMany(store, [currentB, staleB], at(7))).toEqual([true, false]);
+      });
+    });
+    describe("retention of the deduplication record", () => {
+      const DAY = 864e5;
+      it("keeps an event for the retention window after its last delivery completed, however old it is", async () => {
+        const event = createEnvelope();
+        await append([{ event, deliveries: [createDelivery("handler-a")] }]);
+        const [delivery] = await claim(["handler-a"], at(30 * DAY));
+        expect(await store.complete(delivery, at(30 * DAY + 1))).toBe(true);
+        expect(await store.purge({ completedBefore: at(29 * DAY) })).toMatchObject({
+          deletedEvents: 0
+        });
+        const again = await append(
+          [{ event, deliveries: [createDelivery("handler-a")] }],
+          at(30 * DAY + 2)
+        );
+        expect(again.appendedDeliveryCount).toBe(0);
+        expect(await store.purge({ completedBefore: at(31 * DAY) })).toMatchObject({
+          deletedEvents: 1
+        });
+        const afterwards = await append(
+          [{ event, deliveries: [createDelivery("handler-a")] }],
+          at(31 * DAY)
+        );
+        expect(afterwards.appendedEventIds).toEqual([event.id]);
+      });
+      it("keeps the later completion time of an event when completions are recorded out of order", async () => {
+        const event = createEnvelope();
+        await append([
+          { event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }
+        ]);
+        const claimed = await claim(["handler-a", "handler-b"]);
+        expect(claimed).toHaveLength(2);
+        const [first, second] = claimed.sort(
+          (left, right) => left.handler.localeCompare(right.handler)
+        );
+        expect(await store.complete(first, at(200))).toBe(true);
+        expect(await store.complete(second, at(100))).toBe(true);
+        expect(await store.purge({ completedBefore: at(150) })).toMatchObject({ deletedEvents: 0 });
+        const again = await append(
+          [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
+          at(201)
+        );
+        expect(again.appendedDeliveryCount).toBe(0);
+      });
+      it("remembers every handler of an event appended twice in one transaction", async () => {
+        const event = createEnvelope();
+        const result = await store.transaction(async (transaction) => {
+          await store.append(
+            transaction,
+            [{ event, deliveries: [createDelivery("handler-a")] }],
+            at(0)
+          );
+          return store.append(
+            transaction,
+            [{ event, deliveries: [createDelivery("handler-b")] }],
+            at(0)
+          );
+        });
+        expect(result).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 1 });
+        const claimed = await claim(["handler-a", "handler-b"]);
+        expect(claimed).toHaveLength(2);
+        for (const delivery of claimed) {
+          expect(await store.complete(delivery, at(1))).toBe(true);
+        }
+        const again = await append(
+          [{ event, deliveries: [createDelivery("handler-a"), createDelivery("handler-b")] }],
+          at(2)
+        );
+        expect(again).toMatchObject({ duplicateEventIds: [event.id], appendedDeliveryCount: 0 });
+      });
+      it("does not let the dead-letter cutoff remove events whose deliveries completed", async () => {
+        const event = createEnvelope();
+        await append([{ event, deliveries: [createDelivery("handler-a")] }]);
+        const [delivery] = await claim(["handler-a"]);
+        await store.complete(delivery, at(1));
+        expect(await store.purge({ deadBefore: at(100 * DAY) })).toMatchObject({
+          deletedEvents: 0
+        });
+        const again = await append([{ event, deliveries: [createDelivery("handler-a")] }], at(2));
+        expect(again.appendedDeliveryCount).toBe(0);
+      });
+    });
+    describe("lost leases", () => {
+      it("hands a delivery of a crashed worker to the next claimer with the attempt counted", async () => {
+        await append(
+          Array.from({ length: 4 }, () => ({
+            event: createEnvelope(),
+            deliveries: [createDelivery("handler-a")]
+          }))
+        );
+        const first = await claim(["handler-a"], at(0), 100, "worker-1");
+        expect(first.map((delivery) => delivery.attempt)).toEqual([1, 1, 1, 1]);
+        expect(await claim(["handler-a"], at(LEASE_MS - 1), 100, "worker-2")).toEqual([]);
+        const second = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
+        expect(second.map((delivery) => delivery.deliveryId).sort()).toEqual(
+          first.map((delivery) => delivery.deliveryId).sort()
+        );
+        expect(second.map((delivery) => delivery.attempt)).toEqual([2, 2, 2, 2]);
+        for (const stale of first) {
+          expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
+        }
+        expect(await completeMany(store, first, at(LEASE_MS + 1))).toEqual([
+          false,
+          false,
+          false,
+          false
+        ]);
+        expect(await completeMany(store, second, at(LEASE_MS + 2))).toEqual([
+          true,
+          true,
+          true,
+          true
+        ]);
+      });
+      it("keeps counting attempts of a poison delivery until it is dead-lettered", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        for (let crash = 0; crash < 4; crash += 1) {
+          const [delivery] = await claim(
+            ["handler-a"],
+            at(crash * LEASE_MS),
+            100,
+            `worker-${crash}`
+          );
+          expect(delivery.attempt).toBe(crash + 1);
+        }
+        const [last] = await claim(["handler-a"], at(4 * LEASE_MS), 100, "worker-4");
+        expect(last.attempt).toBe(5);
+        expect(
+          await store.bury(last, { name: "Error", message: "poison" }, at(4 * LEASE_MS))
+        ).toBe(true);
+        const [dead] = await store.listDeliveries({ status: "dead" });
+        expect(dead).toMatchObject({ attempts: 5, lastError: { message: "poison" } });
+        expect(await claim(["handler-a"], at(100 * LEASE_MS))).toEqual([]);
+      });
+      it("does not count an attempt that was reported as failed twice", async () => {
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [delivery] = await claim(["handler-a"], at(0), 100, "worker-1");
+        await store.reschedule(delivery, at(10), { name: "Error", message: "x" }, at(1));
+        const [retry] = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
+        expect(retry.attempt).toBe(2);
+        const [afterCrash] = await claim(["handler-a"], at(LEASE_MS * 2), 100, "worker-3");
+        expect(afterCrash.attempt).toBe(3);
+      });
+      it("keeps the order of a key across a takeover", async () => {
+        const [head, next] = await (async () => {
+          const events = [createEnvelope({ key: "k" }), createEnvelope({ key: "k" })];
+          for (const event of events) {
+            await append([{ event, deliveries: [createDelivery("handler-a")] }]);
+          }
+          return events;
+        })();
+        const [stale] = await claim(["handler-a"], at(0), 100, "worker-1");
+        expect(stale.event.id).toBe(head.id);
+        const taken = await claim(["handler-a"], at(LEASE_MS), 100, "worker-2");
+        expect(taken.map((delivery) => delivery.event.id)).toEqual([head.id]);
+        expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
+        expect(await store.complete(taken[0], at(LEASE_MS + 2))).toBe(true);
+        const [following] = await claim(["handler-a"], at(LEASE_MS + 3), 100, "worker-2");
+        expect(following.event.id).toBe(next.id);
+      });
+      it("lets every worker see only deliveries nobody else holds, and none twice", async () => {
+        await append(
+          Array.from({ length: 30 }, () => ({
+            event: createEnvelope(),
+            deliveries: [createDelivery("handler-a")]
+          }))
+        );
+        const seen = [];
+        for (let round = 0; round < 40; round += 1) {
+          const owners = ["worker-1", "worker-2", "worker-3"];
+          const batches = await Promise.all(
+            owners.map((owner) => claim(["handler-a"], at(round), 4, owner))
+          );
+          for (const batch of batches) {
+            seen.push(...batch.map((delivery) => delivery.deliveryId));
+            expect(await completeMany(store, batch, at(round))).toEqual(batch.map(() => true));
+          }
+        }
+        expect(seen).toHaveLength(30);
+        expect(new Set(seen).size).toBe(30);
+      });
+    });
+    describe("relinquish", () => {
+      it("hands idle leases to other workers at once and keeps leases with running work", async (context) => {
+        if (store.relinquish === void 0) {
+          return context.skip();
+        }
+        await append([
+          { event: createEnvelope(), deliveries: [createDelivery("handler-a")] },
+          { event: createEnvelope(), deliveries: [createDelivery("handler-a")] }
+        ]);
+        const claimed = await claim(["handler-a"], at(0), 100, "worker-1");
+        expect(claimed).toHaveLength(2);
+        await store.relinquish("worker-1", at(1));
+        expect(await store.complete(claimed[0], at(2))).toBe(true);
+        expect(await store.complete(claimed[1], at(2))).toBe(true);
+        await store.relinquish("worker-1", at(3));
+        await append(
+          [{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }],
+          at(4)
+        );
+        expect(await claim(["handler-a"], at(4), 100, "worker-2")).toHaveLength(1);
+      });
+      it("does not give away a delivery that is still running", async (context) => {
+        if (store.relinquish === void 0) {
+          return context.skip();
+        }
+        await append([{ event: createEnvelope(), deliveries: [createDelivery("handler-a")] }]);
+        const [running] = await claim(["handler-a"], at(0), 100, "worker-1");
+        await store.relinquish("worker-1", at(1));
+        expect(await claim(["handler-a"], at(2), 100, "worker-2")).toEqual([]);
+        expect(await store.complete(running, at(3))).toBe(true);
+      });
+    });
     describe.skipIf(options.createPeerStore === void 0)("across store instances", () => {
       let peer;
       beforeEach(async () => {
@@ -810,6 +1229,32 @@ function eventStoreContract(name, options) {
         expect(await store.complete(stale, at(LEASE_MS + 1))).toBe(false);
         expect(await peer.complete(current, at(LEASE_MS + 1))).toBe(true);
       });
+      it("completes batches in either process and fences the one that lost its lease", async () => {
+        await append(
+          Array.from({ length: 6 }, () => ({
+            event: createEnvelope(),
+            deliveries: [createDelivery("handler-a")]
+          }))
+        );
+        const request = (owner, now) => ({
+          handlers: ["handler-a"],
+          limit: 100,
+          leaseOwner: owner,
+          leaseExpiresAt: new Date(now.getTime() + LEASE_MS),
+          now
+        });
+        const stale = await store.claim(request("primary", at(0)));
+        const current = await peer.claim(request("peer", at(LEASE_MS)));
+        expect(current).toHaveLength(6);
+        expect(await completeMany(store, stale, at(LEASE_MS + 1))).toEqual(stale.map(() => false));
+        expect(await completeMany(peer, current, at(LEASE_MS + 1))).toEqual(
+          current.map(() => true)
+        );
+        expect(await store.statistics(at(LEASE_MS + 2))).toMatchObject({
+          pending: 0,
+          processing: 0
+        });
+      });
     });
   });
 }