@indigoai-us/hq-cloud 6.18.35 → 6.18.37

package/dist/sync/push-receiver.test.js

@@ -90,6 +90,7 @@ class FakeSqs {
     visibilityFailure = null;
     visibilityBatchFailures = new Map();
     receiveCalls = 0;
+    receivedReceiptHandles = [];
     receiveWaitTimes = [];
     enqueue(messages) {
         this.batches.push({ messages, drainable: false });
@@ -120,8 +121,10 @@ class FakeSqs {
         if (this.released.length > 0 && args.waitTimeSeconds > 1) {
             const messages = this.released.splice(0);
             for (const message of messages) {
-                if (message.ReceiptHandle)
+                if (message.ReceiptHandle) {
                     this.inFlight.set(message.ReceiptHandle, message);
+                    this.receivedReceiptHandles.push(message.ReceiptHandle);
+                }
             }
             return { messages };
         }
@@ -135,8 +138,10 @@ class FakeSqs {
             if (next.messages.length === 0)
                 this.batches.shift();
             for (const message of messages) {
-                if (message.ReceiptHandle)
+                if (message.ReceiptHandle) {
                     this.inFlight.set(message.ReceiptHandle, message);
+                    this.receivedReceiptHandles.push(message.ReceiptHandle);
+                }
             }
             return { messages };
         }
@@ -1922,6 +1927,348 @@ describe("US-009: createPushReceiver factory", () => {
     });
 });
 describe("SqsPushReceiver failed-batch retry visibility", () => {
+    it("releases an overlong drain so the receiver resumes polling after two stuck drains", async () => {
+        vi.useFakeTimers();
+        const sqs = new FakeSqs();
+        const events = [
+            makeEvent({ relativePath: "companies/indigo/stuck-one.md", sequenceNumber: 1 }),
+            makeEvent({ relativePath: "companies/indigo/stuck-two.md", sequenceNumber: 2 }),
+            makeEvent({ relativePath: "companies/indigo/stuck-three.md", sequenceNumber: 3 }),
+        ];
+        sqs.enqueueDrainable([msg(events[0])]);
+        let syncCalls = 0;
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            maxInFlightDrains: 2,
+            visibilityTimeoutSeconds: 600,
+            visibilityHeartbeatIntervalMs: 300_000,
+            disposeDrainMs: 1,
+            syncBatchFn: async () => {
+                if (syncCalls === 0)
+                    sqs.enqueueDrainable([msg(events[1])]);
+                syncCalls += 1;
+                return new Promise(() => undefined);
+            },
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        try {
+            await receiver.start();
+            await vi.waitFor(() => expect(syncCalls).toBe(2));
+            await vi.advanceTimersByTimeAsync(600_000);
+            await vi.waitFor(() => expect(sqs.receiveCalls).toBeGreaterThan(2));
+            expect(sqs.visibilityChanges).toContainEqual({
+                receiptHandle: "rh-1",
+                visibilityTimeoutSeconds: 20,
+            });
+            expect(sqs.visibilityChanges).toContainEqual({
+                receiptHandle: "rh-2",
+                visibilityTimeoutSeconds: 20,
+            });
+        }
+        finally {
+            const disposing = receiver.dispose();
+            await vi.advanceTimersByTimeAsync(10);
+            await disposing;
+            vi.useRealTimers();
+        }
+    });
+    it("settles a completed route group's receipt while another group is still running", async () => {
+        const sqs = new FakeSqs();
+        const fast = makeEvent({ relativePath: "personal/notes/fast.md", sequenceNumber: 1 });
+        const slow = makeEvent({ relativePath: "companies/indigo/slow.md", sequenceNumber: 2 });
+        sqs.enqueueDrainable([msg(fast, "rh-fast"), msg(slow, "rh-slow")]);
+        let started;
+        const passStarted = new Promise((resolve) => { started = resolve; });
+        let releaseSlow;
+        const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            maxInFlightDrains: 1,
+            syncBatchFn: async (context) => {
+                started();
+                await context.settleEvents?.([fast]);
+                await slowGate;
+                await context.settleEvents?.([slow]);
+            },
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        try {
+            await receiver.start();
+            await passStarted;
+            await vi.waitFor(() => expect(sqs.deleted).toContain("rh-fast"));
+            expect(sqs.deleted).not.toContain("rh-slow");
+            releaseSlow();
+            await vi.waitFor(() => expect(sqs.deleted).toContain("rh-slow"));
+        }
+        finally {
+            releaseSlow();
+            await receiver.dispose();
+        }
+    });
+    it("resumes lease heartbeats and accepts a later route settle after one settle throws", async () => {
+        const sqs = new FakeSqs();
+        const failed = makeEvent({ relativePath: "companies/indigo/failed-settle.md", sequenceNumber: 21 });
+        const next = makeEvent({ relativePath: "companies/indigo/next-settle.md", sequenceNumber: 22 });
+        const held = makeEvent({ relativePath: "companies/indigo/held-settle.md", sequenceNumber: 23 });
+        sqs.enqueueDrainable([
+            msg(failed, "rh-failed-settle"),
+            msg(next, "rh-next-settle"),
+            msg(held, "rh-held-settle"),
+        ]);
+        let releasePass;
+        const passGate = new Promise((resolve) => { releasePass = resolve; });
+        let laterSettleFinished = false;
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            visibilityTimeoutSeconds: 20,
+            visibilityHeartbeatIntervalMs: 5,
+            syncBatchFn: async (context) => {
+                const internals = receiver;
+                const originalSafeDelete = internals.safeDelete.bind(receiver);
+                internals.safeDelete = async (message) => {
+                    if (message.ReceiptHandle === "rh-failed-settle") {
+                        throw new Error("simulated acknowledgement failure");
+                    }
+                    return originalSafeDelete(message);
+                };
+                await expect(context.settleEvents?.([failed])).rejects.toThrow("simulated acknowledgement failure");
+                await context.settleEvents?.([next]);
+                laterSettleFinished = true;
+                await passGate;
+            },
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        try {
+            await receiver.start();
+            await vi.waitFor(() => expect(laterSettleFinished).toBe(true));
+            await vi.waitFor(() => expect(sqs.deleted).toContain("rh-next-settle"));
+            await vi.waitFor(() => expect(sqs.visibilityChanges).toContainEqual({
+                receiptHandle: "rh-held-settle",
+                visibilityTimeoutSeconds: 20,
+            }));
+        }
+        finally {
+            releasePass();
+            await receiver.dispose();
+        }
+    });
+    it("keeps the outer heartbeat pause active when a nested delete fails", async () => {
+        const sqs = new FakeSqs();
+        const event = makeEvent({ relativePath: "companies/indigo/nested-settle.md", sequenceNumber: 24 });
+        const message = msg(event, "rh-delete-fails");
+        vi.useFakeTimers();
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            visibilityTimeoutSeconds: 20,
+            visibilityHeartbeatIntervalMs: 5,
+            syncFn: async () => undefined,
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        const internals = receiver;
+        const heartbeat = internals.startReceiptLeaseHeartbeat();
+        heartbeat.add(message);
+        sqs.deleteMessage = async () => { throw new Error("simulated delete failure"); };
+        try {
+            await heartbeat.pause(); // settleEvents owns the outer pause.
+            try {
+                await sqs.deleteMessage({ queueUrl: QUEUE_URL, receiptHandle: "rh-delete-fails" });
+            }
+            catch {
+                await heartbeat.pause(); // acknowledge pauses while releasing a failed delete.
+            }
+            finally {
+                heartbeat.resume(); // acknowledge's finally must not resume the outer pause.
+            }
+            await vi.advanceTimersByTimeAsync(10);
+            expect(sqs.visibilityBatchCalls).toHaveLength(0);
+            heartbeat.resume();
+            await vi.advanceTimersByTimeAsync(5);
+            expect(sqs.visibilityBatchCalls).toHaveLength(1);
+            expect(sqs.visibilityBatchCalls[0]?.entries).toContainEqual({
+                id: "receipt-0",
+                receiptHandle: "rh-delete-fails",
+            });
+        }
+        finally {
+            await heartbeat.stop();
+            vi.useRealTimers();
+            await receiver.dispose();
+        }
+    });
+    it("waits for a timed-out pull to stop before applying its redelivery once", async () => {
+        const sqs = new FakeSqs();
+        const event = makeEvent({ relativePath: "companies/indigo/timeout-redelivery.md" });
+        sqs.enqueueDrainable([msg(event, "rh-timeout-original")]);
+        let oldPullStarted;
+        const oldPullStartedPromise = new Promise((resolve) => { oldPullStarted = resolve; });
+        let syncCalls = 0;
+        let oldPullStopped = false;
+        let applied = 0;
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            maxInFlightDrains: 2,
+            drainTimeoutMs: 100,
+            disposeDrainMs: 1,
+            syncBatchFn: async (context) => {
+                syncCalls += 1;
+                if (syncCalls === 1) {
+                    oldPullStarted();
+                    await new Promise((resolve) => {
+                        if (context.signal.aborted)
+                            resolve();
+                        else
+                            context.signal.addEventListener("abort", () => resolve(), { once: true });
+                    });
+                    oldPullStopped = context.signal.aborted;
+                    return;
+                }
+                expect(oldPullStopped).toBe(true);
+                applied += 1;
+            },
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        try {
+            await receiver.start();
+            await oldPullStartedPromise;
+            await new Promise((resolve) => setTimeout(resolve, 120));
+            await vi.waitFor(() => expect(sqs.visibilityChanges).toContainEqual({
+                receiptHandle: "rh-timeout-original",
+                visibilityTimeoutSeconds: 20,
+            }));
+            sqs.enqueueDrainable([msg(event, "rh-timeout-redelivered")]);
+            await vi.waitFor(() => expect(sqs.receivedReceiptHandles).toContain("rh-timeout-redelivered"));
+            await vi.waitFor(() => expect(applied).toBe(1));
+            expect(oldPullStopped).toBe(true);
+            expect(sqs.deleted).toContain("rh-timeout-redelivered");
+            expect(applied).toBe(1);
+        }
+        finally {
+            await receiver.dispose();
+        }
+    });
+    it("releases a timed-out multi-path batch once without reporting success telemetry", async () => {
+        const sqs = new FakeSqs();
+        const firstEvent = makeEvent({ relativePath: "companies/indigo/timeout-first.md", sequenceNumber: 31 });
+        const secondEvent = makeEvent({ relativePath: "companies/indigo/timeout-second.md", sequenceNumber: 32 });
+        sqs.enqueueDrainable([
+            msg(firstEvent, "rh-timeout-first"),
+            msg(secondEvent, "rh-timeout-second"),
+        ]);
+        const metric = vi.fn(async () => undefined);
+        const infos = [];
+        let syncCalls = 0;
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            maxInFlightDrains: 2,
+            drainTimeoutMs: 30,
+            syncBatchFn: async ({ signal }) => {
+                syncCalls += 1;
+                await new Promise((resolve) => {
+                    if (signal.aborted)
+                        resolve();
+                    else
+                        signal.addEventListener("abort", () => resolve(), { once: true });
+                });
+            },
+            sleep: fastSleep,
+            publishMetric: metric,
+            logger: {
+                debug: () => undefined,
+                warn: () => undefined,
+                error: () => undefined,
+                info: (record) => infos.push(record),
+            },
+        });
+        try {
+            await receiver.start();
+            await vi.waitFor(() => {
+                expect(sqs.visibilityChanges).toEqual(expect.arrayContaining([
+                    { receiptHandle: "rh-timeout-first", visibilityTimeoutSeconds: 20 },
+                    { receiptHandle: "rh-timeout-second", visibilityTimeoutSeconds: 20 },
+                ]));
+            });
+            expect(syncCalls).toBe(1);
+            expect(metric).not.toHaveBeenCalled();
+            expect(infos.some((record) => record.event === "fanout.receive")).toBe(false);
+        }
+        finally {
+            await receiver.dispose();
+        }
+    });
+    it("applies a released receipt once when SQS redelivers it", async () => {
+        vi.useFakeTimers();
+        const sqs = new FakeSqs();
+        const event = makeEvent({ relativePath: "companies/indigo/redelivery-once.md" });
+        sqs.enqueueDrainable([msg(event)]);
+        let syncCalls = 0;
+        let applied = 0;
+        const receiver = new SqsPushReceiver({
+            tenantId: TENANT,
+            queueUrl: QUEUE_URL,
+            sqs,
+            maxInFlightDrains: 2,
+            drainTimeoutMs: 100,
+            visibilityTimeoutSeconds: 600,
+            visibilityHeartbeatIntervalMs: 300_000,
+            disposeDrainMs: 1,
+            syncBatchFn: async ({ signal }) => {
+                syncCalls += 1;
+                if (syncCalls === 1) {
+                    await new Promise((resolve) => {
+                        signal.addEventListener("abort", () => resolve(), { once: true });
+                    });
+                    return;
+                }
+                applied += 1;
+            },
+            sleep: fastSleep,
+            publishMetric: async () => undefined,
+            logger: makeDrainLogger([]),
+        });
+        try {
+            await receiver.start();
+            await vi.waitFor(() => expect(syncCalls).toBe(1));
+            await vi.advanceTimersByTimeAsync(100);
+            expect(sqs.visibilityChanges).toContainEqual({
+                receiptHandle: "rh-1",
+                visibilityTimeoutSeconds: 20,
+            });
+            // A new receipt handle for the same SQS message models the retry delivery.
+            sqs.enqueueDrainable([msg(event, "rh-redelivered")]);
+            await vi.waitFor(() => expect(syncCalls).toBe(2));
+            expect(applied).toBe(1);
+            expect(sqs.deleted).toEqual(["rh-redelivered"]);
+            expect(syncCalls).toBe(2);
+        }
+        finally {
+            const disposing = receiver.dispose();
+            await vi.advanceTimersByTimeAsync(10);
+            await disposing;
+            vi.useRealTimers();
+        }
+    });
     const first = makeEvent({
         relativePath: "companies/indigo/retry-first.md",
         sequenceNumber: 101,