@indigoai-us/hq-cloud 6.18.8 → 6.18.10-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-args.test.d.ts +2 -0
- package/dist/bin/sync-runner-args.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-args.test.js +82 -0
- package/dist/bin/sync-runner-args.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +1 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +3 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +31 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +103 -25
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +71 -0
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -1
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -3
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +15 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +67 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +52 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +314 -4
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +2 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +76 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +13 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +81 -4
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +228 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +9 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +21 -2
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +16 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/sync/content-hash.d.ts +4 -0
- package/dist/sync/content-hash.d.ts.map +1 -0
- package/dist/sync/content-hash.js +11 -0
- package/dist/sync/content-hash.js.map +1 -0
- package/dist/sync/content-hash.test.d.ts +2 -0
- package/dist/sync/content-hash.test.d.ts.map +1 -0
- package/dist/sync/content-hash.test.js +16 -0
- package/dist/sync/content-hash.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +58 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +46 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/logger.test.js +3 -0
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +3 -0
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +52 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +413 -50
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +629 -40
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/unchanged-push-events.d.ts +16 -0
- package/dist/sync/unchanged-push-events.d.ts.map +1 -0
- package/dist/sync/unchanged-push-events.js +51 -0
- package/dist/sync/unchanged-push-events.js.map +1 -0
- package/dist/sync/unchanged-push-events.test.d.ts +2 -0
- package/dist/sync/unchanged-push-events.test.d.ts.map +1 -0
- package/dist/sync/unchanged-push-events.test.js +76 -0
- package/dist/sync/unchanged-push-events.test.js.map +1 -0
- package/package.json +1 -1
|
@@ -52,6 +52,27 @@ function msg(event, receiptHandle = `rh-${event.sequenceNumber}`) {
|
|
|
52
52
|
MessageId: `m-${event.relativePath}-${event.sequenceNumber}`,
|
|
53
53
|
};
|
|
54
54
|
}
|
|
55
|
+
async function changeVisibilityBatchIndividually(changeVisibility, args) {
|
|
56
|
+
const failedEntries = [];
|
|
57
|
+
for (const entry of args.entries) {
|
|
58
|
+
try {
|
|
59
|
+
await changeVisibility({
|
|
60
|
+
queueUrl: args.queueUrl,
|
|
61
|
+
receiptHandle: entry.receiptHandle,
|
|
62
|
+
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
const failure = error;
|
|
67
|
+
failedEntries.push({
|
|
68
|
+
id: entry.id,
|
|
69
|
+
...(failure.code === undefined ? {} : { code: failure.code }),
|
|
70
|
+
message: failure.message ?? String(error),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return { failedEntries };
|
|
75
|
+
}
|
|
55
76
|
/**
|
|
56
77
|
* A scriptable fake SQS client. `enqueue` adds batches that successive
|
|
57
78
|
* `receiveMessage` calls drain (one batch per call); once drained, it blocks
|
|
@@ -65,7 +86,9 @@ class FakeSqs {
|
|
|
65
86
|
failCount = 0;
|
|
66
87
|
deleted = [];
|
|
67
88
|
visibilityChanges = [];
|
|
89
|
+
visibilityBatchCalls = [];
|
|
68
90
|
visibilityFailure = null;
|
|
91
|
+
visibilityBatchFailures = new Map();
|
|
69
92
|
receiveCalls = 0;
|
|
70
93
|
receiveWaitTimes = [];
|
|
71
94
|
enqueue(messages) {
|
|
@@ -81,6 +104,12 @@ class FakeSqs {
|
|
|
81
104
|
failNextVisibilityChange(error = new Error("simulated visibility change failure")) {
|
|
82
105
|
this.visibilityFailure = error;
|
|
83
106
|
}
|
|
107
|
+
failVisibilityBatchEntry(receiptHandle, failure = {
|
|
108
|
+
code: "InvalidParameterValue",
|
|
109
|
+
message: "simulated per-entry visibility failure",
|
|
110
|
+
}) {
|
|
111
|
+
this.visibilityBatchFailures.set(receiptHandle, failure);
|
|
112
|
+
}
|
|
84
113
|
async receiveMessage(args) {
|
|
85
114
|
this.receiveCalls += 1;
|
|
86
115
|
this.receiveWaitTimes.push(args.waitTimeSeconds);
|
|
@@ -102,12 +131,14 @@ class FakeSqs {
|
|
|
102
131
|
// next receive cycle. Dedicated drainable and false-negative fakes cover
|
|
103
132
|
// pages that must be collected by the current drain.
|
|
104
133
|
if (next && (next.drainable || args.waitTimeSeconds > 1)) {
|
|
105
|
-
|
|
106
|
-
|
|
134
|
+
const messages = next.messages.splice(0, args.maxMessages);
|
|
135
|
+
if (next.messages.length === 0)
|
|
136
|
+
this.batches.shift();
|
|
137
|
+
for (const message of messages) {
|
|
107
138
|
if (message.ReceiptHandle)
|
|
108
139
|
this.inFlight.set(message.ReceiptHandle, message);
|
|
109
140
|
}
|
|
110
|
-
return { messages
|
|
141
|
+
return { messages };
|
|
111
142
|
}
|
|
112
143
|
if (args.waitTimeSeconds === 0)
|
|
113
144
|
return { messages: [] };
|
|
@@ -145,6 +176,27 @@ class FakeSqs {
|
|
|
145
176
|
this.inFlight.delete(args.receiptHandle);
|
|
146
177
|
this.released.push(message);
|
|
147
178
|
}
|
|
179
|
+
async changeMessageVisibilityBatch(args) {
|
|
180
|
+
this.visibilityBatchCalls.push({
|
|
181
|
+
entries: args.entries.map(({ id, receiptHandle }) => ({ id, receiptHandle })),
|
|
182
|
+
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
183
|
+
});
|
|
184
|
+
const failedEntries = [];
|
|
185
|
+
for (const entry of args.entries) {
|
|
186
|
+
const failure = this.visibilityBatchFailures.get(entry.receiptHandle);
|
|
187
|
+
if (failure) {
|
|
188
|
+
this.visibilityBatchFailures.delete(entry.receiptHandle);
|
|
189
|
+
failedEntries.push({ id: entry.id, ...failure });
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
await this.changeMessageVisibility({
|
|
193
|
+
queueUrl: args.queueUrl,
|
|
194
|
+
receiptHandle: entry.receiptHandle,
|
|
195
|
+
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
return { failedEntries };
|
|
199
|
+
}
|
|
148
200
|
}
|
|
149
201
|
/** FIFO fake used to prove that a slow pass cannot stall backlog intake. */
|
|
150
202
|
class BacklogSqs {
|
|
@@ -198,6 +250,9 @@ class BacklogSqs {
|
|
|
198
250
|
this.released.push(args.receiptHandle);
|
|
199
251
|
this.visible.unshift(message);
|
|
200
252
|
}
|
|
253
|
+
async changeMessageVisibilityBatch(args) {
|
|
254
|
+
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
255
|
+
}
|
|
201
256
|
}
|
|
202
257
|
/** Models SQS short-poll sampling: receive #2 is empty while work remains. */
|
|
203
258
|
class FalseNegativeSqs {
|
|
@@ -236,6 +291,9 @@ class FalseNegativeSqs {
|
|
|
236
291
|
async changeMessageVisibility(_args) {
|
|
237
292
|
// This fixture never returns more paths than its configured cap.
|
|
238
293
|
}
|
|
294
|
+
async changeMessageVisibilityBatch(_args) {
|
|
295
|
+
return { failedEntries: [] };
|
|
296
|
+
}
|
|
239
297
|
}
|
|
240
298
|
/**
|
|
241
299
|
* Scaled-time SQS fixture: an omitted receive visibility lease expires before
|
|
@@ -250,7 +308,10 @@ class VisibilityExpirySqs {
|
|
|
250
308
|
nextReceipt = 0;
|
|
251
309
|
deliveries = [];
|
|
252
310
|
deleted = [];
|
|
311
|
+
deleteAttempts = [];
|
|
253
312
|
requestedVisibilityTimeouts = [];
|
|
313
|
+
visibilityChanges = [];
|
|
314
|
+
deleteDelayMs = 0;
|
|
254
315
|
constructor(event) {
|
|
255
316
|
this.event = event;
|
|
256
317
|
}
|
|
@@ -275,57 +336,84 @@ class VisibilityExpirySqs {
|
|
|
275
336
|
return { messages: [msg(this.event, receiptHandle)] };
|
|
276
337
|
}
|
|
277
338
|
async deleteMessage(args) {
|
|
339
|
+
if (this.deleteDelayMs > 0) {
|
|
340
|
+
await new Promise((resolve) => setTimeout(resolve, this.deleteDelayMs));
|
|
341
|
+
}
|
|
278
342
|
if (Date.now() >= this.leaseExpiresAt) {
|
|
279
343
|
this.visibleAt = 0;
|
|
344
|
+
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: true });
|
|
280
345
|
throw new Error("stale receipt handle after visibility expiry");
|
|
281
346
|
}
|
|
282
347
|
if (args.receiptHandle !== this.currentReceipt) {
|
|
348
|
+
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: true });
|
|
283
349
|
throw new Error("stale receipt handle");
|
|
284
350
|
}
|
|
285
351
|
this.deleted.push(args.receiptHandle);
|
|
352
|
+
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: false });
|
|
286
353
|
this.visibleAt = Number.POSITIVE_INFINITY;
|
|
287
354
|
}
|
|
288
355
|
async changeMessageVisibility(args) {
|
|
289
356
|
if (args.receiptHandle !== this.currentReceipt)
|
|
290
357
|
throw new Error("stale receipt handle");
|
|
358
|
+
this.visibilityChanges.push({
|
|
359
|
+
receiptHandle: args.receiptHandle,
|
|
360
|
+
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
361
|
+
});
|
|
291
362
|
this.visibleAt = Date.now() + args.visibilityTimeoutSeconds * 5;
|
|
292
363
|
this.leaseExpiresAt = this.visibleAt;
|
|
293
364
|
}
|
|
365
|
+
async changeMessageVisibilityBatch(args) {
|
|
366
|
+
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
367
|
+
}
|
|
294
368
|
}
|
|
295
|
-
/**
|
|
369
|
+
/** Queue fixture that honors ReceiveMessage.MaxNumberOfMessages. */
|
|
296
370
|
class OverCapSqs {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
371
|
+
queued;
|
|
372
|
+
inFlight = new Map();
|
|
373
|
+
receiveCalls = [];
|
|
374
|
+
deleted = [];
|
|
300
375
|
releases = [];
|
|
301
|
-
constructor(
|
|
302
|
-
this.
|
|
376
|
+
constructor(initialMessages) {
|
|
377
|
+
this.queued = [...initialMessages];
|
|
303
378
|
}
|
|
304
379
|
enqueue(message) {
|
|
305
380
|
this.queued.push(message);
|
|
306
381
|
}
|
|
307
382
|
async receiveMessage(args) {
|
|
383
|
+
this.receiveCalls.push({
|
|
384
|
+
waitTimeSeconds: args.waitTimeSeconds,
|
|
385
|
+
maxMessages: args.maxMessages,
|
|
386
|
+
});
|
|
387
|
+
if (this.queued.length > 0) {
|
|
388
|
+
const messages = this.queued.splice(0, args.maxMessages);
|
|
389
|
+
for (const message of messages) {
|
|
390
|
+
if (message.ReceiptHandle)
|
|
391
|
+
this.inFlight.set(message.ReceiptHandle, message);
|
|
392
|
+
}
|
|
393
|
+
return { messages };
|
|
394
|
+
}
|
|
308
395
|
if (args.waitTimeSeconds <= 1)
|
|
309
396
|
return { messages: [] };
|
|
310
|
-
if (this.initial) {
|
|
311
|
-
this.initial = false;
|
|
312
|
-
return { messages: this.firstPage };
|
|
313
|
-
}
|
|
314
|
-
if (this.queued.length > 0)
|
|
315
|
-
return { messages: this.queued.splice(0, args.maxMessages) };
|
|
316
397
|
return new Promise((resolve) => {
|
|
317
398
|
if (args.signal.aborted)
|
|
318
399
|
return resolve({ messages: [] });
|
|
319
400
|
args.signal.addEventListener("abort", () => resolve({ messages: [] }), { once: true });
|
|
320
401
|
});
|
|
321
402
|
}
|
|
322
|
-
async deleteMessage() {
|
|
403
|
+
async deleteMessage(args) {
|
|
404
|
+
this.inFlight.delete(args.receiptHandle);
|
|
405
|
+
this.deleted.push(args.receiptHandle);
|
|
406
|
+
}
|
|
323
407
|
async changeMessageVisibility(args) {
|
|
324
408
|
this.releases.push({ receiptHandle: args.receiptHandle, visibilityTimeoutSeconds: args.visibilityTimeoutSeconds });
|
|
325
|
-
const released = this.
|
|
409
|
+
const released = this.inFlight.get(args.receiptHandle);
|
|
326
410
|
if (!released)
|
|
327
411
|
throw new Error("unknown receipt handle");
|
|
328
|
-
this.
|
|
412
|
+
this.inFlight.delete(args.receiptHandle);
|
|
413
|
+
this.queued.unshift(released);
|
|
414
|
+
}
|
|
415
|
+
async changeMessageVisibilityBatch(args) {
|
|
416
|
+
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
329
417
|
}
|
|
330
418
|
}
|
|
331
419
|
/** Wait until `predicate()` is true or `timeoutMs` elapses. */
|
|
@@ -602,6 +690,7 @@ describe("US-009: SqsPushReceiver — subscribe lifecycle", () => {
|
|
|
602
690
|
},
|
|
603
691
|
deleteMessage: (a) => sqs.deleteMessage(a),
|
|
604
692
|
changeMessageVisibility: (a) => sqs.changeMessageVisibility(a),
|
|
693
|
+
changeMessageVisibilityBatch: (a) => sqs.changeMessageVisibilityBatch(a),
|
|
605
694
|
};
|
|
606
695
|
const receiver = new SqsPushReceiver({
|
|
607
696
|
tenantId: TENANT,
|
|
@@ -631,6 +720,74 @@ describe("US-009: SqsPushReceiver — subscribe lifecycle", () => {
|
|
|
631
720
|
});
|
|
632
721
|
});
|
|
633
722
|
describe("receiver drain diagnostics", () => {
|
|
723
|
+
it("acknowledges paths proven unchanged without dispatching a pull", async () => {
|
|
724
|
+
const event = makeEvent({ relativePath: "companies/indigo/unchanged.md" });
|
|
725
|
+
const sqs = new FakeSqs();
|
|
726
|
+
sqs.enqueueDrainable([msg(event, "rh-unchanged")]);
|
|
727
|
+
const syncBatchFn = vi.fn(async () => undefined);
|
|
728
|
+
const drainRecords = [];
|
|
729
|
+
const receiver = new SqsPushReceiver({
|
|
730
|
+
tenantId: TENANT,
|
|
731
|
+
queueUrl: QUEUE_URL,
|
|
732
|
+
sqs,
|
|
733
|
+
syncBatchFn,
|
|
734
|
+
skipUnchangedFn: async (events) => new Set(events.map((entry) => entry.relativePath)),
|
|
735
|
+
sleep: fastSleep,
|
|
736
|
+
publishMetric: async () => undefined,
|
|
737
|
+
logger: {
|
|
738
|
+
info: (record) => {
|
|
739
|
+
if (record.event === "receiver.drain")
|
|
740
|
+
drainRecords.push(record);
|
|
741
|
+
},
|
|
742
|
+
warn: () => undefined,
|
|
743
|
+
error: () => undefined,
|
|
744
|
+
debug: () => undefined,
|
|
745
|
+
},
|
|
746
|
+
});
|
|
747
|
+
await receiver.start();
|
|
748
|
+
await waitFor(() => sqs.deleted.length === 1 && drainRecords.length === 1);
|
|
749
|
+
await receiver.dispose();
|
|
750
|
+
expect(sqs.deleted).toEqual(["rh-unchanged"]);
|
|
751
|
+
expect(syncBatchFn).not.toHaveBeenCalled();
|
|
752
|
+
expect(drainRecords[0]).toMatchObject({
|
|
753
|
+
received: 1,
|
|
754
|
+
coalesced: 1,
|
|
755
|
+
processed: 0,
|
|
756
|
+
skippedUnchanged: 1,
|
|
757
|
+
deleted: 1,
|
|
758
|
+
remaining: 0,
|
|
759
|
+
});
|
|
760
|
+
});
|
|
761
|
+
it("falls back to a targeted pull when the unchanged proof fails", async () => {
|
|
762
|
+
const event = makeEvent({ relativePath: "companies/indigo/proof-error.md" });
|
|
763
|
+
const sqs = new FakeSqs();
|
|
764
|
+
sqs.enqueueDrainable([msg(event, "rh-proof-error")]);
|
|
765
|
+
const syncBatchFn = vi.fn(async () => undefined);
|
|
766
|
+
const warnings = [];
|
|
767
|
+
const receiver = new SqsPushReceiver({
|
|
768
|
+
tenantId: TENANT,
|
|
769
|
+
queueUrl: QUEUE_URL,
|
|
770
|
+
sqs,
|
|
771
|
+
syncBatchFn,
|
|
772
|
+
skipUnchangedFn: async () => { throw new Error("journal unavailable"); },
|
|
773
|
+
sleep: fastSleep,
|
|
774
|
+
publishMetric: async () => undefined,
|
|
775
|
+
logger: {
|
|
776
|
+
info: () => undefined,
|
|
777
|
+
warn: (record) => warnings.push(record),
|
|
778
|
+
error: () => undefined,
|
|
779
|
+
debug: () => undefined,
|
|
780
|
+
},
|
|
781
|
+
});
|
|
782
|
+
await receiver.start();
|
|
783
|
+
await waitFor(() => sqs.deleted.length === 1 && syncBatchFn.mock.calls.length === 1);
|
|
784
|
+
await receiver.dispose();
|
|
785
|
+
expect(syncBatchFn.mock.calls[0]?.[0].events).toEqual([event]);
|
|
786
|
+
expect(warnings).toContainEqual(expect.objectContaining({
|
|
787
|
+
event: "receiver.unchanged.check.failed",
|
|
788
|
+
err: { message: "journal unavailable" },
|
|
789
|
+
}));
|
|
790
|
+
});
|
|
634
791
|
it("reports redelivery count, oldest sent age, and filtered self echoes", async () => {
|
|
635
792
|
const nowMs = 10_000;
|
|
636
793
|
const sqs = new FakeSqs();
|
|
@@ -781,7 +938,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
781
938
|
expect(sqs.deleted).toHaveLength(500);
|
|
782
939
|
expect(sqs.receiveWaitTimes.slice(0, 3)).toEqual([20, 0, 1]);
|
|
783
940
|
});
|
|
784
|
-
it("
|
|
941
|
+
it("caps each 1,000-receipt drain at 256 paths and processes the remaining receipts", async () => {
|
|
785
942
|
const sqs = new FakeSqs();
|
|
786
943
|
const receipts = Array.from({ length: 1_000 }, (_, index) => msg(makeEvent({
|
|
787
944
|
relativePath: `companies/indigo/deep/${index % 256}.md`,
|
|
@@ -800,12 +957,12 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
800
957
|
publishMetric: async () => undefined,
|
|
801
958
|
});
|
|
802
959
|
await receiver.start();
|
|
803
|
-
await waitFor(() =>
|
|
960
|
+
await waitFor(() => sqs.deleted.length === 1_000, 5_000);
|
|
804
961
|
await receiver.dispose();
|
|
805
|
-
expect(batches).
|
|
806
|
-
expect(batches[0].events).toHaveLength(256);
|
|
962
|
+
expect(batches.map((batch) => batch.events.length)).toEqual([256, 256, 256, 232]);
|
|
807
963
|
expect(sqs.deleted).toHaveLength(1_000);
|
|
808
|
-
expect(sqs.
|
|
964
|
+
expect(sqs.visibilityChanges).toEqual([]);
|
|
965
|
+
expect(sqs.receiveWaitTimes[0]).toBe(20);
|
|
809
966
|
});
|
|
810
967
|
it("reports a nonempty large drain before it can later report an empty long-poll", async () => {
|
|
811
968
|
const sqs = new FakeSqs();
|
|
@@ -827,7 +984,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
827
984
|
publishMetric: async () => undefined,
|
|
828
985
|
});
|
|
829
986
|
await receiver.start();
|
|
830
|
-
await waitFor(() => receiver.processedCount
|
|
987
|
+
await waitFor(() => receiver.processedCount >= 256 && activity.includes(false));
|
|
831
988
|
await receiver.dispose();
|
|
832
989
|
expect(activity[0]).toBe(true);
|
|
833
990
|
expect(activity).toContain(false);
|
|
@@ -873,6 +1030,137 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
873
1030
|
expect(sqs.requestedVisibilityTimeouts[0]).toBe(600);
|
|
874
1031
|
expect(sqs.deliveries).toHaveLength(1);
|
|
875
1032
|
});
|
|
1033
|
+
it("heartbeats a receipt when a targeted pull outlives its visibility lease", async () => {
|
|
1034
|
+
const sqs = new VisibilityExpirySqs(makeEvent({ sequenceNumber: 42 }));
|
|
1035
|
+
const receiver = new SqsPushReceiver({
|
|
1036
|
+
tenantId: TENANT,
|
|
1037
|
+
queueUrl: QUEUE_URL,
|
|
1038
|
+
sqs,
|
|
1039
|
+
visibilityTimeoutSeconds: 1,
|
|
1040
|
+
visibilityHeartbeatIntervalMs: 1,
|
|
1041
|
+
syncBatchFn: async () => {
|
|
1042
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
1043
|
+
},
|
|
1044
|
+
sleep: fastSleep,
|
|
1045
|
+
publishMetric: async () => undefined,
|
|
1046
|
+
});
|
|
1047
|
+
await receiver.start();
|
|
1048
|
+
try {
|
|
1049
|
+
await waitFor(() => sqs.deleteAttempts.length === 1);
|
|
1050
|
+
}
|
|
1051
|
+
finally {
|
|
1052
|
+
await receiver.dispose();
|
|
1053
|
+
}
|
|
1054
|
+
expect(sqs.requestedVisibilityTimeouts[0]).toBe(1);
|
|
1055
|
+
expect(sqs.visibilityChanges.length).toBeGreaterThan(0);
|
|
1056
|
+
expect(sqs.deliveries).toHaveLength(1);
|
|
1057
|
+
expect(sqs.deleted).toHaveLength(1);
|
|
1058
|
+
});
|
|
1059
|
+
it("keeps the receipt lease alive until a slow successful delete completes", async () => {
|
|
1060
|
+
const sqs = new VisibilityExpirySqs(makeEvent({ sequenceNumber: 43 }));
|
|
1061
|
+
sqs.deleteDelayMs = 30;
|
|
1062
|
+
const receiver = new SqsPushReceiver({
|
|
1063
|
+
tenantId: TENANT,
|
|
1064
|
+
queueUrl: QUEUE_URL,
|
|
1065
|
+
sqs,
|
|
1066
|
+
visibilityTimeoutSeconds: 1,
|
|
1067
|
+
visibilityHeartbeatIntervalMs: 1,
|
|
1068
|
+
syncBatchFn: async () => undefined,
|
|
1069
|
+
sleep: fastSleep,
|
|
1070
|
+
publishMetric: async () => undefined,
|
|
1071
|
+
});
|
|
1072
|
+
await receiver.start();
|
|
1073
|
+
try {
|
|
1074
|
+
await waitFor(() => sqs.deleteAttempts.length === 1);
|
|
1075
|
+
}
|
|
1076
|
+
finally {
|
|
1077
|
+
await receiver.dispose();
|
|
1078
|
+
}
|
|
1079
|
+
expect(sqs.visibilityChanges.some((change) => change.visibilityTimeoutSeconds === 1)).toBe(true);
|
|
1080
|
+
expect(sqs.deleteAttempts).toEqual([{ receiptHandle: "visibility-rh-1", expired: false }]);
|
|
1081
|
+
expect(sqs.deliveries).toHaveLength(1);
|
|
1082
|
+
expect(sqs.deleted).toHaveLength(1);
|
|
1083
|
+
});
|
|
1084
|
+
it("waits for an in-flight receipt heartbeat before releasing a failed delete", async () => {
|
|
1085
|
+
let delivered = false;
|
|
1086
|
+
let resolveHeartbeat;
|
|
1087
|
+
const heartbeatGate = new Promise((resolve) => { resolveHeartbeat = resolve; });
|
|
1088
|
+
let releaseDelete;
|
|
1089
|
+
const deleteGate = new Promise((resolve) => { releaseDelete = resolve; });
|
|
1090
|
+
const calls = [];
|
|
1091
|
+
const sqs = {
|
|
1092
|
+
receiveMessage: async ({ signal, waitTimeSeconds }) => {
|
|
1093
|
+
if (!delivered) {
|
|
1094
|
+
delivered = true;
|
|
1095
|
+
return { messages: [msg(makeEvent({ sequenceNumber: 44 }), "rh-failed-delete")] };
|
|
1096
|
+
}
|
|
1097
|
+
if (signal.aborted || waitTimeSeconds <= 1)
|
|
1098
|
+
return { messages: [] };
|
|
1099
|
+
return new Promise((resolve) => {
|
|
1100
|
+
signal.addEventListener("abort", () => resolve({ messages: [] }), { once: true });
|
|
1101
|
+
});
|
|
1102
|
+
},
|
|
1103
|
+
deleteMessage: async () => {
|
|
1104
|
+
calls.push("delete-start");
|
|
1105
|
+
await deleteGate;
|
|
1106
|
+
calls.push("delete-failed");
|
|
1107
|
+
throw new Error("simulated delete failure");
|
|
1108
|
+
},
|
|
1109
|
+
changeMessageVisibility: async ({ visibilityTimeoutSeconds }) => {
|
|
1110
|
+
if (visibilityTimeoutSeconds === 0) {
|
|
1111
|
+
calls.push("release");
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
calls.push("heartbeat-start");
|
|
1115
|
+
await heartbeatGate;
|
|
1116
|
+
calls.push("heartbeat-end");
|
|
1117
|
+
},
|
|
1118
|
+
changeMessageVisibilityBatch: async ({ entries, visibilityTimeoutSeconds }) => {
|
|
1119
|
+
for (const _entry of entries) {
|
|
1120
|
+
if (visibilityTimeoutSeconds === 0) {
|
|
1121
|
+
calls.push("release");
|
|
1122
|
+
}
|
|
1123
|
+
else {
|
|
1124
|
+
calls.push("heartbeat-start");
|
|
1125
|
+
await heartbeatGate;
|
|
1126
|
+
calls.push("heartbeat-end");
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
return { failedEntries: [] };
|
|
1130
|
+
},
|
|
1131
|
+
};
|
|
1132
|
+
const receiver = new SqsPushReceiver({
|
|
1133
|
+
tenantId: TENANT,
|
|
1134
|
+
queueUrl: QUEUE_URL,
|
|
1135
|
+
sqs,
|
|
1136
|
+
visibilityHeartbeatIntervalMs: 5,
|
|
1137
|
+
syncBatchFn: async () => undefined,
|
|
1138
|
+
sleep: fastSleep,
|
|
1139
|
+
publishMetric: async () => undefined,
|
|
1140
|
+
});
|
|
1141
|
+
await receiver.start();
|
|
1142
|
+
try {
|
|
1143
|
+
await waitFor(() => calls.includes("delete-start"));
|
|
1144
|
+
await waitFor(() => calls.includes("heartbeat-start"));
|
|
1145
|
+
releaseDelete();
|
|
1146
|
+
await waitFor(() => calls.includes("delete-failed"));
|
|
1147
|
+
expect(calls).toEqual(["delete-start", "heartbeat-start", "delete-failed"]);
|
|
1148
|
+
resolveHeartbeat();
|
|
1149
|
+
await waitFor(() => calls.includes("release"));
|
|
1150
|
+
expect(calls).toEqual([
|
|
1151
|
+
"delete-start",
|
|
1152
|
+
"heartbeat-start",
|
|
1153
|
+
"delete-failed",
|
|
1154
|
+
"heartbeat-end",
|
|
1155
|
+
"release",
|
|
1156
|
+
]);
|
|
1157
|
+
}
|
|
1158
|
+
finally {
|
|
1159
|
+
releaseDelete();
|
|
1160
|
+
resolveHeartbeat();
|
|
1161
|
+
await receiver.dispose();
|
|
1162
|
+
}
|
|
1163
|
+
});
|
|
876
1164
|
it("REGRESSION: drains a 5,000-message FIFO backlog through bounded slow passes without lease-expiry redelivery", async () => {
|
|
877
1165
|
const total = 5_000;
|
|
878
1166
|
const sqs = new BacklogSqs(Array.from({ length: total }, (_, index) => msg(makeEvent({
|
|
@@ -925,7 +1213,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
925
1213
|
expect.objectContaining({ remaining: 0 }),
|
|
926
1214
|
]));
|
|
927
1215
|
}, 15_000);
|
|
928
|
-
it("
|
|
1216
|
+
it("stops a capped distinct-path drain before receiving and releasing more messages", async () => {
|
|
929
1217
|
const first = [
|
|
930
1218
|
msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a"),
|
|
931
1219
|
msg(makeEvent({ relativePath: "companies/indigo/b.md", sequenceNumber: 2 }), "rh-b"),
|
|
@@ -951,8 +1239,13 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
951
1239
|
await receiver.start();
|
|
952
1240
|
await waitFor(() => batches.some((batch) => batch.includes("companies/indigo/fresh.md")));
|
|
953
1241
|
await receiver.dispose();
|
|
954
|
-
expect(sqs.
|
|
955
|
-
|
|
1242
|
+
expect(sqs.receiveCalls[0]).toEqual({ waitTimeSeconds: 20, maxMessages: 2 });
|
|
1243
|
+
expect(sqs.receiveCalls.filter((call) => call.waitTimeSeconds === 0)).toEqual([]);
|
|
1244
|
+
expect(sqs.releases).toEqual([]);
|
|
1245
|
+
expect(sqs.deleted).toHaveLength(4);
|
|
1246
|
+
expect(batches[0]).toEqual([
|
|
1247
|
+
"companies/indigo/a.md",
|
|
1248
|
+
"companies/indigo/b.md",
|
|
956
1249
|
]);
|
|
957
1250
|
expect(batches[1]).toContain("companies/indigo/fresh.md");
|
|
958
1251
|
});
|
|
@@ -986,6 +1279,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
986
1279
|
},
|
|
987
1280
|
deleteMessage: async ({ receiptHandle }) => { deleted.push(receiptHandle); },
|
|
988
1281
|
changeMessageVisibility: async () => { },
|
|
1282
|
+
changeMessageVisibilityBatch: async () => ({ failedEntries: [] }),
|
|
989
1283
|
};
|
|
990
1284
|
const batches = [];
|
|
991
1285
|
const receiver = new SqsPushReceiver({
|
|
@@ -1232,7 +1526,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1232
1526
|
});
|
|
1233
1527
|
});
|
|
1234
1528
|
describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
|
|
1235
|
-
it("leaves
|
|
1529
|
+
it("leaves only the failed path unacknowledged and retries it", async () => {
|
|
1236
1530
|
const sqs = new FakeSqs();
|
|
1237
1531
|
const events = [
|
|
1238
1532
|
makeEvent({ relativePath: "companies/indigo/retry-a.md", sequenceNumber: 1 }),
|
|
@@ -1240,25 +1534,38 @@ describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
|
|
|
1240
1534
|
makeEvent({ relativePath: "companies/indigo/retry-c.md", sequenceNumber: 1 }),
|
|
1241
1535
|
];
|
|
1242
1536
|
sqs.enqueue(events.map((event, index) => msg(event, `rh-failed-${index}`)));
|
|
1243
|
-
sqs.enqueue(
|
|
1244
|
-
let
|
|
1537
|
+
sqs.enqueue([msg(events[0], "rh-redelivered-a")]);
|
|
1538
|
+
let batchCalls = 0;
|
|
1539
|
+
let retryAPathCalls = 0;
|
|
1245
1540
|
const receiver = new SqsPushReceiver({
|
|
1246
1541
|
tenantId: TENANT,
|
|
1247
1542
|
queueUrl: QUEUE_URL,
|
|
1248
1543
|
sqs,
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
if (
|
|
1252
|
-
|
|
1544
|
+
maxInFlightDrains: 1,
|
|
1545
|
+
syncBatchFn: async ({ events: batch }) => {
|
|
1546
|
+
if (batch.length > 1) {
|
|
1547
|
+
batchCalls += 1;
|
|
1548
|
+
throw new Error("coalesced pull failed for one path");
|
|
1549
|
+
}
|
|
1550
|
+
if (batch[0]?.relativePath === events[0].relativePath) {
|
|
1551
|
+
retryAPathCalls += 1;
|
|
1552
|
+
if (retryAPathCalls === 1)
|
|
1553
|
+
throw new Error("retry-a is temporarily unavailable");
|
|
1554
|
+
}
|
|
1253
1555
|
},
|
|
1254
1556
|
sleep: fastSleep,
|
|
1255
1557
|
publishMetric: async () => undefined,
|
|
1256
1558
|
});
|
|
1257
1559
|
await receiver.start();
|
|
1258
|
-
await waitFor(() =>
|
|
1560
|
+
await waitFor(() => receiver.processedCount === 3 && sqs.deleted.includes("rh-redelivered-a"));
|
|
1259
1561
|
await receiver.dispose();
|
|
1260
|
-
expect(
|
|
1261
|
-
expect(
|
|
1562
|
+
expect(batchCalls).toBe(2);
|
|
1563
|
+
expect(retryAPathCalls).toBe(2);
|
|
1564
|
+
expect(sqs.deleted).not.toContain("rh-failed-0");
|
|
1565
|
+
expect(sqs.deleted).toEqual(expect.arrayContaining(["rh-failed-1", "rh-failed-2", "rh-redelivered-a"]));
|
|
1566
|
+
expect(sqs.visibilityChanges).toEqual([
|
|
1567
|
+
{ receiptHandle: "rh-failed-0", visibilityTimeoutSeconds: 20 },
|
|
1568
|
+
]);
|
|
1262
1569
|
});
|
|
1263
1570
|
it("recovers from transient receiveMessage failures (backoff + resume) and replays the queue", async () => {
|
|
1264
1571
|
const sqs = new FakeSqs();
|
|
@@ -1580,6 +1887,82 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1580
1887
|
expect(sqs.deleted).toContain("rh-next-second");
|
|
1581
1888
|
expect(sqs.deleted).not.toContain("rh-next-first");
|
|
1582
1889
|
});
|
|
1890
|
+
it("acks successful paths and retries only paths that fail inside a coalesced batch", async () => {
|
|
1891
|
+
const blocked = makeEvent({
|
|
1892
|
+
relativePath: "companies/indigo/out-of-scope.md",
|
|
1893
|
+
sequenceNumber: 103,
|
|
1894
|
+
});
|
|
1895
|
+
const allowed = makeEvent({
|
|
1896
|
+
relativePath: "companies/indigo/in-scope.md",
|
|
1897
|
+
sequenceNumber: 104,
|
|
1898
|
+
});
|
|
1899
|
+
const sqs = new FakeSqs();
|
|
1900
|
+
sqs.enqueueDrainable([
|
|
1901
|
+
msg(blocked, "rh-out-of-scope"),
|
|
1902
|
+
msg(allowed, "rh-in-scope"),
|
|
1903
|
+
]);
|
|
1904
|
+
const calls = [];
|
|
1905
|
+
const drains = [];
|
|
1906
|
+
const syncBatchFn = async ({ events }) => {
|
|
1907
|
+
const paths = events.map((event) => event.relativePath);
|
|
1908
|
+
calls.push(paths);
|
|
1909
|
+
if (paths.length > 1 || paths[0] === blocked.relativePath) {
|
|
1910
|
+
throw new Error("targeted batch includes an out-of-scope path");
|
|
1911
|
+
}
|
|
1912
|
+
};
|
|
1913
|
+
const receiver = new SqsPushReceiver({
|
|
1914
|
+
tenantId: TENANT,
|
|
1915
|
+
queueUrl: QUEUE_URL,
|
|
1916
|
+
sqs,
|
|
1917
|
+
syncBatchFn,
|
|
1918
|
+
sleep: fastSleep,
|
|
1919
|
+
publishMetric: async () => undefined,
|
|
1920
|
+
logger: makeDrainLogger(drains),
|
|
1921
|
+
});
|
|
1922
|
+
await receiver.start();
|
|
1923
|
+
await waitFor(() => drains.length === 1);
|
|
1924
|
+
await receiver.dispose();
|
|
1925
|
+
expect(calls).toEqual([
|
|
1926
|
+
[blocked.relativePath, allowed.relativePath],
|
|
1927
|
+
[blocked.relativePath],
|
|
1928
|
+
[allowed.relativePath],
|
|
1929
|
+
]);
|
|
1930
|
+
expect(sqs.deleted).toEqual(["rh-in-scope"]);
|
|
1931
|
+
expect(sqs.visibilityChanges).toEqual([
|
|
1932
|
+
{ receiptHandle: "rh-out-of-scope", visibilityTimeoutSeconds: 20 },
|
|
1933
|
+
]);
|
|
1934
|
+
expect(drains[0]).toMatchObject({ deleted: 1, retried: 1 });
|
|
1935
|
+
});
|
|
1936
|
+
it("holds the lease and resumes locally after a realtime-admission timeout", async () => {
|
|
1937
|
+
const sqs = new FakeSqs();
|
|
1938
|
+
sqs.enqueueDrainable([msg(first, "rh-admission-timeout")]);
|
|
1939
|
+
const drains = [];
|
|
1940
|
+
const warnings = [];
|
|
1941
|
+
let attempts = 0;
|
|
1942
|
+
const admissionTimeout = new Error("realtime admission timeout after 120000ms");
|
|
1943
|
+
admissionTimeout.name = "RealtimeAdmissionTimeout";
|
|
1944
|
+
const receiver = new SqsPushReceiver({
|
|
1945
|
+
tenantId: TENANT,
|
|
1946
|
+
queueUrl: QUEUE_URL,
|
|
1947
|
+
sqs,
|
|
1948
|
+
syncBatchFn: async () => {
|
|
1949
|
+
attempts += 1;
|
|
1950
|
+
if (attempts === 1)
|
|
1951
|
+
throw admissionTimeout;
|
|
1952
|
+
},
|
|
1953
|
+
sleep: fastSleep,
|
|
1954
|
+
publishMetric: async () => undefined,
|
|
1955
|
+
logger: makeDrainLogger(drains, warnings),
|
|
1956
|
+
});
|
|
1957
|
+
await receiver.start();
|
|
1958
|
+
await waitFor(() => sqs.deleted.length === 1);
|
|
1959
|
+
await receiver.dispose();
|
|
1960
|
+
expect(attempts).toBe(2);
|
|
1961
|
+
expect(sqs.visibilityChanges).toEqual([]);
|
|
1962
|
+
expect(sqs.deleted).toEqual(["rh-admission-timeout"]);
|
|
1963
|
+
expect(warnings.some((warning) => warning.event === "receiver.retry.failed")).toBe(false);
|
|
1964
|
+
expect(drains[0]).toMatchObject({ deleted: 1, retried: 0 });
|
|
1965
|
+
});
|
|
1583
1966
|
it("keeps the all-success path free of visibility changes and deletes every receipt", async () => {
|
|
1584
1967
|
const sqs = new FakeSqs();
|
|
1585
1968
|
sqs.enqueueDrainable([
|
|
@@ -1637,6 +2020,197 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1637
2020
|
messageId: `m-${first.relativePath}-${first.sequenceNumber}`,
|
|
1638
2021
|
}));
|
|
1639
2022
|
});
|
|
2023
|
+
it("bounds a 64-path transport failure to one dispatch and batches the retry lease", async () => {
|
|
2024
|
+
const total = 64;
|
|
2025
|
+
const events = Array.from({ length: total }, (_, index) => makeEvent({
|
|
2026
|
+
relativePath: `companies/indigo/transport-${index}.md`,
|
|
2027
|
+
sequenceNumber: index + 1,
|
|
2028
|
+
}));
|
|
2029
|
+
const sqs = new FakeSqs();
|
|
2030
|
+
sqs.enqueueDrainable(events.map((event, index) => msg(event, `rh-transport-${index}`)));
|
|
2031
|
+
const drains = [];
|
|
2032
|
+
let dispatchCalls = 0;
|
|
2033
|
+
const transportError = Object.assign(new Error("ECONNRESET: socket hang up while pulling remote objects"), { code: "ECONNRESET" });
|
|
2034
|
+
const startedAt = Date.now();
|
|
2035
|
+
const receiver = new SqsPushReceiver({
|
|
2036
|
+
tenantId: TENANT,
|
|
2037
|
+
queueUrl: QUEUE_URL,
|
|
2038
|
+
sqs,
|
|
2039
|
+
drainMaxMessages: total,
|
|
2040
|
+
batchMaxPaths: total,
|
|
2041
|
+
syncBatchFn: async () => {
|
|
2042
|
+
dispatchCalls += 1;
|
|
2043
|
+
throw transportError;
|
|
2044
|
+
},
|
|
2045
|
+
sleep: fastSleep,
|
|
2046
|
+
publishMetric: async () => undefined,
|
|
2047
|
+
logger: makeDrainLogger(drains),
|
|
2048
|
+
});
|
|
2049
|
+
await receiver.start();
|
|
2050
|
+
await waitFor(() => drains.length === 1, 5_000);
|
|
2051
|
+
await receiver.dispose();
|
|
2052
|
+
expect(dispatchCalls).toBe(1);
|
|
2053
|
+
expect(Date.now() - startedAt).toBeLessThan(5_000);
|
|
2054
|
+
expect(sqs.deleted).toEqual([]);
|
|
2055
|
+
expect(sqs.visibilityChanges.filter((change) => change.visibilityTimeoutSeconds === 20))
|
|
2056
|
+
.toHaveLength(total);
|
|
2057
|
+
const retryCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 20);
|
|
2058
|
+
expect(retryCalls).toHaveLength(7);
|
|
2059
|
+
expect(retryCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2060
|
+
});
|
|
2061
|
+
it("bisects a path-specific batch failure and acknowledges the other 63 paths", async () => {
|
|
2062
|
+
const total = 64;
|
|
2063
|
+
const blocked = makeEvent({
|
|
2064
|
+
relativePath: `companies/indigo/blocked-${total - 1}.md`,
|
|
2065
|
+
sequenceNumber: total,
|
|
2066
|
+
});
|
|
2067
|
+
const events = Array.from({ length: total - 1 }, (_, index) => makeEvent({
|
|
2068
|
+
relativePath: `companies/indigo/allowed-${index}.md`,
|
|
2069
|
+
sequenceNumber: index + 1,
|
|
2070
|
+
})).concat(blocked);
|
|
2071
|
+
const sqs = new FakeSqs();
|
|
2072
|
+
sqs.enqueueDrainable(events.map((event, index) => msg(event, `rh-path-${index}`)));
|
|
2073
|
+
const drains = [];
|
|
2074
|
+
let dispatchCalls = 0;
|
|
2075
|
+
const receiver = new SqsPushReceiver({
|
|
2076
|
+
tenantId: TENANT,
|
|
2077
|
+
queueUrl: QUEUE_URL,
|
|
2078
|
+
sqs,
|
|
2079
|
+
drainMaxMessages: total,
|
|
2080
|
+
batchMaxPaths: total,
|
|
2081
|
+
syncBatchFn: async ({ events: batch }) => {
|
|
2082
|
+
dispatchCalls += 1;
|
|
2083
|
+
if (batch.some((event) => event.relativePath === blocked.relativePath)) {
|
|
2084
|
+
throw new Error("out-of-scope path rejected by targeted pull");
|
|
2085
|
+
}
|
|
2086
|
+
},
|
|
2087
|
+
sleep: fastSleep,
|
|
2088
|
+
publishMetric: async () => undefined,
|
|
2089
|
+
logger: makeDrainLogger(drains),
|
|
2090
|
+
});
|
|
2091
|
+
await receiver.start();
|
|
2092
|
+
await waitFor(() => drains.length === 1, 5_000);
|
|
2093
|
+
await receiver.dispose();
|
|
2094
|
+
expect(dispatchCalls).toBeLessThanOrEqual(13);
|
|
2095
|
+
expect(sqs.deleted).toHaveLength(total - 1);
|
|
2096
|
+
expect(sqs.deleted).not.toContain(`rh-path-${total - 1}`);
|
|
2097
|
+
expect(sqs.visibilityChanges.filter((change) => change.visibilityTimeoutSeconds === 20))
|
|
2098
|
+
.toEqual([{ receiptHandle: `rh-path-${total - 1}`, visibilityTimeoutSeconds: 20 }]);
|
|
2099
|
+
expect(drains[0]).toMatchObject({ processed: total - 1, deleted: total - 1, retried: 1 });
|
|
2100
|
+
});
|
|
2101
|
+
it("reports failed retry entries separately from successful entries", async () => {
|
|
2102
|
+
const sqs = new FakeSqs();
|
|
2103
|
+
sqs.failVisibilityBatchEntry("rh-retry-entry-failed", {
|
|
2104
|
+
code: "ReceiptHandleIsInvalid",
|
|
2105
|
+
message: "one receipt was rejected",
|
|
2106
|
+
senderFault: true,
|
|
2107
|
+
});
|
|
2108
|
+
sqs.enqueueDrainable([
|
|
2109
|
+
msg(first, "rh-retry-entry-failed"),
|
|
2110
|
+
msg(second, "rh-retry-entry-succeeded"),
|
|
2111
|
+
]);
|
|
2112
|
+
const warnings = [];
|
|
2113
|
+
const drains = [];
|
|
2114
|
+
const transportError = Object.assign(new Error("network connection reset"), {
|
|
2115
|
+
code: "ECONNRESET",
|
|
2116
|
+
});
|
|
2117
|
+
const receiver = new SqsPushReceiver({
|
|
2118
|
+
tenantId: TENANT,
|
|
2119
|
+
queueUrl: QUEUE_URL,
|
|
2120
|
+
sqs,
|
|
2121
|
+
syncBatchFn: async () => { throw transportError; },
|
|
2122
|
+
sleep: fastSleep,
|
|
2123
|
+
publishMetric: async () => undefined,
|
|
2124
|
+
logger: makeDrainLogger(drains, warnings),
|
|
2125
|
+
});
|
|
2126
|
+
await receiver.start();
|
|
2127
|
+
await waitFor(() => drains.length === 1);
|
|
2128
|
+
await receiver.dispose();
|
|
2129
|
+
const failures = warnings.filter((warning) => warning.event === "receiver.retry.failed");
|
|
2130
|
+
expect(failures).toHaveLength(1);
|
|
2131
|
+
expect(failures[0]).toMatchObject({
|
|
2132
|
+
messageId: `m-${first.relativePath}-${first.sequenceNumber}`,
|
|
2133
|
+
code: "ReceiptHandleIsInvalid",
|
|
2134
|
+
failureMessage: "one receipt was rejected",
|
|
2135
|
+
senderFault: true,
|
|
2136
|
+
});
|
|
2137
|
+
expect(sqs.visibilityChanges).toEqual([
|
|
2138
|
+
{ receiptHandle: "rh-retry-entry-succeeded", visibilityTimeoutSeconds: 20 },
|
|
2139
|
+
]);
|
|
2140
|
+
expect(drains[0]).toMatchObject({ retried: 1 });
|
|
2141
|
+
expect(failures[0]).not.toMatchObject({
|
|
2142
|
+
messageId: `m-${second.relativePath}-${second.sequenceNumber}`,
|
|
2143
|
+
});
|
|
2144
|
+
});
|
|
2145
|
+
it("uses about 100 ten-receipt API calls for a 1,000-receipt heartbeat", async () => {
|
|
2146
|
+
const total = 1_000;
|
|
2147
|
+
const sqs = new FakeSqs();
|
|
2148
|
+
const receipts = Array.from({ length: total }, (_, index) => msg(makeEvent({
|
|
2149
|
+
relativePath: `companies/indigo/heartbeat-${index}.md`,
|
|
2150
|
+
sequenceNumber: index + 1,
|
|
2151
|
+
}), `rh-heartbeat-${index}`));
|
|
2152
|
+
for (let offset = 0; offset < receipts.length; offset += 10) {
|
|
2153
|
+
sqs.enqueueDrainable(receipts.slice(offset, offset + 10));
|
|
2154
|
+
}
|
|
2155
|
+
let startPass;
|
|
2156
|
+
const passStarted = new Promise((resolve) => { startPass = resolve; });
|
|
2157
|
+
let releasePass;
|
|
2158
|
+
const passGate = new Promise((resolve) => { releasePass = resolve; });
|
|
2159
|
+
let finishDrain;
|
|
2160
|
+
const drainFinished = new Promise((resolve) => { finishDrain = resolve; });
|
|
2161
|
+
const receiver = new SqsPushReceiver({
|
|
2162
|
+
tenantId: TENANT,
|
|
2163
|
+
queueUrl: QUEUE_URL,
|
|
2164
|
+
sqs,
|
|
2165
|
+
drainMaxMessages: total,
|
|
2166
|
+
batchMaxPaths: total,
|
|
2167
|
+
maxInFlightDrains: 1,
|
|
2168
|
+
visibilityTimeoutSeconds: 600,
|
|
2169
|
+
visibilityHeartbeatIntervalMs: 1_000,
|
|
2170
|
+
syncBatchFn: async () => {
|
|
2171
|
+
startPass();
|
|
2172
|
+
await passGate;
|
|
2173
|
+
throw Object.assign(new Error("ECONNRESET while pulling the batch"), {
|
|
2174
|
+
code: "ECONNRESET",
|
|
2175
|
+
});
|
|
2176
|
+
},
|
|
2177
|
+
sleep: fastSleep,
|
|
2178
|
+
publishMetric: async () => undefined,
|
|
2179
|
+
logger: {
|
|
2180
|
+
debug: () => undefined,
|
|
2181
|
+
warn: () => undefined,
|
|
2182
|
+
error: () => undefined,
|
|
2183
|
+
info: (record) => {
|
|
2184
|
+
if (record.event === "receiver.drain")
|
|
2185
|
+
finishDrain();
|
|
2186
|
+
},
|
|
2187
|
+
},
|
|
2188
|
+
});
|
|
2189
|
+
vi.useFakeTimers();
|
|
2190
|
+
try {
|
|
2191
|
+
await receiver.start();
|
|
2192
|
+
await passStarted;
|
|
2193
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
2194
|
+
const heartbeatCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 600);
|
|
2195
|
+
expect(heartbeatCalls.length).toBeLessThanOrEqual(110);
|
|
2196
|
+
expect(heartbeatCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2197
|
+
expect(heartbeatCalls.reduce((count, call) => count + call.entries.length, 0))
|
|
2198
|
+
.toBe(total);
|
|
2199
|
+
releasePass();
|
|
2200
|
+
await drainFinished;
|
|
2201
|
+
await receiver.dispose();
|
|
2202
|
+
const retryCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 20);
|
|
2203
|
+
expect(retryCalls.length).toBeLessThanOrEqual(110);
|
|
2204
|
+
expect(retryCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2205
|
+
expect(retryCalls.reduce((count, call) => count + call.entries.length, 0))
|
|
2206
|
+
.toBe(total);
|
|
2207
|
+
}
|
|
2208
|
+
finally {
|
|
2209
|
+
releasePass();
|
|
2210
|
+
await receiver.dispose();
|
|
2211
|
+
vi.useRealTimers();
|
|
2212
|
+
}
|
|
2213
|
+
});
|
|
1640
2214
|
it("updates a large failed batch with bounded parallel visibility calls", async () => {
|
|
1641
2215
|
const total = 512;
|
|
1642
2216
|
const baseSqs = new FakeSqs();
|
|
@@ -1647,7 +2221,7 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1647
2221
|
for (let offset = 0; offset < receipts.length; offset += 10) {
|
|
1648
2222
|
baseSqs.enqueueDrainable(receipts.slice(offset, offset + 10));
|
|
1649
2223
|
}
|
|
1650
|
-
const
|
|
2224
|
+
const visibilityBatchCalls = [];
|
|
1651
2225
|
let activeCalls = 0;
|
|
1652
2226
|
let maxActiveCalls = 0;
|
|
1653
2227
|
let releaseVisibility;
|
|
@@ -1662,12 +2236,24 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1662
2236
|
receiveMessage: (args) => baseSqs.receiveMessage(args),
|
|
1663
2237
|
deleteMessage: (args) => baseSqs.deleteMessage(args),
|
|
1664
2238
|
changeMessageVisibility: async ({ receiptHandle }) => {
|
|
1665
|
-
|
|
2239
|
+
visibilityBatchCalls.push({ receiptHandles: [receiptHandle], visibilityTimeoutSeconds: 20 });
|
|
2240
|
+
activeCalls += 1;
|
|
2241
|
+
maxActiveCalls = Math.max(maxActiveCalls, activeCalls);
|
|
2242
|
+
firstCallStarted();
|
|
2243
|
+
await visibilityGate;
|
|
2244
|
+
activeCalls -= 1;
|
|
2245
|
+
},
|
|
2246
|
+
changeMessageVisibilityBatch: async ({ entries, visibilityTimeoutSeconds }) => {
|
|
2247
|
+
visibilityBatchCalls.push({
|
|
2248
|
+
receiptHandles: entries.map((entry) => entry.receiptHandle),
|
|
2249
|
+
visibilityTimeoutSeconds,
|
|
2250
|
+
});
|
|
1666
2251
|
activeCalls += 1;
|
|
1667
2252
|
maxActiveCalls = Math.max(maxActiveCalls, activeCalls);
|
|
1668
2253
|
firstCallStarted();
|
|
1669
2254
|
await visibilityGate;
|
|
1670
2255
|
activeCalls -= 1;
|
|
2256
|
+
return { failedEntries: [] };
|
|
1671
2257
|
},
|
|
1672
2258
|
};
|
|
1673
2259
|
const drains = [];
|
|
@@ -1691,7 +2277,10 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1691
2277
|
await receiver.dispose();
|
|
1692
2278
|
expect(parallelCallsBeforeRelease).toBeGreaterThan(1);
|
|
1693
2279
|
expect(parallelCallsBeforeRelease).toBeLessThan(total);
|
|
1694
|
-
expect(
|
|
2280
|
+
expect(visibilityBatchCalls).toHaveLength(Math.ceil(total / 10));
|
|
2281
|
+
expect(visibilityBatchCalls.every((call) => call.receiptHandles.length <= 10)).toBe(true);
|
|
2282
|
+
expect(visibilityBatchCalls.flatMap((call) => call.receiptHandles)).toHaveLength(total);
|
|
2283
|
+
expect(visibilityBatchCalls.every((call) => call.visibilityTimeoutSeconds === 20)).toBe(true);
|
|
1695
2284
|
expect(baseSqs.deleted).toEqual([]);
|
|
1696
2285
|
});
|
|
1697
2286
|
});
|