@indigoai-us/hq-cloud 6.18.10-alpha.0 → 6.18.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/rescue-runner.js +20 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -1
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +0 -3
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -31
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +12 -58
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +0 -71
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +3 -14
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +0 -15
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +0 -67
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +1 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +3 -23
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +1 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +2 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +87 -4
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-restore-flag.d.ts +14 -0
- package/dist/cli/rescue-restore-flag.d.ts.map +1 -0
- package/dist/cli/rescue-restore-flag.js +62 -0
- package/dist/cli/rescue-restore-flag.js.map +1 -0
- package/dist/cli/rescue-restore-flag.test.d.ts +2 -0
- package/dist/cli/rescue-restore-flag.test.d.ts.map +1 -0
- package/dist/cli/rescue-restore-flag.test.js +82 -0
- package/dist/cli/rescue-restore-flag.test.js.map +1 -0
- package/dist/cli/rescue-rsync-status.test.js +121 -2
- package/dist/cli/rescue-rsync-status.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +4 -2
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +7 -4
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.reindex.test.js +18 -2
- package/dist/cli/rescue.reindex.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +0 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +0 -76
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +2 -13
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +4 -81
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +0 -228
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth-lock-no-hard-links.test.d.ts +11 -0
- package/dist/cognito-auth-lock-no-hard-links.test.d.ts.map +1 -0
- package/dist/cognito-auth-lock-no-hard-links.test.js +76 -0
- package/dist/cognito-auth-lock-no-hard-links.test.js.map +1 -0
- package/dist/cognito-auth.d.ts +35 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +66 -1
- package/dist/cognito-auth.js.map +1 -1
- package/dist/journal.d.ts +0 -9
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +2 -21
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +1 -16
- package/dist/journal.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +1 -58
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +0 -46
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/logger.test.js +0 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -52
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +50 -413
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +40 -629
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/bin/sync-runner-args.test.d.ts +0 -2
- package/dist/bin/sync-runner-args.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-args.test.js +0 -82
- package/dist/bin/sync-runner-args.test.js.map +0 -1
- package/dist/sync/content-hash.d.ts +0 -4
- package/dist/sync/content-hash.d.ts.map +0 -1
- package/dist/sync/content-hash.js +0 -11
- package/dist/sync/content-hash.js.map +0 -1
- package/dist/sync/content-hash.test.d.ts +0 -2
- package/dist/sync/content-hash.test.d.ts.map +0 -1
- package/dist/sync/content-hash.test.js +0 -16
- package/dist/sync/content-hash.test.js.map +0 -1
- package/dist/sync/unchanged-push-events.d.ts +0 -16
- package/dist/sync/unchanged-push-events.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.js +0 -51
- package/dist/sync/unchanged-push-events.js.map +0 -1
- package/dist/sync/unchanged-push-events.test.d.ts +0 -2
- package/dist/sync/unchanged-push-events.test.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.test.js +0 -76
- package/dist/sync/unchanged-push-events.test.js.map +0 -1
|
@@ -52,27 +52,6 @@ function msg(event, receiptHandle = `rh-${event.sequenceNumber}`) {
|
|
|
52
52
|
MessageId: `m-${event.relativePath}-${event.sequenceNumber}`,
|
|
53
53
|
};
|
|
54
54
|
}
|
|
55
|
-
async function changeVisibilityBatchIndividually(changeVisibility, args) {
|
|
56
|
-
const failedEntries = [];
|
|
57
|
-
for (const entry of args.entries) {
|
|
58
|
-
try {
|
|
59
|
-
await changeVisibility({
|
|
60
|
-
queueUrl: args.queueUrl,
|
|
61
|
-
receiptHandle: entry.receiptHandle,
|
|
62
|
-
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
catch (error) {
|
|
66
|
-
const failure = error;
|
|
67
|
-
failedEntries.push({
|
|
68
|
-
id: entry.id,
|
|
69
|
-
...(failure.code === undefined ? {} : { code: failure.code }),
|
|
70
|
-
message: failure.message ?? String(error),
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
return { failedEntries };
|
|
75
|
-
}
|
|
76
55
|
/**
|
|
77
56
|
* A scriptable fake SQS client. `enqueue` adds batches that successive
|
|
78
57
|
* `receiveMessage` calls drain (one batch per call); once drained, it blocks
|
|
@@ -86,9 +65,7 @@ class FakeSqs {
|
|
|
86
65
|
failCount = 0;
|
|
87
66
|
deleted = [];
|
|
88
67
|
visibilityChanges = [];
|
|
89
|
-
visibilityBatchCalls = [];
|
|
90
68
|
visibilityFailure = null;
|
|
91
|
-
visibilityBatchFailures = new Map();
|
|
92
69
|
receiveCalls = 0;
|
|
93
70
|
receiveWaitTimes = [];
|
|
94
71
|
enqueue(messages) {
|
|
@@ -104,12 +81,6 @@ class FakeSqs {
|
|
|
104
81
|
failNextVisibilityChange(error = new Error("simulated visibility change failure")) {
|
|
105
82
|
this.visibilityFailure = error;
|
|
106
83
|
}
|
|
107
|
-
failVisibilityBatchEntry(receiptHandle, failure = {
|
|
108
|
-
code: "InvalidParameterValue",
|
|
109
|
-
message: "simulated per-entry visibility failure",
|
|
110
|
-
}) {
|
|
111
|
-
this.visibilityBatchFailures.set(receiptHandle, failure);
|
|
112
|
-
}
|
|
113
84
|
async receiveMessage(args) {
|
|
114
85
|
this.receiveCalls += 1;
|
|
115
86
|
this.receiveWaitTimes.push(args.waitTimeSeconds);
|
|
@@ -131,14 +102,12 @@ class FakeSqs {
|
|
|
131
102
|
// next receive cycle. Dedicated drainable and false-negative fakes cover
|
|
132
103
|
// pages that must be collected by the current drain.
|
|
133
104
|
if (next && (next.drainable || args.waitTimeSeconds > 1)) {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
this.batches.shift();
|
|
137
|
-
for (const message of messages) {
|
|
105
|
+
this.batches.shift();
|
|
106
|
+
for (const message of next.messages) {
|
|
138
107
|
if (message.ReceiptHandle)
|
|
139
108
|
this.inFlight.set(message.ReceiptHandle, message);
|
|
140
109
|
}
|
|
141
|
-
return { messages };
|
|
110
|
+
return { messages: next.messages };
|
|
142
111
|
}
|
|
143
112
|
if (args.waitTimeSeconds === 0)
|
|
144
113
|
return { messages: [] };
|
|
@@ -176,27 +145,6 @@ class FakeSqs {
|
|
|
176
145
|
this.inFlight.delete(args.receiptHandle);
|
|
177
146
|
this.released.push(message);
|
|
178
147
|
}
|
|
179
|
-
async changeMessageVisibilityBatch(args) {
|
|
180
|
-
this.visibilityBatchCalls.push({
|
|
181
|
-
entries: args.entries.map(({ id, receiptHandle }) => ({ id, receiptHandle })),
|
|
182
|
-
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
183
|
-
});
|
|
184
|
-
const failedEntries = [];
|
|
185
|
-
for (const entry of args.entries) {
|
|
186
|
-
const failure = this.visibilityBatchFailures.get(entry.receiptHandle);
|
|
187
|
-
if (failure) {
|
|
188
|
-
this.visibilityBatchFailures.delete(entry.receiptHandle);
|
|
189
|
-
failedEntries.push({ id: entry.id, ...failure });
|
|
190
|
-
continue;
|
|
191
|
-
}
|
|
192
|
-
await this.changeMessageVisibility({
|
|
193
|
-
queueUrl: args.queueUrl,
|
|
194
|
-
receiptHandle: entry.receiptHandle,
|
|
195
|
-
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
196
|
-
});
|
|
197
|
-
}
|
|
198
|
-
return { failedEntries };
|
|
199
|
-
}
|
|
200
148
|
}
|
|
201
149
|
/** FIFO fake used to prove that a slow pass cannot stall backlog intake. */
|
|
202
150
|
class BacklogSqs {
|
|
@@ -250,9 +198,6 @@ class BacklogSqs {
|
|
|
250
198
|
this.released.push(args.receiptHandle);
|
|
251
199
|
this.visible.unshift(message);
|
|
252
200
|
}
|
|
253
|
-
async changeMessageVisibilityBatch(args) {
|
|
254
|
-
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
255
|
-
}
|
|
256
201
|
}
|
|
257
202
|
/** Models SQS short-poll sampling: receive #2 is empty while work remains. */
|
|
258
203
|
class FalseNegativeSqs {
|
|
@@ -291,9 +236,6 @@ class FalseNegativeSqs {
|
|
|
291
236
|
async changeMessageVisibility(_args) {
|
|
292
237
|
// This fixture never returns more paths than its configured cap.
|
|
293
238
|
}
|
|
294
|
-
async changeMessageVisibilityBatch(_args) {
|
|
295
|
-
return { failedEntries: [] };
|
|
296
|
-
}
|
|
297
239
|
}
|
|
298
240
|
/**
|
|
299
241
|
* Scaled-time SQS fixture: an omitted receive visibility lease expires before
|
|
@@ -308,10 +250,7 @@ class VisibilityExpirySqs {
|
|
|
308
250
|
nextReceipt = 0;
|
|
309
251
|
deliveries = [];
|
|
310
252
|
deleted = [];
|
|
311
|
-
deleteAttempts = [];
|
|
312
253
|
requestedVisibilityTimeouts = [];
|
|
313
|
-
visibilityChanges = [];
|
|
314
|
-
deleteDelayMs = 0;
|
|
315
254
|
constructor(event) {
|
|
316
255
|
this.event = event;
|
|
317
256
|
}
|
|
@@ -336,84 +275,57 @@ class VisibilityExpirySqs {
|
|
|
336
275
|
return { messages: [msg(this.event, receiptHandle)] };
|
|
337
276
|
}
|
|
338
277
|
async deleteMessage(args) {
|
|
339
|
-
if (this.deleteDelayMs > 0) {
|
|
340
|
-
await new Promise((resolve) => setTimeout(resolve, this.deleteDelayMs));
|
|
341
|
-
}
|
|
342
278
|
if (Date.now() >= this.leaseExpiresAt) {
|
|
343
279
|
this.visibleAt = 0;
|
|
344
|
-
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: true });
|
|
345
280
|
throw new Error("stale receipt handle after visibility expiry");
|
|
346
281
|
}
|
|
347
282
|
if (args.receiptHandle !== this.currentReceipt) {
|
|
348
|
-
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: true });
|
|
349
283
|
throw new Error("stale receipt handle");
|
|
350
284
|
}
|
|
351
285
|
this.deleted.push(args.receiptHandle);
|
|
352
|
-
this.deleteAttempts.push({ receiptHandle: args.receiptHandle, expired: false });
|
|
353
286
|
this.visibleAt = Number.POSITIVE_INFINITY;
|
|
354
287
|
}
|
|
355
288
|
async changeMessageVisibility(args) {
|
|
356
289
|
if (args.receiptHandle !== this.currentReceipt)
|
|
357
290
|
throw new Error("stale receipt handle");
|
|
358
|
-
this.visibilityChanges.push({
|
|
359
|
-
receiptHandle: args.receiptHandle,
|
|
360
|
-
visibilityTimeoutSeconds: args.visibilityTimeoutSeconds,
|
|
361
|
-
});
|
|
362
291
|
this.visibleAt = Date.now() + args.visibilityTimeoutSeconds * 5;
|
|
363
292
|
this.leaseExpiresAt = this.visibleAt;
|
|
364
293
|
}
|
|
365
|
-
async changeMessageVisibilityBatch(args) {
|
|
366
|
-
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
367
|
-
}
|
|
368
294
|
}
|
|
369
|
-
/**
|
|
295
|
+
/** Delivers an over-cap path before a fresh event added while the first pass runs. */
|
|
370
296
|
class OverCapSqs {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
deleted = [];
|
|
297
|
+
firstPage;
|
|
298
|
+
initial = true;
|
|
299
|
+
queued = [];
|
|
375
300
|
releases = [];
|
|
376
|
-
constructor(
|
|
377
|
-
this.
|
|
301
|
+
constructor(firstPage) {
|
|
302
|
+
this.firstPage = firstPage;
|
|
378
303
|
}
|
|
379
304
|
enqueue(message) {
|
|
380
305
|
this.queued.push(message);
|
|
381
306
|
}
|
|
382
307
|
async receiveMessage(args) {
|
|
383
|
-
this.receiveCalls.push({
|
|
384
|
-
waitTimeSeconds: args.waitTimeSeconds,
|
|
385
|
-
maxMessages: args.maxMessages,
|
|
386
|
-
});
|
|
387
|
-
if (this.queued.length > 0) {
|
|
388
|
-
const messages = this.queued.splice(0, args.maxMessages);
|
|
389
|
-
for (const message of messages) {
|
|
390
|
-
if (message.ReceiptHandle)
|
|
391
|
-
this.inFlight.set(message.ReceiptHandle, message);
|
|
392
|
-
}
|
|
393
|
-
return { messages };
|
|
394
|
-
}
|
|
395
308
|
if (args.waitTimeSeconds <= 1)
|
|
396
309
|
return { messages: [] };
|
|
310
|
+
if (this.initial) {
|
|
311
|
+
this.initial = false;
|
|
312
|
+
return { messages: this.firstPage };
|
|
313
|
+
}
|
|
314
|
+
if (this.queued.length > 0)
|
|
315
|
+
return { messages: this.queued.splice(0, args.maxMessages) };
|
|
397
316
|
return new Promise((resolve) => {
|
|
398
317
|
if (args.signal.aborted)
|
|
399
318
|
return resolve({ messages: [] });
|
|
400
319
|
args.signal.addEventListener("abort", () => resolve({ messages: [] }), { once: true });
|
|
401
320
|
});
|
|
402
321
|
}
|
|
403
|
-
async deleteMessage(
|
|
404
|
-
this.inFlight.delete(args.receiptHandle);
|
|
405
|
-
this.deleted.push(args.receiptHandle);
|
|
406
|
-
}
|
|
322
|
+
async deleteMessage() { }
|
|
407
323
|
async changeMessageVisibility(args) {
|
|
408
324
|
this.releases.push({ receiptHandle: args.receiptHandle, visibilityTimeoutSeconds: args.visibilityTimeoutSeconds });
|
|
409
|
-
const released = this.
|
|
325
|
+
const released = this.firstPage.find((message) => message.ReceiptHandle === args.receiptHandle);
|
|
410
326
|
if (!released)
|
|
411
327
|
throw new Error("unknown receipt handle");
|
|
412
|
-
this.
|
|
413
|
-
this.queued.unshift(released);
|
|
414
|
-
}
|
|
415
|
-
async changeMessageVisibilityBatch(args) {
|
|
416
|
-
return changeVisibilityBatchIndividually((entry) => this.changeMessageVisibility(entry), args);
|
|
328
|
+
this.queued.push(released);
|
|
417
329
|
}
|
|
418
330
|
}
|
|
419
331
|
/** Wait until `predicate()` is true or `timeoutMs` elapses. */
|
|
@@ -690,7 +602,6 @@ describe("US-009: SqsPushReceiver — subscribe lifecycle", () => {
|
|
|
690
602
|
},
|
|
691
603
|
deleteMessage: (a) => sqs.deleteMessage(a),
|
|
692
604
|
changeMessageVisibility: (a) => sqs.changeMessageVisibility(a),
|
|
693
|
-
changeMessageVisibilityBatch: (a) => sqs.changeMessageVisibilityBatch(a),
|
|
694
605
|
};
|
|
695
606
|
const receiver = new SqsPushReceiver({
|
|
696
607
|
tenantId: TENANT,
|
|
@@ -720,74 +631,6 @@ describe("US-009: SqsPushReceiver — subscribe lifecycle", () => {
|
|
|
720
631
|
});
|
|
721
632
|
});
|
|
722
633
|
describe("receiver drain diagnostics", () => {
|
|
723
|
-
it("acknowledges paths proven unchanged without dispatching a pull", async () => {
|
|
724
|
-
const event = makeEvent({ relativePath: "companies/indigo/unchanged.md" });
|
|
725
|
-
const sqs = new FakeSqs();
|
|
726
|
-
sqs.enqueueDrainable([msg(event, "rh-unchanged")]);
|
|
727
|
-
const syncBatchFn = vi.fn(async () => undefined);
|
|
728
|
-
const drainRecords = [];
|
|
729
|
-
const receiver = new SqsPushReceiver({
|
|
730
|
-
tenantId: TENANT,
|
|
731
|
-
queueUrl: QUEUE_URL,
|
|
732
|
-
sqs,
|
|
733
|
-
syncBatchFn,
|
|
734
|
-
skipUnchangedFn: async (events) => new Set(events.map((entry) => entry.relativePath)),
|
|
735
|
-
sleep: fastSleep,
|
|
736
|
-
publishMetric: async () => undefined,
|
|
737
|
-
logger: {
|
|
738
|
-
info: (record) => {
|
|
739
|
-
if (record.event === "receiver.drain")
|
|
740
|
-
drainRecords.push(record);
|
|
741
|
-
},
|
|
742
|
-
warn: () => undefined,
|
|
743
|
-
error: () => undefined,
|
|
744
|
-
debug: () => undefined,
|
|
745
|
-
},
|
|
746
|
-
});
|
|
747
|
-
await receiver.start();
|
|
748
|
-
await waitFor(() => sqs.deleted.length === 1 && drainRecords.length === 1);
|
|
749
|
-
await receiver.dispose();
|
|
750
|
-
expect(sqs.deleted).toEqual(["rh-unchanged"]);
|
|
751
|
-
expect(syncBatchFn).not.toHaveBeenCalled();
|
|
752
|
-
expect(drainRecords[0]).toMatchObject({
|
|
753
|
-
received: 1,
|
|
754
|
-
coalesced: 1,
|
|
755
|
-
processed: 0,
|
|
756
|
-
skippedUnchanged: 1,
|
|
757
|
-
deleted: 1,
|
|
758
|
-
remaining: 0,
|
|
759
|
-
});
|
|
760
|
-
});
|
|
761
|
-
it("falls back to a targeted pull when the unchanged proof fails", async () => {
|
|
762
|
-
const event = makeEvent({ relativePath: "companies/indigo/proof-error.md" });
|
|
763
|
-
const sqs = new FakeSqs();
|
|
764
|
-
sqs.enqueueDrainable([msg(event, "rh-proof-error")]);
|
|
765
|
-
const syncBatchFn = vi.fn(async () => undefined);
|
|
766
|
-
const warnings = [];
|
|
767
|
-
const receiver = new SqsPushReceiver({
|
|
768
|
-
tenantId: TENANT,
|
|
769
|
-
queueUrl: QUEUE_URL,
|
|
770
|
-
sqs,
|
|
771
|
-
syncBatchFn,
|
|
772
|
-
skipUnchangedFn: async () => { throw new Error("journal unavailable"); },
|
|
773
|
-
sleep: fastSleep,
|
|
774
|
-
publishMetric: async () => undefined,
|
|
775
|
-
logger: {
|
|
776
|
-
info: () => undefined,
|
|
777
|
-
warn: (record) => warnings.push(record),
|
|
778
|
-
error: () => undefined,
|
|
779
|
-
debug: () => undefined,
|
|
780
|
-
},
|
|
781
|
-
});
|
|
782
|
-
await receiver.start();
|
|
783
|
-
await waitFor(() => sqs.deleted.length === 1 && syncBatchFn.mock.calls.length === 1);
|
|
784
|
-
await receiver.dispose();
|
|
785
|
-
expect(syncBatchFn.mock.calls[0]?.[0].events).toEqual([event]);
|
|
786
|
-
expect(warnings).toContainEqual(expect.objectContaining({
|
|
787
|
-
event: "receiver.unchanged.check.failed",
|
|
788
|
-
err: { message: "journal unavailable" },
|
|
789
|
-
}));
|
|
790
|
-
});
|
|
791
634
|
it("reports redelivery count, oldest sent age, and filtered self echoes", async () => {
|
|
792
635
|
const nowMs = 10_000;
|
|
793
636
|
const sqs = new FakeSqs();
|
|
@@ -938,7 +781,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
938
781
|
expect(sqs.deleted).toHaveLength(500);
|
|
939
782
|
expect(sqs.receiveWaitTimes.slice(0, 3)).toEqual([20, 0, 1]);
|
|
940
783
|
});
|
|
941
|
-
it("
|
|
784
|
+
it("amortizes 1,000 receipts across 256 paths into one pass and deletes them after success", async () => {
|
|
942
785
|
const sqs = new FakeSqs();
|
|
943
786
|
const receipts = Array.from({ length: 1_000 }, (_, index) => msg(makeEvent({
|
|
944
787
|
relativePath: `companies/indigo/deep/${index % 256}.md`,
|
|
@@ -957,12 +800,12 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
957
800
|
publishMetric: async () => undefined,
|
|
958
801
|
});
|
|
959
802
|
await receiver.start();
|
|
960
|
-
await waitFor(() =>
|
|
803
|
+
await waitFor(() => receiver.processedCount === 256);
|
|
961
804
|
await receiver.dispose();
|
|
962
|
-
expect(batches
|
|
805
|
+
expect(batches).toHaveLength(1);
|
|
806
|
+
expect(batches[0].events).toHaveLength(256);
|
|
963
807
|
expect(sqs.deleted).toHaveLength(1_000);
|
|
964
|
-
expect(sqs.
|
|
965
|
-
expect(sqs.receiveWaitTimes[0]).toBe(20);
|
|
808
|
+
expect(sqs.receiveWaitTimes.slice(0, 100)).toEqual([20, ...Array(99).fill(0)]);
|
|
966
809
|
});
|
|
967
810
|
it("reports a nonempty large drain before it can later report an empty long-poll", async () => {
|
|
968
811
|
const sqs = new FakeSqs();
|
|
@@ -984,7 +827,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
984
827
|
publishMetric: async () => undefined,
|
|
985
828
|
});
|
|
986
829
|
await receiver.start();
|
|
987
|
-
await waitFor(() => receiver.processedCount
|
|
830
|
+
await waitFor(() => receiver.processedCount === 256 && activity.includes(false));
|
|
988
831
|
await receiver.dispose();
|
|
989
832
|
expect(activity[0]).toBe(true);
|
|
990
833
|
expect(activity).toContain(false);
|
|
@@ -1030,137 +873,6 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1030
873
|
expect(sqs.requestedVisibilityTimeouts[0]).toBe(600);
|
|
1031
874
|
expect(sqs.deliveries).toHaveLength(1);
|
|
1032
875
|
});
|
|
1033
|
-
it("heartbeats a receipt when a targeted pull outlives its visibility lease", async () => {
|
|
1034
|
-
const sqs = new VisibilityExpirySqs(makeEvent({ sequenceNumber: 42 }));
|
|
1035
|
-
const receiver = new SqsPushReceiver({
|
|
1036
|
-
tenantId: TENANT,
|
|
1037
|
-
queueUrl: QUEUE_URL,
|
|
1038
|
-
sqs,
|
|
1039
|
-
visibilityTimeoutSeconds: 1,
|
|
1040
|
-
visibilityHeartbeatIntervalMs: 1,
|
|
1041
|
-
syncBatchFn: async () => {
|
|
1042
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
1043
|
-
},
|
|
1044
|
-
sleep: fastSleep,
|
|
1045
|
-
publishMetric: async () => undefined,
|
|
1046
|
-
});
|
|
1047
|
-
await receiver.start();
|
|
1048
|
-
try {
|
|
1049
|
-
await waitFor(() => sqs.deleteAttempts.length === 1);
|
|
1050
|
-
}
|
|
1051
|
-
finally {
|
|
1052
|
-
await receiver.dispose();
|
|
1053
|
-
}
|
|
1054
|
-
expect(sqs.requestedVisibilityTimeouts[0]).toBe(1);
|
|
1055
|
-
expect(sqs.visibilityChanges.length).toBeGreaterThan(0);
|
|
1056
|
-
expect(sqs.deliveries).toHaveLength(1);
|
|
1057
|
-
expect(sqs.deleted).toHaveLength(1);
|
|
1058
|
-
});
|
|
1059
|
-
it("keeps the receipt lease alive until a slow successful delete completes", async () => {
|
|
1060
|
-
const sqs = new VisibilityExpirySqs(makeEvent({ sequenceNumber: 43 }));
|
|
1061
|
-
sqs.deleteDelayMs = 30;
|
|
1062
|
-
const receiver = new SqsPushReceiver({
|
|
1063
|
-
tenantId: TENANT,
|
|
1064
|
-
queueUrl: QUEUE_URL,
|
|
1065
|
-
sqs,
|
|
1066
|
-
visibilityTimeoutSeconds: 1,
|
|
1067
|
-
visibilityHeartbeatIntervalMs: 1,
|
|
1068
|
-
syncBatchFn: async () => undefined,
|
|
1069
|
-
sleep: fastSleep,
|
|
1070
|
-
publishMetric: async () => undefined,
|
|
1071
|
-
});
|
|
1072
|
-
await receiver.start();
|
|
1073
|
-
try {
|
|
1074
|
-
await waitFor(() => sqs.deleteAttempts.length === 1);
|
|
1075
|
-
}
|
|
1076
|
-
finally {
|
|
1077
|
-
await receiver.dispose();
|
|
1078
|
-
}
|
|
1079
|
-
expect(sqs.visibilityChanges.some((change) => change.visibilityTimeoutSeconds === 1)).toBe(true);
|
|
1080
|
-
expect(sqs.deleteAttempts).toEqual([{ receiptHandle: "visibility-rh-1", expired: false }]);
|
|
1081
|
-
expect(sqs.deliveries).toHaveLength(1);
|
|
1082
|
-
expect(sqs.deleted).toHaveLength(1);
|
|
1083
|
-
});
|
|
1084
|
-
it("waits for an in-flight receipt heartbeat before releasing a failed delete", async () => {
|
|
1085
|
-
let delivered = false;
|
|
1086
|
-
let resolveHeartbeat;
|
|
1087
|
-
const heartbeatGate = new Promise((resolve) => { resolveHeartbeat = resolve; });
|
|
1088
|
-
let releaseDelete;
|
|
1089
|
-
const deleteGate = new Promise((resolve) => { releaseDelete = resolve; });
|
|
1090
|
-
const calls = [];
|
|
1091
|
-
const sqs = {
|
|
1092
|
-
receiveMessage: async ({ signal, waitTimeSeconds }) => {
|
|
1093
|
-
if (!delivered) {
|
|
1094
|
-
delivered = true;
|
|
1095
|
-
return { messages: [msg(makeEvent({ sequenceNumber: 44 }), "rh-failed-delete")] };
|
|
1096
|
-
}
|
|
1097
|
-
if (signal.aborted || waitTimeSeconds <= 1)
|
|
1098
|
-
return { messages: [] };
|
|
1099
|
-
return new Promise((resolve) => {
|
|
1100
|
-
signal.addEventListener("abort", () => resolve({ messages: [] }), { once: true });
|
|
1101
|
-
});
|
|
1102
|
-
},
|
|
1103
|
-
deleteMessage: async () => {
|
|
1104
|
-
calls.push("delete-start");
|
|
1105
|
-
await deleteGate;
|
|
1106
|
-
calls.push("delete-failed");
|
|
1107
|
-
throw new Error("simulated delete failure");
|
|
1108
|
-
},
|
|
1109
|
-
changeMessageVisibility: async ({ visibilityTimeoutSeconds }) => {
|
|
1110
|
-
if (visibilityTimeoutSeconds === 0) {
|
|
1111
|
-
calls.push("release");
|
|
1112
|
-
return;
|
|
1113
|
-
}
|
|
1114
|
-
calls.push("heartbeat-start");
|
|
1115
|
-
await heartbeatGate;
|
|
1116
|
-
calls.push("heartbeat-end");
|
|
1117
|
-
},
|
|
1118
|
-
changeMessageVisibilityBatch: async ({ entries, visibilityTimeoutSeconds }) => {
|
|
1119
|
-
for (const _entry of entries) {
|
|
1120
|
-
if (visibilityTimeoutSeconds === 0) {
|
|
1121
|
-
calls.push("release");
|
|
1122
|
-
}
|
|
1123
|
-
else {
|
|
1124
|
-
calls.push("heartbeat-start");
|
|
1125
|
-
await heartbeatGate;
|
|
1126
|
-
calls.push("heartbeat-end");
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
return { failedEntries: [] };
|
|
1130
|
-
},
|
|
1131
|
-
};
|
|
1132
|
-
const receiver = new SqsPushReceiver({
|
|
1133
|
-
tenantId: TENANT,
|
|
1134
|
-
queueUrl: QUEUE_URL,
|
|
1135
|
-
sqs,
|
|
1136
|
-
visibilityHeartbeatIntervalMs: 5,
|
|
1137
|
-
syncBatchFn: async () => undefined,
|
|
1138
|
-
sleep: fastSleep,
|
|
1139
|
-
publishMetric: async () => undefined,
|
|
1140
|
-
});
|
|
1141
|
-
await receiver.start();
|
|
1142
|
-
try {
|
|
1143
|
-
await waitFor(() => calls.includes("delete-start"));
|
|
1144
|
-
await waitFor(() => calls.includes("heartbeat-start"));
|
|
1145
|
-
releaseDelete();
|
|
1146
|
-
await waitFor(() => calls.includes("delete-failed"));
|
|
1147
|
-
expect(calls).toEqual(["delete-start", "heartbeat-start", "delete-failed"]);
|
|
1148
|
-
resolveHeartbeat();
|
|
1149
|
-
await waitFor(() => calls.includes("release"));
|
|
1150
|
-
expect(calls).toEqual([
|
|
1151
|
-
"delete-start",
|
|
1152
|
-
"heartbeat-start",
|
|
1153
|
-
"delete-failed",
|
|
1154
|
-
"heartbeat-end",
|
|
1155
|
-
"release",
|
|
1156
|
-
]);
|
|
1157
|
-
}
|
|
1158
|
-
finally {
|
|
1159
|
-
releaseDelete();
|
|
1160
|
-
resolveHeartbeat();
|
|
1161
|
-
await receiver.dispose();
|
|
1162
|
-
}
|
|
1163
|
-
});
|
|
1164
876
|
it("REGRESSION: drains a 5,000-message FIFO backlog through bounded slow passes without lease-expiry redelivery", async () => {
|
|
1165
877
|
const total = 5_000;
|
|
1166
878
|
const sqs = new BacklogSqs(Array.from({ length: total }, (_, index) => msg(makeEvent({
|
|
@@ -1213,7 +925,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1213
925
|
expect.objectContaining({ remaining: 0 }),
|
|
1214
926
|
]));
|
|
1215
927
|
}, 15_000);
|
|
1216
|
-
it("
|
|
928
|
+
it("REGRESSION: releases over-cap receipts and brings a fresh event into the next pass", async () => {
|
|
1217
929
|
const first = [
|
|
1218
930
|
msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a"),
|
|
1219
931
|
msg(makeEvent({ relativePath: "companies/indigo/b.md", sequenceNumber: 2 }), "rh-b"),
|
|
@@ -1239,13 +951,8 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1239
951
|
await receiver.start();
|
|
1240
952
|
await waitFor(() => batches.some((batch) => batch.includes("companies/indigo/fresh.md")));
|
|
1241
953
|
await receiver.dispose();
|
|
1242
|
-
expect(sqs.
|
|
1243
|
-
|
|
1244
|
-
expect(sqs.releases).toEqual([]);
|
|
1245
|
-
expect(sqs.deleted).toHaveLength(4);
|
|
1246
|
-
expect(batches[0]).toEqual([
|
|
1247
|
-
"companies/indigo/a.md",
|
|
1248
|
-
"companies/indigo/b.md",
|
|
954
|
+
expect(sqs.releases).toEqual([
|
|
955
|
+
{ receiptHandle: "rh-backlog", visibilityTimeoutSeconds: 0 },
|
|
1249
956
|
]);
|
|
1250
957
|
expect(batches[1]).toContain("companies/indigo/fresh.md");
|
|
1251
958
|
});
|
|
@@ -1279,7 +986,6 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1279
986
|
},
|
|
1280
987
|
deleteMessage: async ({ receiptHandle }) => { deleted.push(receiptHandle); },
|
|
1281
988
|
changeMessageVisibility: async () => { },
|
|
1282
|
-
changeMessageVisibilityBatch: async () => ({ failedEntries: [] }),
|
|
1283
989
|
};
|
|
1284
990
|
const batches = [];
|
|
1285
991
|
const receiver = new SqsPushReceiver({
|
|
@@ -1526,7 +1232,7 @@ describe("US-009: SqsPushReceiver — dedupe", () => {
|
|
|
1526
1232
|
});
|
|
1527
1233
|
});
|
|
1528
1234
|
describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
|
|
1529
|
-
it("leaves
|
|
1235
|
+
it("leaves every coalesced receipt unacknowledged on failure and retries the batch", async () => {
|
|
1530
1236
|
const sqs = new FakeSqs();
|
|
1531
1237
|
const events = [
|
|
1532
1238
|
makeEvent({ relativePath: "companies/indigo/retry-a.md", sequenceNumber: 1 }),
|
|
@@ -1534,38 +1240,25 @@ describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
|
|
|
1534
1240
|
makeEvent({ relativePath: "companies/indigo/retry-c.md", sequenceNumber: 1 }),
|
|
1535
1241
|
];
|
|
1536
1242
|
sqs.enqueue(events.map((event, index) => msg(event, `rh-failed-${index}`)));
|
|
1537
|
-
sqs.enqueue(
|
|
1538
|
-
let
|
|
1539
|
-
let retryAPathCalls = 0;
|
|
1243
|
+
sqs.enqueue(events.map((event, index) => msg(event, `rh-redelivered-${index}`)));
|
|
1244
|
+
let passCount = 0;
|
|
1540
1245
|
const receiver = new SqsPushReceiver({
|
|
1541
1246
|
tenantId: TENANT,
|
|
1542
1247
|
queueUrl: QUEUE_URL,
|
|
1543
1248
|
sqs,
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
if (
|
|
1547
|
-
|
|
1548
|
-
throw new Error("coalesced pull failed for one path");
|
|
1549
|
-
}
|
|
1550
|
-
if (batch[0]?.relativePath === events[0].relativePath) {
|
|
1551
|
-
retryAPathCalls += 1;
|
|
1552
|
-
if (retryAPathCalls === 1)
|
|
1553
|
-
throw new Error("retry-a is temporarily unavailable");
|
|
1554
|
-
}
|
|
1249
|
+
syncBatchFn: async () => {
|
|
1250
|
+
passCount += 1;
|
|
1251
|
+
if (passCount === 1)
|
|
1252
|
+
throw new Error("coalesced pull failed");
|
|
1555
1253
|
},
|
|
1556
1254
|
sleep: fastSleep,
|
|
1557
1255
|
publishMetric: async () => undefined,
|
|
1558
1256
|
});
|
|
1559
1257
|
await receiver.start();
|
|
1560
|
-
await waitFor(() => receiver.processedCount === 3
|
|
1258
|
+
await waitFor(() => passCount === 2 && receiver.processedCount === 3);
|
|
1561
1259
|
await receiver.dispose();
|
|
1562
|
-
expect(
|
|
1563
|
-
expect(
|
|
1564
|
-
expect(sqs.deleted).not.toContain("rh-failed-0");
|
|
1565
|
-
expect(sqs.deleted).toEqual(expect.arrayContaining(["rh-failed-1", "rh-failed-2", "rh-redelivered-a"]));
|
|
1566
|
-
expect(sqs.visibilityChanges).toEqual([
|
|
1567
|
-
{ receiptHandle: "rh-failed-0", visibilityTimeoutSeconds: 20 },
|
|
1568
|
-
]);
|
|
1260
|
+
expect(sqs.deleted).not.toEqual(expect.arrayContaining(["rh-failed-0", "rh-failed-1", "rh-failed-2"]));
|
|
1261
|
+
expect(sqs.deleted).toEqual(expect.arrayContaining(["rh-redelivered-0", "rh-redelivered-1", "rh-redelivered-2"]));
|
|
1569
1262
|
});
|
|
1570
1263
|
it("recovers from transient receiveMessage failures (backoff + resume) and replays the queue", async () => {
|
|
1571
1264
|
const sqs = new FakeSqs();
|
|
@@ -1887,82 +1580,6 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
1887
1580
|
expect(sqs.deleted).toContain("rh-next-second");
|
|
1888
1581
|
expect(sqs.deleted).not.toContain("rh-next-first");
|
|
1889
1582
|
});
|
|
1890
|
-
it("acks successful paths and retries only paths that fail inside a coalesced batch", async () => {
|
|
1891
|
-
const blocked = makeEvent({
|
|
1892
|
-
relativePath: "companies/indigo/out-of-scope.md",
|
|
1893
|
-
sequenceNumber: 103,
|
|
1894
|
-
});
|
|
1895
|
-
const allowed = makeEvent({
|
|
1896
|
-
relativePath: "companies/indigo/in-scope.md",
|
|
1897
|
-
sequenceNumber: 104,
|
|
1898
|
-
});
|
|
1899
|
-
const sqs = new FakeSqs();
|
|
1900
|
-
sqs.enqueueDrainable([
|
|
1901
|
-
msg(blocked, "rh-out-of-scope"),
|
|
1902
|
-
msg(allowed, "rh-in-scope"),
|
|
1903
|
-
]);
|
|
1904
|
-
const calls = [];
|
|
1905
|
-
const drains = [];
|
|
1906
|
-
const syncBatchFn = async ({ events }) => {
|
|
1907
|
-
const paths = events.map((event) => event.relativePath);
|
|
1908
|
-
calls.push(paths);
|
|
1909
|
-
if (paths.length > 1 || paths[0] === blocked.relativePath) {
|
|
1910
|
-
throw new Error("targeted batch includes an out-of-scope path");
|
|
1911
|
-
}
|
|
1912
|
-
};
|
|
1913
|
-
const receiver = new SqsPushReceiver({
|
|
1914
|
-
tenantId: TENANT,
|
|
1915
|
-
queueUrl: QUEUE_URL,
|
|
1916
|
-
sqs,
|
|
1917
|
-
syncBatchFn,
|
|
1918
|
-
sleep: fastSleep,
|
|
1919
|
-
publishMetric: async () => undefined,
|
|
1920
|
-
logger: makeDrainLogger(drains),
|
|
1921
|
-
});
|
|
1922
|
-
await receiver.start();
|
|
1923
|
-
await waitFor(() => drains.length === 1);
|
|
1924
|
-
await receiver.dispose();
|
|
1925
|
-
expect(calls).toEqual([
|
|
1926
|
-
[blocked.relativePath, allowed.relativePath],
|
|
1927
|
-
[blocked.relativePath],
|
|
1928
|
-
[allowed.relativePath],
|
|
1929
|
-
]);
|
|
1930
|
-
expect(sqs.deleted).toEqual(["rh-in-scope"]);
|
|
1931
|
-
expect(sqs.visibilityChanges).toEqual([
|
|
1932
|
-
{ receiptHandle: "rh-out-of-scope", visibilityTimeoutSeconds: 20 },
|
|
1933
|
-
]);
|
|
1934
|
-
expect(drains[0]).toMatchObject({ deleted: 1, retried: 1 });
|
|
1935
|
-
});
|
|
1936
|
-
it("holds the lease and resumes locally after a realtime-admission timeout", async () => {
|
|
1937
|
-
const sqs = new FakeSqs();
|
|
1938
|
-
sqs.enqueueDrainable([msg(first, "rh-admission-timeout")]);
|
|
1939
|
-
const drains = [];
|
|
1940
|
-
const warnings = [];
|
|
1941
|
-
let attempts = 0;
|
|
1942
|
-
const admissionTimeout = new Error("realtime admission timeout after 120000ms");
|
|
1943
|
-
admissionTimeout.name = "RealtimeAdmissionTimeout";
|
|
1944
|
-
const receiver = new SqsPushReceiver({
|
|
1945
|
-
tenantId: TENANT,
|
|
1946
|
-
queueUrl: QUEUE_URL,
|
|
1947
|
-
sqs,
|
|
1948
|
-
syncBatchFn: async () => {
|
|
1949
|
-
attempts += 1;
|
|
1950
|
-
if (attempts === 1)
|
|
1951
|
-
throw admissionTimeout;
|
|
1952
|
-
},
|
|
1953
|
-
sleep: fastSleep,
|
|
1954
|
-
publishMetric: async () => undefined,
|
|
1955
|
-
logger: makeDrainLogger(drains, warnings),
|
|
1956
|
-
});
|
|
1957
|
-
await receiver.start();
|
|
1958
|
-
await waitFor(() => sqs.deleted.length === 1);
|
|
1959
|
-
await receiver.dispose();
|
|
1960
|
-
expect(attempts).toBe(2);
|
|
1961
|
-
expect(sqs.visibilityChanges).toEqual([]);
|
|
1962
|
-
expect(sqs.deleted).toEqual(["rh-admission-timeout"]);
|
|
1963
|
-
expect(warnings.some((warning) => warning.event === "receiver.retry.failed")).toBe(false);
|
|
1964
|
-
expect(drains[0]).toMatchObject({ deleted: 1, retried: 0 });
|
|
1965
|
-
});
|
|
1966
1583
|
it("keeps the all-success path free of visibility changes and deletes every receipt", async () => {
|
|
1967
1584
|
const sqs = new FakeSqs();
|
|
1968
1585
|
sqs.enqueueDrainable([
|
|
@@ -2020,197 +1637,6 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
2020
1637
|
messageId: `m-${first.relativePath}-${first.sequenceNumber}`,
|
|
2021
1638
|
}));
|
|
2022
1639
|
});
|
|
2023
|
-
it("bounds a 64-path transport failure to one dispatch and batches the retry lease", async () => {
|
|
2024
|
-
const total = 64;
|
|
2025
|
-
const events = Array.from({ length: total }, (_, index) => makeEvent({
|
|
2026
|
-
relativePath: `companies/indigo/transport-${index}.md`,
|
|
2027
|
-
sequenceNumber: index + 1,
|
|
2028
|
-
}));
|
|
2029
|
-
const sqs = new FakeSqs();
|
|
2030
|
-
sqs.enqueueDrainable(events.map((event, index) => msg(event, `rh-transport-${index}`)));
|
|
2031
|
-
const drains = [];
|
|
2032
|
-
let dispatchCalls = 0;
|
|
2033
|
-
const transportError = Object.assign(new Error("ECONNRESET: socket hang up while pulling remote objects"), { code: "ECONNRESET" });
|
|
2034
|
-
const startedAt = Date.now();
|
|
2035
|
-
const receiver = new SqsPushReceiver({
|
|
2036
|
-
tenantId: TENANT,
|
|
2037
|
-
queueUrl: QUEUE_URL,
|
|
2038
|
-
sqs,
|
|
2039
|
-
drainMaxMessages: total,
|
|
2040
|
-
batchMaxPaths: total,
|
|
2041
|
-
syncBatchFn: async () => {
|
|
2042
|
-
dispatchCalls += 1;
|
|
2043
|
-
throw transportError;
|
|
2044
|
-
},
|
|
2045
|
-
sleep: fastSleep,
|
|
2046
|
-
publishMetric: async () => undefined,
|
|
2047
|
-
logger: makeDrainLogger(drains),
|
|
2048
|
-
});
|
|
2049
|
-
await receiver.start();
|
|
2050
|
-
await waitFor(() => drains.length === 1, 5_000);
|
|
2051
|
-
await receiver.dispose();
|
|
2052
|
-
expect(dispatchCalls).toBe(1);
|
|
2053
|
-
expect(Date.now() - startedAt).toBeLessThan(5_000);
|
|
2054
|
-
expect(sqs.deleted).toEqual([]);
|
|
2055
|
-
expect(sqs.visibilityChanges.filter((change) => change.visibilityTimeoutSeconds === 20))
|
|
2056
|
-
.toHaveLength(total);
|
|
2057
|
-
const retryCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 20);
|
|
2058
|
-
expect(retryCalls).toHaveLength(7);
|
|
2059
|
-
expect(retryCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2060
|
-
});
|
|
2061
|
-
it("bisects a path-specific batch failure and acknowledges the other 63 paths", async () => {
|
|
2062
|
-
const total = 64;
|
|
2063
|
-
const blocked = makeEvent({
|
|
2064
|
-
relativePath: `companies/indigo/blocked-${total - 1}.md`,
|
|
2065
|
-
sequenceNumber: total,
|
|
2066
|
-
});
|
|
2067
|
-
const events = Array.from({ length: total - 1 }, (_, index) => makeEvent({
|
|
2068
|
-
relativePath: `companies/indigo/allowed-${index}.md`,
|
|
2069
|
-
sequenceNumber: index + 1,
|
|
2070
|
-
})).concat(blocked);
|
|
2071
|
-
const sqs = new FakeSqs();
|
|
2072
|
-
sqs.enqueueDrainable(events.map((event, index) => msg(event, `rh-path-${index}`)));
|
|
2073
|
-
const drains = [];
|
|
2074
|
-
let dispatchCalls = 0;
|
|
2075
|
-
const receiver = new SqsPushReceiver({
|
|
2076
|
-
tenantId: TENANT,
|
|
2077
|
-
queueUrl: QUEUE_URL,
|
|
2078
|
-
sqs,
|
|
2079
|
-
drainMaxMessages: total,
|
|
2080
|
-
batchMaxPaths: total,
|
|
2081
|
-
syncBatchFn: async ({ events: batch }) => {
|
|
2082
|
-
dispatchCalls += 1;
|
|
2083
|
-
if (batch.some((event) => event.relativePath === blocked.relativePath)) {
|
|
2084
|
-
throw new Error("out-of-scope path rejected by targeted pull");
|
|
2085
|
-
}
|
|
2086
|
-
},
|
|
2087
|
-
sleep: fastSleep,
|
|
2088
|
-
publishMetric: async () => undefined,
|
|
2089
|
-
logger: makeDrainLogger(drains),
|
|
2090
|
-
});
|
|
2091
|
-
await receiver.start();
|
|
2092
|
-
await waitFor(() => drains.length === 1, 5_000);
|
|
2093
|
-
await receiver.dispose();
|
|
2094
|
-
expect(dispatchCalls).toBeLessThanOrEqual(13);
|
|
2095
|
-
expect(sqs.deleted).toHaveLength(total - 1);
|
|
2096
|
-
expect(sqs.deleted).not.toContain(`rh-path-${total - 1}`);
|
|
2097
|
-
expect(sqs.visibilityChanges.filter((change) => change.visibilityTimeoutSeconds === 20))
|
|
2098
|
-
.toEqual([{ receiptHandle: `rh-path-${total - 1}`, visibilityTimeoutSeconds: 20 }]);
|
|
2099
|
-
expect(drains[0]).toMatchObject({ processed: total - 1, deleted: total - 1, retried: 1 });
|
|
2100
|
-
});
|
|
2101
|
-
it("reports failed retry entries separately from successful entries", async () => {
|
|
2102
|
-
const sqs = new FakeSqs();
|
|
2103
|
-
sqs.failVisibilityBatchEntry("rh-retry-entry-failed", {
|
|
2104
|
-
code: "ReceiptHandleIsInvalid",
|
|
2105
|
-
message: "one receipt was rejected",
|
|
2106
|
-
senderFault: true,
|
|
2107
|
-
});
|
|
2108
|
-
sqs.enqueueDrainable([
|
|
2109
|
-
msg(first, "rh-retry-entry-failed"),
|
|
2110
|
-
msg(second, "rh-retry-entry-succeeded"),
|
|
2111
|
-
]);
|
|
2112
|
-
const warnings = [];
|
|
2113
|
-
const drains = [];
|
|
2114
|
-
const transportError = Object.assign(new Error("network connection reset"), {
|
|
2115
|
-
code: "ECONNRESET",
|
|
2116
|
-
});
|
|
2117
|
-
const receiver = new SqsPushReceiver({
|
|
2118
|
-
tenantId: TENANT,
|
|
2119
|
-
queueUrl: QUEUE_URL,
|
|
2120
|
-
sqs,
|
|
2121
|
-
syncBatchFn: async () => { throw transportError; },
|
|
2122
|
-
sleep: fastSleep,
|
|
2123
|
-
publishMetric: async () => undefined,
|
|
2124
|
-
logger: makeDrainLogger(drains, warnings),
|
|
2125
|
-
});
|
|
2126
|
-
await receiver.start();
|
|
2127
|
-
await waitFor(() => drains.length === 1);
|
|
2128
|
-
await receiver.dispose();
|
|
2129
|
-
const failures = warnings.filter((warning) => warning.event === "receiver.retry.failed");
|
|
2130
|
-
expect(failures).toHaveLength(1);
|
|
2131
|
-
expect(failures[0]).toMatchObject({
|
|
2132
|
-
messageId: `m-${first.relativePath}-${first.sequenceNumber}`,
|
|
2133
|
-
code: "ReceiptHandleIsInvalid",
|
|
2134
|
-
failureMessage: "one receipt was rejected",
|
|
2135
|
-
senderFault: true,
|
|
2136
|
-
});
|
|
2137
|
-
expect(sqs.visibilityChanges).toEqual([
|
|
2138
|
-
{ receiptHandle: "rh-retry-entry-succeeded", visibilityTimeoutSeconds: 20 },
|
|
2139
|
-
]);
|
|
2140
|
-
expect(drains[0]).toMatchObject({ retried: 1 });
|
|
2141
|
-
expect(failures[0]).not.toMatchObject({
|
|
2142
|
-
messageId: `m-${second.relativePath}-${second.sequenceNumber}`,
|
|
2143
|
-
});
|
|
2144
|
-
});
|
|
2145
|
-
it("uses about 100 ten-receipt API calls for a 1,000-receipt heartbeat", async () => {
|
|
2146
|
-
const total = 1_000;
|
|
2147
|
-
const sqs = new FakeSqs();
|
|
2148
|
-
const receipts = Array.from({ length: total }, (_, index) => msg(makeEvent({
|
|
2149
|
-
relativePath: `companies/indigo/heartbeat-${index}.md`,
|
|
2150
|
-
sequenceNumber: index + 1,
|
|
2151
|
-
}), `rh-heartbeat-${index}`));
|
|
2152
|
-
for (let offset = 0; offset < receipts.length; offset += 10) {
|
|
2153
|
-
sqs.enqueueDrainable(receipts.slice(offset, offset + 10));
|
|
2154
|
-
}
|
|
2155
|
-
let startPass;
|
|
2156
|
-
const passStarted = new Promise((resolve) => { startPass = resolve; });
|
|
2157
|
-
let releasePass;
|
|
2158
|
-
const passGate = new Promise((resolve) => { releasePass = resolve; });
|
|
2159
|
-
let finishDrain;
|
|
2160
|
-
const drainFinished = new Promise((resolve) => { finishDrain = resolve; });
|
|
2161
|
-
const receiver = new SqsPushReceiver({
|
|
2162
|
-
tenantId: TENANT,
|
|
2163
|
-
queueUrl: QUEUE_URL,
|
|
2164
|
-
sqs,
|
|
2165
|
-
drainMaxMessages: total,
|
|
2166
|
-
batchMaxPaths: total,
|
|
2167
|
-
maxInFlightDrains: 1,
|
|
2168
|
-
visibilityTimeoutSeconds: 600,
|
|
2169
|
-
visibilityHeartbeatIntervalMs: 1_000,
|
|
2170
|
-
syncBatchFn: async () => {
|
|
2171
|
-
startPass();
|
|
2172
|
-
await passGate;
|
|
2173
|
-
throw Object.assign(new Error("ECONNRESET while pulling the batch"), {
|
|
2174
|
-
code: "ECONNRESET",
|
|
2175
|
-
});
|
|
2176
|
-
},
|
|
2177
|
-
sleep: fastSleep,
|
|
2178
|
-
publishMetric: async () => undefined,
|
|
2179
|
-
logger: {
|
|
2180
|
-
debug: () => undefined,
|
|
2181
|
-
warn: () => undefined,
|
|
2182
|
-
error: () => undefined,
|
|
2183
|
-
info: (record) => {
|
|
2184
|
-
if (record.event === "receiver.drain")
|
|
2185
|
-
finishDrain();
|
|
2186
|
-
},
|
|
2187
|
-
},
|
|
2188
|
-
});
|
|
2189
|
-
vi.useFakeTimers();
|
|
2190
|
-
try {
|
|
2191
|
-
await receiver.start();
|
|
2192
|
-
await passStarted;
|
|
2193
|
-
await vi.advanceTimersByTimeAsync(1_000);
|
|
2194
|
-
const heartbeatCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 600);
|
|
2195
|
-
expect(heartbeatCalls.length).toBeLessThanOrEqual(110);
|
|
2196
|
-
expect(heartbeatCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2197
|
-
expect(heartbeatCalls.reduce((count, call) => count + call.entries.length, 0))
|
|
2198
|
-
.toBe(total);
|
|
2199
|
-
releasePass();
|
|
2200
|
-
await drainFinished;
|
|
2201
|
-
await receiver.dispose();
|
|
2202
|
-
const retryCalls = sqs.visibilityBatchCalls.filter((call) => call.visibilityTimeoutSeconds === 20);
|
|
2203
|
-
expect(retryCalls.length).toBeLessThanOrEqual(110);
|
|
2204
|
-
expect(retryCalls.every((call) => call.entries.length <= 10)).toBe(true);
|
|
2205
|
-
expect(retryCalls.reduce((count, call) => count + call.entries.length, 0))
|
|
2206
|
-
.toBe(total);
|
|
2207
|
-
}
|
|
2208
|
-
finally {
|
|
2209
|
-
releasePass();
|
|
2210
|
-
await receiver.dispose();
|
|
2211
|
-
vi.useRealTimers();
|
|
2212
|
-
}
|
|
2213
|
-
});
|
|
2214
1640
|
it("updates a large failed batch with bounded parallel visibility calls", async () => {
|
|
2215
1641
|
const total = 512;
|
|
2216
1642
|
const baseSqs = new FakeSqs();
|
|
@@ -2221,7 +1647,7 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
2221
1647
|
for (let offset = 0; offset < receipts.length; offset += 10) {
|
|
2222
1648
|
baseSqs.enqueueDrainable(receipts.slice(offset, offset + 10));
|
|
2223
1649
|
}
|
|
2224
|
-
const
|
|
1650
|
+
const visibilityCalls = [];
|
|
2225
1651
|
let activeCalls = 0;
|
|
2226
1652
|
let maxActiveCalls = 0;
|
|
2227
1653
|
let releaseVisibility;
|
|
@@ -2236,24 +1662,12 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
2236
1662
|
receiveMessage: (args) => baseSqs.receiveMessage(args),
|
|
2237
1663
|
deleteMessage: (args) => baseSqs.deleteMessage(args),
|
|
2238
1664
|
changeMessageVisibility: async ({ receiptHandle }) => {
|
|
2239
|
-
|
|
2240
|
-
activeCalls += 1;
|
|
2241
|
-
maxActiveCalls = Math.max(maxActiveCalls, activeCalls);
|
|
2242
|
-
firstCallStarted();
|
|
2243
|
-
await visibilityGate;
|
|
2244
|
-
activeCalls -= 1;
|
|
2245
|
-
},
|
|
2246
|
-
changeMessageVisibilityBatch: async ({ entries, visibilityTimeoutSeconds }) => {
|
|
2247
|
-
visibilityBatchCalls.push({
|
|
2248
|
-
receiptHandles: entries.map((entry) => entry.receiptHandle),
|
|
2249
|
-
visibilityTimeoutSeconds,
|
|
2250
|
-
});
|
|
1665
|
+
visibilityCalls.push(receiptHandle);
|
|
2251
1666
|
activeCalls += 1;
|
|
2252
1667
|
maxActiveCalls = Math.max(maxActiveCalls, activeCalls);
|
|
2253
1668
|
firstCallStarted();
|
|
2254
1669
|
await visibilityGate;
|
|
2255
1670
|
activeCalls -= 1;
|
|
2256
|
-
return { failedEntries: [] };
|
|
2257
1671
|
},
|
|
2258
1672
|
};
|
|
2259
1673
|
const drains = [];
|
|
@@ -2277,10 +1691,7 @@ describe("SqsPushReceiver failed-batch retry visibility", () => {
|
|
|
2277
1691
|
await receiver.dispose();
|
|
2278
1692
|
expect(parallelCallsBeforeRelease).toBeGreaterThan(1);
|
|
2279
1693
|
expect(parallelCallsBeforeRelease).toBeLessThan(total);
|
|
2280
|
-
expect(
|
|
2281
|
-
expect(visibilityBatchCalls.every((call) => call.receiptHandles.length <= 10)).toBe(true);
|
|
2282
|
-
expect(visibilityBatchCalls.flatMap((call) => call.receiptHandles)).toHaveLength(total);
|
|
2283
|
-
expect(visibilityBatchCalls.every((call) => call.visibilityTimeoutSeconds === 20)).toBe(true);
|
|
1694
|
+
expect(visibilityCalls).toHaveLength(total);
|
|
2284
1695
|
expect(baseSqs.deleted).toEqual([]);
|
|
2285
1696
|
});
|
|
2286
1697
|
});
|