@indigoai-us/hq-cloud 6.18.11 → 6.18.12-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-args.test.d.ts +2 -0
- package/dist/bin/sync-runner-args.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-args.test.js +82 -0
- package/dist/bin/sync-runner-args.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +1 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +3 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +31 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts +2 -0
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js +500 -0
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js.map +1 -0
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts +2 -0
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js +317 -0
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +195 -20
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +71 -0
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -1
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +7 -2
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +15 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +71 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +52 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +119 -3
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +2 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +76 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +13 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +81 -4
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +228 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +3 -0
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +8 -0
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal.d.ts +15 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +66 -2
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +16 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest-resident-parity.test.js +5 -3
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +1 -0
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +54 -17
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +109 -0
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/content-hash.d.ts +4 -0
- package/dist/sync/content-hash.d.ts.map +1 -0
- package/dist/sync/content-hash.js +11 -0
- package/dist/sync/content-hash.js.map +1 -0
- package/dist/sync/content-hash.test.d.ts +2 -0
- package/dist/sync/content-hash.test.d.ts.map +1 -0
- package/dist/sync/content-hash.test.js +16 -0
- package/dist/sync/content-hash.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +58 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +46 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/logger.test.js +3 -0
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +3 -0
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +52 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +416 -50
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +673 -40
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/unchanged-push-events.d.ts +16 -0
- package/dist/sync/unchanged-push-events.d.ts.map +1 -0
- package/dist/sync/unchanged-push-events.js +51 -0
- package/dist/sync/unchanged-push-events.js.map +1 -0
- package/dist/sync/unchanged-push-events.test.d.ts +2 -0
- package/dist/sync/unchanged-push-events.test.d.ts.map +1 -0
- package/dist/sync/unchanged-push-events.test.js +76 -0
- package/dist/sync/unchanged-push-events.test.js.map +1 -0
- package/dist/watcher.d.ts +18 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +78 -13
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +72 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
|
@@ -115,8 +115,14 @@ export const DEFAULT_MAX_MESSAGES = 10;
|
|
|
115
115
|
export const DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS = 600;
|
|
116
116
|
/** Retry a failed targeted pull promptly while preserving queue ownership. */
|
|
117
117
|
export const RECEIVER_RETRY_VISIBILITY_SECONDS = 20;
|
|
118
|
+
/** SQS ChangeMessageVisibilityBatch accepts at most ten receipt entries. */
|
|
119
|
+
const RECEIVER_VISIBILITY_BATCH_SIZE = 10;
|
|
118
120
|
/** Bound the retry fanout while resetting the default 1,000-receipt drain promptly. */
|
|
119
121
|
const RECEIVER_RETRY_VISIBILITY_CONCURRENCY = 256;
|
|
122
|
+
const RECEIVER_SKIP_ACK_CONCURRENCY = 64;
|
|
123
|
+
/** Keep an admitted realtime pull local when a full chore temporarily denies admission. */
|
|
124
|
+
const RECEIVER_ADMISSION_RETRY_INITIAL_MS = 5_000;
|
|
125
|
+
const RECEIVER_ADMISSION_RETRY_MAX_MS = 60_000;
|
|
120
126
|
/** Reconnect backoff defaults. */
|
|
121
127
|
export const DEFAULT_RECONNECT_INITIAL_MS = 250;
|
|
122
128
|
export const DEFAULT_RECONNECT_MAX_MS = 30_000;
|
|
@@ -170,6 +176,86 @@ class BoundedSequenceDedupe {
|
|
|
170
176
|
}
|
|
171
177
|
}
|
|
172
178
|
}
|
|
179
|
+
/**
|
|
180
|
+
* A production pull bridge can carry the runner's failure reason and class
|
|
181
|
+
* across its exit-code boundary so the receiver does not split a transport
|
|
182
|
+
* failure into per-path retries.
|
|
183
|
+
*/
|
|
184
|
+
export class ReceiverDispatchError extends Error {
|
|
185
|
+
failureClass;
|
|
186
|
+
failureReason;
|
|
187
|
+
constructor(failureClass, failureReason) {
|
|
188
|
+
super(failureReason);
|
|
189
|
+
this.failureClass = failureClass;
|
|
190
|
+
this.failureReason = failureReason;
|
|
191
|
+
this.name = "ReceiverDispatchError";
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function isRealtimeAdmissionTimeout(error) {
|
|
195
|
+
if (typeof error !== "object" || error === null)
|
|
196
|
+
return false;
|
|
197
|
+
const candidate = error;
|
|
198
|
+
return candidate.name === "RealtimeAdmissionTimeout" ||
|
|
199
|
+
(typeof candidate.message === "string" &&
|
|
200
|
+
/^realtime admission timeout\b/i.test(candidate.message));
|
|
201
|
+
}
|
|
202
|
+
const TRANSPORT_FAILURE_CODES = new Set([
|
|
203
|
+
"ABORT_ERR",
|
|
204
|
+
"ACCESSDENIED",
|
|
205
|
+
"ACCESSDENIEDEXCEPTION",
|
|
206
|
+
"CREDENTIALSEXPIRED",
|
|
207
|
+
"ECONNABORTED",
|
|
208
|
+
"ECONNREFUSED",
|
|
209
|
+
"ECONNRESET",
|
|
210
|
+
"EDQUOT",
|
|
211
|
+
"EHOSTUNREACH",
|
|
212
|
+
"EAI_AGAIN",
|
|
213
|
+
"ENETDOWN",
|
|
214
|
+
"ENETUNREACH",
|
|
215
|
+
"ENOSPC",
|
|
216
|
+
"ENOTFOUND",
|
|
217
|
+
"ETIMEDOUT",
|
|
218
|
+
"EXPIREDTOKEN",
|
|
219
|
+
"EXPIREDTOKENEXCEPTION",
|
|
220
|
+
"FORBIDDEN",
|
|
221
|
+
"INVALIDCLIENTTOKENID",
|
|
222
|
+
"REQUESTEXPIRED",
|
|
223
|
+
"TOKENEXPIRED",
|
|
224
|
+
"UNAUTHORIZED",
|
|
225
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
226
|
+
"UND_ERR_SOCKET",
|
|
227
|
+
]);
|
|
228
|
+
const TRANSPORT_FAILURE_MESSAGE = /(?:\b(?:ECONNABORTED|ECONNREFUSED|ECONNRESET|EDQUOT|EHOSTUNREACH|EAI_AGAIN|ENETDOWN|ENETUNREACH|ENOSPC|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_SOCKET)\b|no space left on device|disk full|network (?:error|failure|unavailable)|socket (?:hang up|closed)|connection (?:reset|timed? out|refused)|\babort(?:ed|ing)?\b|\btimed? ?out\b|\b(?:HTTP\s*)?(?:401|403|408|429|5\d\d)\b|\b(?:forbidden|unauthorized)\b|(?:access|credential|token).*(?:expired|expiry|invalid|forbidden|unauthorized))/i;
|
|
229
|
+
function isTransportClassError(error, visited = new Set()) {
|
|
230
|
+
if (typeof error === "string")
|
|
231
|
+
return TRANSPORT_FAILURE_MESSAGE.test(error);
|
|
232
|
+
if (typeof error !== "object" || error === null || visited.has(error))
|
|
233
|
+
return false;
|
|
234
|
+
visited.add(error);
|
|
235
|
+
if (error instanceof ReceiverDispatchError) {
|
|
236
|
+
return error.failureClass === "transport";
|
|
237
|
+
}
|
|
238
|
+
const candidate = error;
|
|
239
|
+
const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : "";
|
|
240
|
+
if (TRANSPORT_FAILURE_CODES.has(code) ||
|
|
241
|
+
/^(?:HTTP)?(?:401|403|408|429|5\d\d)$/.test(code) ||
|
|
242
|
+
(typeof candidate.name === "string" && /^(?:AbortError|TimeoutError|ExpiredTokenException)$/i.test(candidate.name))) {
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
const status = Number(candidate.status ?? candidate.statusCode ?? candidate.$metadata?.httpStatusCode);
|
|
246
|
+
if ([401, 403, 408, 429].includes(status) || status >= 500)
|
|
247
|
+
return true;
|
|
248
|
+
if (typeof candidate.message === "string" &&
|
|
249
|
+
TRANSPORT_FAILURE_MESSAGE.test(candidate.message)) {
|
|
250
|
+
return true;
|
|
251
|
+
}
|
|
252
|
+
return candidate.cause === undefined
|
|
253
|
+
? false
|
|
254
|
+
: isTransportClassError(candidate.cause, visited);
|
|
255
|
+
}
|
|
256
|
+
export function isReceiverTransportFailure(error) {
|
|
257
|
+
return isTransportClassError(error);
|
|
258
|
+
}
|
|
173
259
|
const NOOP_LOGGER = {
|
|
174
260
|
info: () => undefined,
|
|
175
261
|
warn: () => undefined,
|
|
@@ -311,6 +397,7 @@ export class SqsPushReceiver {
|
|
|
311
397
|
/** Legacy per-event seam, retained to preserve the old embedding contract. */
|
|
312
398
|
syncFn;
|
|
313
399
|
shouldProcess;
|
|
400
|
+
skipUnchangedFn;
|
|
314
401
|
onFilteredEvents;
|
|
315
402
|
onReceiveActivity;
|
|
316
403
|
logger;
|
|
@@ -318,6 +405,7 @@ export class SqsPushReceiver {
|
|
|
318
405
|
waitTimeSeconds;
|
|
319
406
|
maxMessages;
|
|
320
407
|
visibilityTimeoutSeconds;
|
|
408
|
+
visibilityHeartbeatIntervalMs;
|
|
321
409
|
disposeDrainMs;
|
|
322
410
|
reconnectInitialMs;
|
|
323
411
|
reconnectMaxMs;
|
|
@@ -369,6 +457,7 @@ export class SqsPushReceiver {
|
|
|
369
457
|
throw new Error("SqsPushReceiver: syncBatchFn is required");
|
|
370
458
|
}
|
|
371
459
|
this.shouldProcess = opts.shouldProcess ?? (() => true);
|
|
460
|
+
this.skipUnchangedFn = opts.skipUnchangedFn;
|
|
372
461
|
this.onFilteredEvents = opts.onFilteredEvents;
|
|
373
462
|
this.onReceiveActivity = opts.onReceiveActivity;
|
|
374
463
|
this.logger = opts.logger ?? NOOP_LOGGER;
|
|
@@ -376,6 +465,7 @@ export class SqsPushReceiver {
|
|
|
376
465
|
this.waitTimeSeconds = opts.waitTimeSeconds ?? DEFAULT_WAIT_TIME_SECONDS;
|
|
377
466
|
this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
|
|
378
467
|
this.visibilityTimeoutSeconds = Math.max(1, Math.min(43_200, Math.floor(opts.visibilityTimeoutSeconds ?? DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS)));
|
|
468
|
+
this.visibilityHeartbeatIntervalMs = Math.max(1, Math.min(2_147_483_647, Math.floor(opts.visibilityHeartbeatIntervalMs ?? this.visibilityTimeoutSeconds * 500)));
|
|
379
469
|
this.disposeDrainMs =
|
|
380
470
|
opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
|
|
381
471
|
this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
|
|
@@ -492,7 +582,7 @@ export class SqsPushReceiver {
|
|
|
492
582
|
try {
|
|
493
583
|
received = await this.sqs.receiveMessage({
|
|
494
584
|
queueUrl: this.queueUrl,
|
|
495
|
-
maxMessages: this.maxMessages,
|
|
585
|
+
maxMessages: Math.min(this.maxMessages, this.batchMaxPaths, this.drainMaxMessages),
|
|
496
586
|
waitTimeSeconds: this.waitTimeSeconds,
|
|
497
587
|
visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
|
|
498
588
|
signal,
|
|
@@ -578,6 +668,7 @@ export class SqsPushReceiver {
|
|
|
578
668
|
let releasedCount = 0;
|
|
579
669
|
let retriedCount = 0;
|
|
580
670
|
let processedCount = 0;
|
|
671
|
+
let skippedUnchangedCount = 0;
|
|
581
672
|
let redeliveredCount = 0;
|
|
582
673
|
let maxReceiveCount = 0;
|
|
583
674
|
let oldestSentTimestampMs;
|
|
@@ -589,15 +680,32 @@ export class SqsPushReceiver {
|
|
|
589
680
|
intakeCompleted = true;
|
|
590
681
|
onIntakeComplete();
|
|
591
682
|
};
|
|
683
|
+
const leaseHeartbeat = this.startReceiptLeaseHeartbeat();
|
|
592
684
|
const acknowledge = async (msg) => {
|
|
593
|
-
|
|
594
|
-
|
|
685
|
+
let heartbeatPaused = false;
|
|
686
|
+
try {
|
|
687
|
+
if (await this.safeDelete(msg)) {
|
|
688
|
+
deletedCount += 1;
|
|
689
|
+
return true;
|
|
690
|
+
}
|
|
691
|
+
// Drain any extension already in flight and remove this receipt from
|
|
692
|
+
// future snapshots before releasing it. Otherwise a late heartbeat
|
|
693
|
+
// can restore the full lease after the zero-visibility retry release.
|
|
694
|
+
await leaseHeartbeat.pause();
|
|
695
|
+
heartbeatPaused = true;
|
|
696
|
+
leaseHeartbeat.remove(msg);
|
|
697
|
+
if (await this.releaseUnselected(msg)) {
|
|
698
|
+
// A failed delete is not allowed to strand a stale receipt until
|
|
699
|
+
// its normal lease expires. Releasing returns retry ownership to
|
|
700
|
+
// SQS immediately while successful receipts remain heartbeatable.
|
|
701
|
+
releasedCount += 1;
|
|
702
|
+
}
|
|
703
|
+
return false;
|
|
595
704
|
}
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
releasedCount += 1;
|
|
705
|
+
finally {
|
|
706
|
+
leaseHeartbeat.remove(msg);
|
|
707
|
+
if (heartbeatPaused)
|
|
708
|
+
leaseHeartbeat.resume();
|
|
601
709
|
}
|
|
602
710
|
};
|
|
603
711
|
const countOrigin = (originDeviceId) => {
|
|
@@ -671,6 +779,7 @@ export class SqsPushReceiver {
|
|
|
671
779
|
const receipts = receiptsByPath.get(event.relativePath) ?? [];
|
|
672
780
|
receipts.push(msg);
|
|
673
781
|
receiptsByPath.set(event.relativePath, receipts);
|
|
782
|
+
leaseHeartbeat.add(msg);
|
|
674
783
|
if (!selectedEvent || event.sequenceNumber > selectedEvent.sequenceNumber) {
|
|
675
784
|
selected.set(event.relativePath, event);
|
|
676
785
|
}
|
|
@@ -693,6 +802,7 @@ export class SqsPushReceiver {
|
|
|
693
802
|
let emptyConfirmations = 0;
|
|
694
803
|
let waitTimeSeconds = 0;
|
|
695
804
|
while (!saturated &&
|
|
805
|
+
selected.size < this.batchMaxPaths &&
|
|
696
806
|
receivedCount < this.drainMaxMessages &&
|
|
697
807
|
!signal.aborted &&
|
|
698
808
|
this.now() < fillDeadline) {
|
|
@@ -706,7 +816,7 @@ export class SqsPushReceiver {
|
|
|
706
816
|
try {
|
|
707
817
|
next = await this.sqs.receiveMessage({
|
|
708
818
|
queueUrl: this.queueUrl,
|
|
709
|
-
maxMessages: Math.min(this.maxMessages, this.drainMaxMessages - receivedCount),
|
|
819
|
+
maxMessages: Math.min(this.maxMessages, this.batchMaxPaths - selected.size, this.drainMaxMessages - receivedCount),
|
|
710
820
|
waitTimeSeconds,
|
|
711
821
|
visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
|
|
712
822
|
signal,
|
|
@@ -740,50 +850,71 @@ export class SqsPushReceiver {
|
|
|
740
850
|
}
|
|
741
851
|
completeIntake();
|
|
742
852
|
const events = [...selected.values()];
|
|
743
|
-
|
|
853
|
+
let eventsToDispatch = events;
|
|
854
|
+
if (this.skipUnchangedFn && events.length > 0) {
|
|
855
|
+
let unchangedPaths = new Set();
|
|
856
|
+
try {
|
|
857
|
+
unchangedPaths = await this.skipUnchangedFn(events);
|
|
858
|
+
}
|
|
859
|
+
catch (err) {
|
|
860
|
+
// A journal/stat read is an optimization only. Preserve delivery by
|
|
861
|
+
// falling back to the ordinary targeted pull when the proof fails.
|
|
862
|
+
this.logger.warn({
|
|
863
|
+
event: "receiver.unchanged.check.failed",
|
|
864
|
+
tenantId: this.tenantId,
|
|
865
|
+
pathCount: events.length,
|
|
866
|
+
err: { message: err?.message },
|
|
867
|
+
}, "push receiver could not prove unchanged local content; dispatching pull");
|
|
868
|
+
}
|
|
869
|
+
const unchangedEvents = [];
|
|
870
|
+
eventsToDispatch = events.filter((event) => {
|
|
871
|
+
if (!unchangedPaths.has(event.relativePath))
|
|
872
|
+
return true;
|
|
873
|
+
unchangedEvents.push(event);
|
|
874
|
+
return false;
|
|
875
|
+
});
|
|
876
|
+
skippedUnchangedCount += unchangedEvents.length;
|
|
877
|
+
for (let offset = 0; offset < unchangedEvents.length; offset += RECEIVER_SKIP_ACK_CONCURRENCY) {
|
|
878
|
+
const acknowledgementBatch = unchangedEvents.slice(offset, offset + RECEIVER_SKIP_ACK_CONCURRENCY);
|
|
879
|
+
await Promise.all(acknowledgementBatch.map(async (event) => {
|
|
880
|
+
const receipts = receiptsByPath.get(event.relativePath) ?? [];
|
|
881
|
+
const acknowledgements = await Promise.all(receipts.map(acknowledge));
|
|
882
|
+
if (acknowledgements.every(Boolean)) {
|
|
883
|
+
this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
|
|
884
|
+
}
|
|
885
|
+
}));
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
const batches = eventsToDispatch.length === 0
|
|
889
|
+
? []
|
|
890
|
+
: this.syncBatchFn.partition?.(eventsToDispatch) ?? [eventsToDispatch];
|
|
744
891
|
for (const batch of batches) {
|
|
745
892
|
if (batch.length === 0)
|
|
746
893
|
continue;
|
|
747
|
-
const
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
await this.sqs.changeMessageVisibility({
|
|
762
|
-
queueUrl: this.queueUrl,
|
|
763
|
-
receiptHandle: msg.ReceiptHandle,
|
|
764
|
-
visibilityTimeoutSeconds: RECEIVER_RETRY_VISIBILITY_SECONDS,
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
catch (err) {
|
|
768
|
-
this.logger.warn({
|
|
769
|
-
event: "receiver.retry.failed",
|
|
770
|
-
tenantId: this.tenantId,
|
|
771
|
-
messageId: msg.MessageId,
|
|
772
|
-
err: { message: err?.message },
|
|
773
|
-
}, "push receiver failed to shorten retry visibility");
|
|
774
|
-
}
|
|
775
|
-
};
|
|
776
|
-
for (let offset = 0; offset < retryReceipts.length; offset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
|
|
777
|
-
const retryBatch = retryReceipts.slice(offset, offset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
|
|
778
|
-
await Promise.all(retryBatch.map(retryReceipt));
|
|
894
|
+
const successfulEvents = [];
|
|
895
|
+
const failedEvents = [];
|
|
896
|
+
const result = await this.dispatchBatchUntilAdmitted(batch, signal);
|
|
897
|
+
const recovery = await this.recoverFailedBatch(batch, result, signal);
|
|
898
|
+
if (recovery.interrupted)
|
|
899
|
+
return;
|
|
900
|
+
successfulEvents.push(...recovery.successfulEvents);
|
|
901
|
+
failedEvents.push(...recovery.failedEvents);
|
|
902
|
+
// Stop and drain the current heartbeat before changing failed receipts
|
|
903
|
+
// to their retry lease. Keep later partition receipts heartbeating.
|
|
904
|
+
await leaseHeartbeat.pause();
|
|
905
|
+
for (const event of failedEvents) {
|
|
906
|
+
for (const msg of receiptsByPath.get(event.relativePath) ?? []) {
|
|
907
|
+
leaseHeartbeat.remove(msg);
|
|
779
908
|
}
|
|
780
|
-
continue;
|
|
781
909
|
}
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
910
|
+
leaseHeartbeat.resume();
|
|
911
|
+
await this.retryFailedEvents(failedEvents, receiptsByPath, (count) => {
|
|
912
|
+
retriedCount += count;
|
|
913
|
+
});
|
|
914
|
+
// A slow lane failure leaves only its own events for redelivery; the
|
|
915
|
+
// successful lanes are acknowledged even when their original aggregate
|
|
916
|
+
// pull needed per-path recovery.
|
|
917
|
+
for (const event of successfulEvents) {
|
|
787
918
|
processedCount += 1;
|
|
788
919
|
for (const msg of receiptsByPath.get(event.relativePath) ?? []) {
|
|
789
920
|
await acknowledge(msg);
|
|
@@ -793,6 +924,7 @@ export class SqsPushReceiver {
|
|
|
793
924
|
}
|
|
794
925
|
finally {
|
|
795
926
|
completeIntake();
|
|
927
|
+
await leaseHeartbeat.stop();
|
|
796
928
|
if (filteredEvents.length > 0)
|
|
797
929
|
this.onFilteredEvents?.(filteredEvents);
|
|
798
930
|
this.logger.info({
|
|
@@ -801,6 +933,7 @@ export class SqsPushReceiver {
|
|
|
801
933
|
received: receivedCount,
|
|
802
934
|
coalesced: selected.size,
|
|
803
935
|
processed: processedCount,
|
|
936
|
+
skippedUnchanged: skippedUnchangedCount,
|
|
804
937
|
deleted: deletedCount,
|
|
805
938
|
released: releasedCount,
|
|
806
939
|
retried: retriedCount,
|
|
@@ -822,18 +955,250 @@ export class SqsPushReceiver {
|
|
|
822
955
|
}, "push receiver drain completed");
|
|
823
956
|
}
|
|
824
957
|
}
|
|
958
|
+
async dispatchBatchUntilAdmitted(events, signal) {
|
|
959
|
+
let result = await this.dispatchBatch(events);
|
|
960
|
+
let retryDelayMs = RECEIVER_ADMISSION_RETRY_INITIAL_MS;
|
|
961
|
+
while (!result.handled &&
|
|
962
|
+
isRealtimeAdmissionTimeout(result.error) &&
|
|
963
|
+
!signal.aborted &&
|
|
964
|
+
!this.disposing &&
|
|
965
|
+
!this.disposed) {
|
|
966
|
+
this.logger.info({
|
|
967
|
+
event: "receiver.sync.admission-timeout",
|
|
968
|
+
tenantId: this.tenantId,
|
|
969
|
+
pathCount: events.length,
|
|
970
|
+
retryDelayMs,
|
|
971
|
+
}, "push receiver retaining receipt lease while realtime admission is busy");
|
|
972
|
+
await this.sleep(retryDelayMs, signal);
|
|
973
|
+
if (signal.aborted || this.disposing || this.disposed)
|
|
974
|
+
return result;
|
|
975
|
+
result = await this.dispatchBatch(events);
|
|
976
|
+
retryDelayMs = Math.min(RECEIVER_ADMISSION_RETRY_MAX_MS, retryDelayMs * 2);
|
|
977
|
+
}
|
|
978
|
+
return result;
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* Isolate deterministic path failures by bisecting failed batches. A
|
|
982
|
+
* transport-class failure never fans out to singleton pulls; if both halves
|
|
983
|
+
* fail for transport reasons, the whole unresolved range returns to SQS.
|
|
984
|
+
*/
|
|
985
|
+
async recoverFailedBatch(events, initialResult, signal) {
|
|
986
|
+
const successfulEvents = [];
|
|
987
|
+
const failedEvents = [];
|
|
988
|
+
let interrupted = false;
|
|
989
|
+
const recover = async (batch, result) => {
|
|
990
|
+
if (interrupted)
|
|
991
|
+
return;
|
|
992
|
+
if (result.handled) {
|
|
993
|
+
successfulEvents.push(...batch);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
if (signal.aborted || this.disposing || this.disposed) {
|
|
997
|
+
interrupted = true;
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
if (batch.length === 1 || isTransportClassError(result.error)) {
|
|
1001
|
+
failedEvents.push(...batch);
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
const middle = Math.floor(batch.length / 2);
|
|
1005
|
+
const left = batch.slice(0, middle);
|
|
1006
|
+
const right = batch.slice(middle);
|
|
1007
|
+
const [leftResult, rightResult] = await Promise.all([
|
|
1008
|
+
this.dispatchBatchUntilAdmitted(left, signal),
|
|
1009
|
+
this.dispatchBatchUntilAdmitted(right, signal),
|
|
1010
|
+
]);
|
|
1011
|
+
if (signal.aborted || this.disposing || this.disposed) {
|
|
1012
|
+
interrupted = true;
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
if (!leftResult.handled &&
|
|
1016
|
+
!rightResult.handled &&
|
|
1017
|
+
isTransportClassError(leftResult.error) &&
|
|
1018
|
+
isTransportClassError(rightResult.error)) {
|
|
1019
|
+
failedEvents.push(...batch);
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
await recover(left, leftResult);
|
|
1023
|
+
await recover(right, rightResult);
|
|
1024
|
+
};
|
|
1025
|
+
await recover(events, initialResult);
|
|
1026
|
+
return { successfulEvents, failedEvents, interrupted };
|
|
1027
|
+
}
|
|
1028
|
+
async retryFailedEvents(events, receiptsByPath, onRetried) {
|
|
1029
|
+
const retryReceipts = events
|
|
1030
|
+
.flatMap((event) => receiptsByPath.get(event.relativePath) ?? [])
|
|
1031
|
+
.filter((message) => Boolean(message.ReceiptHandle));
|
|
1032
|
+
const updatedReceipts = await this.changeReceiptVisibilityBatches(retryReceipts, RECEIVER_RETRY_VISIBILITY_SECONDS, "receiver.retry.failed", "push receiver failed to shorten retry visibility");
|
|
1033
|
+
onRetried(updatedReceipts);
|
|
1034
|
+
}
|
|
1035
|
+
async changeReceiptVisibilityBatches(messages, visibilityTimeoutSeconds, failureEvent, failureDescription, onInvalidReceipt) {
|
|
1036
|
+
let updatedReceipts = 0;
|
|
1037
|
+
for (let waveOffset = 0; waveOffset < messages.length; waveOffset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
|
|
1038
|
+
const wave = messages.slice(waveOffset, waveOffset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
|
|
1039
|
+
const calls = [];
|
|
1040
|
+
for (let batchOffset = 0; batchOffset < wave.length; batchOffset += RECEIVER_VISIBILITY_BATCH_SIZE) {
|
|
1041
|
+
const batch = wave.slice(batchOffset, batchOffset + RECEIVER_VISIBILITY_BATCH_SIZE);
|
|
1042
|
+
const entries = batch.map((message, index) => ({
|
|
1043
|
+
id: `receipt-${index}`,
|
|
1044
|
+
receiptHandle: message.ReceiptHandle,
|
|
1045
|
+
}));
|
|
1046
|
+
const messageByEntryId = new Map(entries.map((entry, index) => [entry.id, batch[index]]));
|
|
1047
|
+
calls.push((async () => {
|
|
1048
|
+
try {
|
|
1049
|
+
const result = await this.sqs.changeMessageVisibilityBatch({
|
|
1050
|
+
queueUrl: this.queueUrl,
|
|
1051
|
+
entries,
|
|
1052
|
+
visibilityTimeoutSeconds,
|
|
1053
|
+
});
|
|
1054
|
+
const failuresById = new Map();
|
|
1055
|
+
for (const failure of result.failedEntries) {
|
|
1056
|
+
if (!failuresById.has(failure.id))
|
|
1057
|
+
failuresById.set(failure.id, failure);
|
|
1058
|
+
}
|
|
1059
|
+
for (const failure of failuresById.values()) {
|
|
1060
|
+
const message = messageByEntryId.get(failure.id);
|
|
1061
|
+
if (message) {
|
|
1062
|
+
if (failure.code === "ReceiptHandleIsInvalid") {
|
|
1063
|
+
onInvalidReceipt?.(message);
|
|
1064
|
+
}
|
|
1065
|
+
this.logVisibilityFailure(failureEvent, message, failureDescription, failure);
|
|
1066
|
+
}
|
|
1067
|
+
else {
|
|
1068
|
+
this.logger.warn({
|
|
1069
|
+
event: failureEvent,
|
|
1070
|
+
tenantId: this.tenantId,
|
|
1071
|
+
entryId: failure.id,
|
|
1072
|
+
...(failure.code ? { code: failure.code } : {}),
|
|
1073
|
+
...(failure.message ? { failureMessage: failure.message } : {}),
|
|
1074
|
+
...(failure.senderFault === undefined
|
|
1075
|
+
? {}
|
|
1076
|
+
: { senderFault: failure.senderFault }),
|
|
1077
|
+
failureReason: "SQS returned a failed visibility entry id not in the request",
|
|
1078
|
+
}, failureDescription);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
const knownFailedCount = [...failuresById.keys()].filter((id) => messageByEntryId.has(id)).length;
|
|
1082
|
+
// Entries absent from SQS's Failed list succeeded. Per-entry
|
|
1083
|
+
// failures remain on the original lease and are excluded from the
|
|
1084
|
+
// retried count; retrying them one by one could amplify a service
|
|
1085
|
+
// or throttling failure into a long serial tail.
|
|
1086
|
+
return batch.length - knownFailedCount;
|
|
1087
|
+
}
|
|
1088
|
+
catch (error) {
|
|
1089
|
+
for (const message of batch) {
|
|
1090
|
+
this.logVisibilityFailure(failureEvent, message, failureDescription, undefined, error);
|
|
1091
|
+
}
|
|
1092
|
+
return 0;
|
|
1093
|
+
}
|
|
1094
|
+
})());
|
|
1095
|
+
}
|
|
1096
|
+
const waveResults = await Promise.all(calls);
|
|
1097
|
+
updatedReceipts += waveResults.reduce((count, updated) => count + updated, 0);
|
|
1098
|
+
}
|
|
1099
|
+
return updatedReceipts;
|
|
1100
|
+
}
|
|
1101
|
+
logVisibilityFailure(event, message, description, failure, error) {
|
|
1102
|
+
const errno = error;
|
|
1103
|
+
this.logger.warn({
|
|
1104
|
+
event,
|
|
1105
|
+
tenantId: this.tenantId,
|
|
1106
|
+
...(message.MessageId === undefined ? {} : { messageId: message.MessageId }),
|
|
1107
|
+
...(failure?.id === undefined ? {} : { entryId: failure.id }),
|
|
1108
|
+
...(failure?.code === undefined ? {} : { code: failure.code }),
|
|
1109
|
+
...(failure?.message === undefined ? {} : { failureMessage: failure.message }),
|
|
1110
|
+
...(failure?.senderFault === undefined
|
|
1111
|
+
? {}
|
|
1112
|
+
: { senderFault: failure.senderFault }),
|
|
1113
|
+
...(error === undefined
|
|
1114
|
+
? {}
|
|
1115
|
+
: {
|
|
1116
|
+
err: {
|
|
1117
|
+
message: errno?.message ?? String(error),
|
|
1118
|
+
...(errno?.code === undefined ? {} : { code: errno.code }),
|
|
1119
|
+
},
|
|
1120
|
+
}),
|
|
1121
|
+
}, description);
|
|
1122
|
+
}
|
|
1123
|
+
/**
|
|
1124
|
+
* Keep SQS receipts valid while a targeted sync operation is still running.
|
|
1125
|
+
* A pull may outlive the receive lease; extending the selected receipts
|
|
1126
|
+
* prevents the queue from issuing a new receipt handle before ack/retry.
|
|
1127
|
+
*/
|
|
1128
|
+
startReceiptLeaseHeartbeat() {
|
|
1129
|
+
const receipts = new Set();
|
|
1130
|
+
let stopped = false;
|
|
1131
|
+
let paused = false;
|
|
1132
|
+
let inFlight = null;
|
|
1133
|
+
let timer = null;
|
|
1134
|
+
const heartbeat = () => {
|
|
1135
|
+
if (stopped || paused || inFlight !== null)
|
|
1136
|
+
return;
|
|
1137
|
+
const activeReceipts = [...receipts].filter((message) => Boolean(message.ReceiptHandle));
|
|
1138
|
+
if (activeReceipts.length === 0)
|
|
1139
|
+
return;
|
|
1140
|
+
inFlight = this.changeReceiptVisibilityBatches(activeReceipts, this.visibilityTimeoutSeconds, "receiver.lease.extend.failed", "push receiver failed to extend an in-flight receipt lease", (message) => receipts.delete(message)).then(() => undefined).finally(() => {
|
|
1141
|
+
inFlight = null;
|
|
1142
|
+
});
|
|
1143
|
+
};
|
|
1144
|
+
const clearTimer = () => {
|
|
1145
|
+
if (timer === null)
|
|
1146
|
+
return;
|
|
1147
|
+
clearInterval(timer);
|
|
1148
|
+
timer = null;
|
|
1149
|
+
};
|
|
1150
|
+
const armTimer = () => {
|
|
1151
|
+
if (stopped || paused || timer !== null)
|
|
1152
|
+
return;
|
|
1153
|
+
timer = setInterval(heartbeat, this.visibilityHeartbeatIntervalMs);
|
|
1154
|
+
timer.unref?.();
|
|
1155
|
+
};
|
|
1156
|
+
armTimer();
|
|
1157
|
+
return {
|
|
1158
|
+
add: (message) => {
|
|
1159
|
+
if (!message.ReceiptHandle)
|
|
1160
|
+
return;
|
|
1161
|
+
receipts.add(message);
|
|
1162
|
+
},
|
|
1163
|
+
remove: (message) => {
|
|
1164
|
+
receipts.delete(message);
|
|
1165
|
+
},
|
|
1166
|
+
pause: async () => {
|
|
1167
|
+
if (stopped)
|
|
1168
|
+
return;
|
|
1169
|
+
paused = true;
|
|
1170
|
+
clearTimer();
|
|
1171
|
+
await inFlight;
|
|
1172
|
+
},
|
|
1173
|
+
resume: () => {
|
|
1174
|
+
if (stopped)
|
|
1175
|
+
return;
|
|
1176
|
+
paused = false;
|
|
1177
|
+
armTimer();
|
|
1178
|
+
},
|
|
1179
|
+
stop: async () => {
|
|
1180
|
+
if (stopped)
|
|
1181
|
+
return;
|
|
1182
|
+
stopped = true;
|
|
1183
|
+
paused = true;
|
|
1184
|
+
clearTimer();
|
|
1185
|
+
await inFlight;
|
|
1186
|
+
},
|
|
1187
|
+
};
|
|
1188
|
+
}
|
|
825
1189
|
/**
|
|
826
1190
|
* Invoke one coalesced sync pass. The dedupe high-water marks advance only
|
|
827
1191
|
* after it completes, preserving redelivery after any pass failure.
|
|
828
1192
|
*/
|
|
829
1193
|
async dispatchBatch(events) {
|
|
830
1194
|
if (this.disposing || this.disposed)
|
|
831
|
-
return false;
|
|
1195
|
+
return { handled: false };
|
|
832
1196
|
const controller = new AbortController();
|
|
833
1197
|
this.inFlightAborts.add(controller);
|
|
834
1198
|
const ctx = { events, signal: controller.signal };
|
|
835
1199
|
const startMs = this.now();
|
|
836
1200
|
let handled = false;
|
|
1201
|
+
let failure;
|
|
837
1202
|
const p = (async () => {
|
|
838
1203
|
try {
|
|
839
1204
|
this.logger.debug({ event: "receiver.sync.start", tenantId: this.tenantId, pathCount: events.length }, "push receiver invoking coalesced sync engine");
|
|
@@ -886,6 +1251,7 @@ export class SqsPushReceiver {
|
|
|
886
1251
|
// crash the receiver loop. Leaving the SQS message undeleted lets
|
|
887
1252
|
// redelivery retry the failed targeted pull.
|
|
888
1253
|
const e = err;
|
|
1254
|
+
failure = err;
|
|
889
1255
|
this.logger.error({
|
|
890
1256
|
event: "receiver.sync.failed",
|
|
891
1257
|
tenantId: this.tenantId,
|
|
@@ -898,7 +1264,7 @@ export class SqsPushReceiver {
|
|
|
898
1264
|
}
|
|
899
1265
|
})();
|
|
900
1266
|
await p;
|
|
901
|
-
return handled;
|
|
1267
|
+
return { handled, ...(failure === undefined ? {} : { error: failure }) };
|
|
902
1268
|
}
|
|
903
1269
|
/** Delete a message, swallowing transport errors (redelivery is harmless). */
|
|
904
1270
|
async safeDelete(msg) {
|