@indigoai-us/hq-cloud 6.18.10 → 6.18.11-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/dist/bin/sync-runner-args.test.d.ts +2 -0
  2. package/dist/bin/sync-runner-args.test.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-args.test.js +82 -0
  4. package/dist/bin/sync-runner-args.test.js.map +1 -0
  5. package/dist/bin/sync-runner-company.d.ts +1 -0
  6. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-company.js +3 -0
  8. package/dist/bin/sync-runner-company.js.map +1 -1
  9. package/dist/bin/sync-runner-company.test.js +31 -0
  10. package/dist/bin/sync-runner-company.test.js.map +1 -1
  11. package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts +2 -0
  12. package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts.map +1 -0
  13. package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js +366 -0
  14. package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js.map +1 -0
  15. package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts +2 -0
  16. package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts.map +1 -0
  17. package/dist/bin/sync-runner-delete-snapshot.bench.test.js +317 -0
  18. package/dist/bin/sync-runner-delete-snapshot.bench.test.js.map +1 -0
  19. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  20. package/dist/bin/sync-runner-watch-loop.js +191 -12
  21. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  22. package/dist/bin/sync-runner-watch-loop.test.js +71 -0
  23. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  24. package/dist/bin/sync-runner-watch-routes.d.ts +2 -1
  25. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  26. package/dist/bin/sync-runner-watch-routes.js +14 -3
  27. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  28. package/dist/bin/sync-runner-watch-routes.test.js +15 -0
  29. package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
  30. package/dist/bin/sync-runner.d.ts +69 -1
  31. package/dist/bin/sync-runner.d.ts.map +1 -1
  32. package/dist/bin/sync-runner.js +52 -1
  33. package/dist/bin/sync-runner.js.map +1 -1
  34. package/dist/bin/sync-runner.test.js +119 -3
  35. package/dist/bin/sync-runner.test.js.map +1 -1
  36. package/dist/cli/core-source-stamp.d.ts +16 -0
  37. package/dist/cli/core-source-stamp.d.ts.map +1 -0
  38. package/dist/cli/core-source-stamp.js +146 -0
  39. package/dist/cli/core-source-stamp.js.map +1 -0
  40. package/dist/cli/rescue-core-stamp.test.d.ts +2 -0
  41. package/dist/cli/rescue-core-stamp.test.d.ts.map +1 -0
  42. package/dist/cli/rescue-core-stamp.test.js +364 -0
  43. package/dist/cli/rescue-core-stamp.test.js.map +1 -0
  44. package/dist/cli/rescue-core.d.ts +3 -0
  45. package/dist/cli/rescue-core.d.ts.map +1 -1
  46. package/dist/cli/rescue-core.js +36 -72
  47. package/dist/cli/rescue-core.js.map +1 -1
  48. package/dist/cli/share.d.ts.map +1 -1
  49. package/dist/cli/share.js +2 -0
  50. package/dist/cli/share.js.map +1 -1
  51. package/dist/cli/share.test.js +76 -0
  52. package/dist/cli/share.test.js.map +1 -1
  53. package/dist/cli/sync.d.ts +13 -2
  54. package/dist/cli/sync.d.ts.map +1 -1
  55. package/dist/cli/sync.js +81 -4
  56. package/dist/cli/sync.js.map +1 -1
  57. package/dist/cli/sync.test.js +228 -0
  58. package/dist/cli/sync.test.js.map +1 -1
  59. package/dist/ignore.d.ts.map +1 -1
  60. package/dist/ignore.js +3 -0
  61. package/dist/ignore.js.map +1 -1
  62. package/dist/ignore.test.js +8 -0
  63. package/dist/ignore.test.js.map +1 -1
  64. package/dist/journal.d.ts +15 -0
  65. package/dist/journal.d.ts.map +1 -1
  66. package/dist/journal.js +66 -2
  67. package/dist/journal.js.map +1 -1
  68. package/dist/journal.test.js +16 -1
  69. package/dist/journal.test.js.map +1 -1
  70. package/dist/manifest/build-manifest-resident-parity.test.js +5 -3
  71. package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
  72. package/dist/sync/content-hash.d.ts +4 -0
  73. package/dist/sync/content-hash.d.ts.map +1 -0
  74. package/dist/sync/content-hash.js +11 -0
  75. package/dist/sync/content-hash.js.map +1 -0
  76. package/dist/sync/content-hash.test.d.ts +2 -0
  77. package/dist/sync/content-hash.test.d.ts.map +1 -0
  78. package/dist/sync/content-hash.test.js +16 -0
  79. package/dist/sync/content-hash.test.js.map +1 -0
  80. package/dist/sync/event-sync.d.ts.map +1 -1
  81. package/dist/sync/event-sync.js +58 -1
  82. package/dist/sync/event-sync.js.map +1 -1
  83. package/dist/sync/event-sync.test.js +46 -0
  84. package/dist/sync/event-sync.test.js.map +1 -1
  85. package/dist/sync/logger.test.js +3 -0
  86. package/dist/sync/logger.test.js.map +1 -1
  87. package/dist/sync/metrics.test.js +3 -0
  88. package/dist/sync/metrics.test.js.map +1 -1
  89. package/dist/sync/push-receiver.d.ts +52 -0
  90. package/dist/sync/push-receiver.d.ts.map +1 -1
  91. package/dist/sync/push-receiver.js +416 -50
  92. package/dist/sync/push-receiver.js.map +1 -1
  93. package/dist/sync/push-receiver.test.js +673 -40
  94. package/dist/sync/push-receiver.test.js.map +1 -1
  95. package/dist/sync/unchanged-push-events.d.ts +16 -0
  96. package/dist/sync/unchanged-push-events.d.ts.map +1 -0
  97. package/dist/sync/unchanged-push-events.js +51 -0
  98. package/dist/sync/unchanged-push-events.js.map +1 -0
  99. package/dist/sync/unchanged-push-events.test.d.ts +2 -0
  100. package/dist/sync/unchanged-push-events.test.d.ts.map +1 -0
  101. package/dist/sync/unchanged-push-events.test.js +76 -0
  102. package/dist/sync/unchanged-push-events.test.js.map +1 -0
  103. package/dist/watcher.d.ts +18 -0
  104. package/dist/watcher.d.ts.map +1 -1
  105. package/dist/watcher.js +78 -13
  106. package/dist/watcher.js.map +1 -1
  107. package/dist/watcher.test.js +72 -0
  108. package/dist/watcher.test.js.map +1 -1
  109. package/package.json +2 -1
@@ -115,8 +115,14 @@ export const DEFAULT_MAX_MESSAGES = 10;
115
115
  export const DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS = 600;
116
116
  /** Retry a failed targeted pull promptly while preserving queue ownership. */
117
117
  export const RECEIVER_RETRY_VISIBILITY_SECONDS = 20;
118
+ /** SQS ChangeMessageVisibilityBatch accepts at most ten receipt entries. */
119
+ const RECEIVER_VISIBILITY_BATCH_SIZE = 10;
118
120
  /** Bound the retry fanout while resetting the default 1,000-receipt drain promptly. */
119
121
  const RECEIVER_RETRY_VISIBILITY_CONCURRENCY = 256;
122
+ const RECEIVER_SKIP_ACK_CONCURRENCY = 64;
123
+ /** Keep an admitted realtime pull local when a full chore temporarily denies admission. */
124
+ const RECEIVER_ADMISSION_RETRY_INITIAL_MS = 5_000;
125
+ const RECEIVER_ADMISSION_RETRY_MAX_MS = 60_000;
120
126
  /** Reconnect backoff defaults. */
121
127
  export const DEFAULT_RECONNECT_INITIAL_MS = 250;
122
128
  export const DEFAULT_RECONNECT_MAX_MS = 30_000;
@@ -170,6 +176,86 @@ class BoundedSequenceDedupe {
170
176
  }
171
177
  }
172
178
  }
179
+ /**
180
+ * A production pull bridge can carry the runner's failure reason and class
181
+ * across its exit-code boundary so the receiver does not split a transport
182
+ * failure into per-path retries.
183
+ */
184
+ export class ReceiverDispatchError extends Error {
185
+ failureClass;
186
+ failureReason;
187
+ constructor(failureClass, failureReason) {
188
+ super(failureReason);
189
+ this.failureClass = failureClass;
190
+ this.failureReason = failureReason;
191
+ this.name = "ReceiverDispatchError";
192
+ }
193
+ }
194
+ function isRealtimeAdmissionTimeout(error) {
195
+ if (typeof error !== "object" || error === null)
196
+ return false;
197
+ const candidate = error;
198
+ return candidate.name === "RealtimeAdmissionTimeout" ||
199
+ (typeof candidate.message === "string" &&
200
+ /^realtime admission timeout\b/i.test(candidate.message));
201
+ }
202
+ const TRANSPORT_FAILURE_CODES = new Set([
203
+ "ABORT_ERR",
204
+ "ACCESSDENIED",
205
+ "ACCESSDENIEDEXCEPTION",
206
+ "CREDENTIALSEXPIRED",
207
+ "ECONNABORTED",
208
+ "ECONNREFUSED",
209
+ "ECONNRESET",
210
+ "EDQUOT",
211
+ "EHOSTUNREACH",
212
+ "EAI_AGAIN",
213
+ "ENETDOWN",
214
+ "ENETUNREACH",
215
+ "ENOSPC",
216
+ "ENOTFOUND",
217
+ "ETIMEDOUT",
218
+ "EXPIREDTOKEN",
219
+ "EXPIREDTOKENEXCEPTION",
220
+ "FORBIDDEN",
221
+ "INVALIDCLIENTTOKENID",
222
+ "REQUESTEXPIRED",
223
+ "TOKENEXPIRED",
224
+ "UNAUTHORIZED",
225
+ "UND_ERR_CONNECT_TIMEOUT",
226
+ "UND_ERR_SOCKET",
227
+ ]);
228
+ const TRANSPORT_FAILURE_MESSAGE = /(?:\b(?:ECONNABORTED|ECONNREFUSED|ECONNRESET|EDQUOT|EHOSTUNREACH|EAI_AGAIN|ENETDOWN|ENETUNREACH|ENOSPC|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_SOCKET)\b|no space left on device|disk full|network (?:error|failure|unavailable)|socket (?:hang up|closed)|connection (?:reset|timed? out|refused)|\babort(?:ed|ing)?\b|\btimed? ?out\b|\b(?:HTTP\s*)?(?:401|403|408|429|5\d\d)\b|\b(?:forbidden|unauthorized)\b|(?:access|credential|token).*(?:expired|expiry|invalid|forbidden|unauthorized))/i;
229
+ function isTransportClassError(error, visited = new Set()) {
230
+ if (typeof error === "string")
231
+ return TRANSPORT_FAILURE_MESSAGE.test(error);
232
+ if (typeof error !== "object" || error === null || visited.has(error))
233
+ return false;
234
+ visited.add(error);
235
+ if (error instanceof ReceiverDispatchError) {
236
+ return error.failureClass === "transport";
237
+ }
238
+ const candidate = error;
239
+ const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : "";
240
+ if (TRANSPORT_FAILURE_CODES.has(code) ||
241
+ /^(?:HTTP)?(?:401|403|408|429|5\d\d)$/.test(code) ||
242
+ (typeof candidate.name === "string" && /^(?:AbortError|TimeoutError|ExpiredTokenException)$/i.test(candidate.name))) {
243
+ return true;
244
+ }
245
+ const status = Number(candidate.status ?? candidate.statusCode ?? candidate.$metadata?.httpStatusCode);
246
+ if ([401, 403, 408, 429].includes(status) || status >= 500)
247
+ return true;
248
+ if (typeof candidate.message === "string" &&
249
+ TRANSPORT_FAILURE_MESSAGE.test(candidate.message)) {
250
+ return true;
251
+ }
252
+ return candidate.cause === undefined
253
+ ? false
254
+ : isTransportClassError(candidate.cause, visited);
255
+ }
256
+ export function isReceiverTransportFailure(error) {
257
+ return isTransportClassError(error);
258
+ }
173
259
  const NOOP_LOGGER = {
174
260
  info: () => undefined,
175
261
  warn: () => undefined,
@@ -311,6 +397,7 @@ export class SqsPushReceiver {
311
397
  /** Legacy per-event seam, retained to preserve the old embedding contract. */
312
398
  syncFn;
313
399
  shouldProcess;
400
+ skipUnchangedFn;
314
401
  onFilteredEvents;
315
402
  onReceiveActivity;
316
403
  logger;
@@ -318,6 +405,7 @@ export class SqsPushReceiver {
318
405
  waitTimeSeconds;
319
406
  maxMessages;
320
407
  visibilityTimeoutSeconds;
408
+ visibilityHeartbeatIntervalMs;
321
409
  disposeDrainMs;
322
410
  reconnectInitialMs;
323
411
  reconnectMaxMs;
@@ -369,6 +457,7 @@ export class SqsPushReceiver {
369
457
  throw new Error("SqsPushReceiver: syncBatchFn is required");
370
458
  }
371
459
  this.shouldProcess = opts.shouldProcess ?? (() => true);
460
+ this.skipUnchangedFn = opts.skipUnchangedFn;
372
461
  this.onFilteredEvents = opts.onFilteredEvents;
373
462
  this.onReceiveActivity = opts.onReceiveActivity;
374
463
  this.logger = opts.logger ?? NOOP_LOGGER;
@@ -376,6 +465,7 @@ export class SqsPushReceiver {
376
465
  this.waitTimeSeconds = opts.waitTimeSeconds ?? DEFAULT_WAIT_TIME_SECONDS;
377
466
  this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
378
467
  this.visibilityTimeoutSeconds = Math.max(1, Math.min(43_200, Math.floor(opts.visibilityTimeoutSeconds ?? DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS)));
468
+ this.visibilityHeartbeatIntervalMs = Math.max(1, Math.min(2_147_483_647, Math.floor(opts.visibilityHeartbeatIntervalMs ?? this.visibilityTimeoutSeconds * 500)));
379
469
  this.disposeDrainMs =
380
470
  opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
381
471
  this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
@@ -492,7 +582,7 @@ export class SqsPushReceiver {
492
582
  try {
493
583
  received = await this.sqs.receiveMessage({
494
584
  queueUrl: this.queueUrl,
495
- maxMessages: this.maxMessages,
585
+ maxMessages: Math.min(this.maxMessages, this.batchMaxPaths, this.drainMaxMessages),
496
586
  waitTimeSeconds: this.waitTimeSeconds,
497
587
  visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
498
588
  signal,
@@ -578,6 +668,7 @@ export class SqsPushReceiver {
578
668
  let releasedCount = 0;
579
669
  let retriedCount = 0;
580
670
  let processedCount = 0;
671
+ let skippedUnchangedCount = 0;
581
672
  let redeliveredCount = 0;
582
673
  let maxReceiveCount = 0;
583
674
  let oldestSentTimestampMs;
@@ -589,15 +680,32 @@ export class SqsPushReceiver {
589
680
  intakeCompleted = true;
590
681
  onIntakeComplete();
591
682
  };
683
+ const leaseHeartbeat = this.startReceiptLeaseHeartbeat();
592
684
  const acknowledge = async (msg) => {
593
- if (await this.safeDelete(msg)) {
594
- deletedCount += 1;
685
+ let heartbeatPaused = false;
686
+ try {
687
+ if (await this.safeDelete(msg)) {
688
+ deletedCount += 1;
689
+ return true;
690
+ }
691
+ // Drain any extension already in flight and remove this receipt from
692
+ // future snapshots before releasing it. Otherwise a late heartbeat
693
+ // can restore the full lease after the zero-visibility retry release.
694
+ await leaseHeartbeat.pause();
695
+ heartbeatPaused = true;
696
+ leaseHeartbeat.remove(msg);
697
+ if (await this.releaseUnselected(msg)) {
698
+ // A failed delete is not allowed to strand a stale receipt until
699
+ // its normal lease expires. Releasing returns retry ownership to
700
+ // SQS immediately while successful receipts remain heartbeatable.
701
+ releasedCount += 1;
702
+ }
703
+ return false;
595
704
  }
596
- else if (await this.releaseUnselected(msg)) {
597
- // A failed delete is not allowed to strand a stale receipt until its
598
- // normal lease expires. Releasing preserves delete-after-success while
599
- // returning retry ownership to SQS immediately.
600
- releasedCount += 1;
705
+ finally {
706
+ leaseHeartbeat.remove(msg);
707
+ if (heartbeatPaused)
708
+ leaseHeartbeat.resume();
601
709
  }
602
710
  };
603
711
  const countOrigin = (originDeviceId) => {
@@ -671,6 +779,7 @@ export class SqsPushReceiver {
671
779
  const receipts = receiptsByPath.get(event.relativePath) ?? [];
672
780
  receipts.push(msg);
673
781
  receiptsByPath.set(event.relativePath, receipts);
782
+ leaseHeartbeat.add(msg);
674
783
  if (!selectedEvent || event.sequenceNumber > selectedEvent.sequenceNumber) {
675
784
  selected.set(event.relativePath, event);
676
785
  }
@@ -693,6 +802,7 @@ export class SqsPushReceiver {
693
802
  let emptyConfirmations = 0;
694
803
  let waitTimeSeconds = 0;
695
804
  while (!saturated &&
805
+ selected.size < this.batchMaxPaths &&
696
806
  receivedCount < this.drainMaxMessages &&
697
807
  !signal.aborted &&
698
808
  this.now() < fillDeadline) {
@@ -706,7 +816,7 @@ export class SqsPushReceiver {
706
816
  try {
707
817
  next = await this.sqs.receiveMessage({
708
818
  queueUrl: this.queueUrl,
709
- maxMessages: Math.min(this.maxMessages, this.drainMaxMessages - receivedCount),
819
+ maxMessages: Math.min(this.maxMessages, this.batchMaxPaths - selected.size, this.drainMaxMessages - receivedCount),
710
820
  waitTimeSeconds,
711
821
  visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
712
822
  signal,
@@ -740,50 +850,71 @@ export class SqsPushReceiver {
740
850
  }
741
851
  completeIntake();
742
852
  const events = [...selected.values()];
743
- const batches = this.syncBatchFn.partition?.(events) ?? [events];
853
+ let eventsToDispatch = events;
854
+ if (this.skipUnchangedFn && events.length > 0) {
855
+ let unchangedPaths = new Set();
856
+ try {
857
+ unchangedPaths = await this.skipUnchangedFn(events);
858
+ }
859
+ catch (err) {
860
+ // A journal/stat read is an optimization only. Preserve delivery by
861
+ // falling back to the ordinary targeted pull when the proof fails.
862
+ this.logger.warn({
863
+ event: "receiver.unchanged.check.failed",
864
+ tenantId: this.tenantId,
865
+ pathCount: events.length,
866
+ err: { message: err?.message },
867
+ }, "push receiver could not prove unchanged local content; dispatching pull");
868
+ }
869
+ const unchangedEvents = [];
870
+ eventsToDispatch = events.filter((event) => {
871
+ if (!unchangedPaths.has(event.relativePath))
872
+ return true;
873
+ unchangedEvents.push(event);
874
+ return false;
875
+ });
876
+ skippedUnchangedCount += unchangedEvents.length;
877
+ for (let offset = 0; offset < unchangedEvents.length; offset += RECEIVER_SKIP_ACK_CONCURRENCY) {
878
+ const acknowledgementBatch = unchangedEvents.slice(offset, offset + RECEIVER_SKIP_ACK_CONCURRENCY);
879
+ await Promise.all(acknowledgementBatch.map(async (event) => {
880
+ const receipts = receiptsByPath.get(event.relativePath) ?? [];
881
+ const acknowledgements = await Promise.all(receipts.map(acknowledge));
882
+ if (acknowledgements.every(Boolean)) {
883
+ this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
884
+ }
885
+ }));
886
+ }
887
+ }
888
+ const batches = eventsToDispatch.length === 0
889
+ ? []
890
+ : this.syncBatchFn.partition?.(eventsToDispatch) ?? [eventsToDispatch];
744
891
  for (const batch of batches) {
745
892
  if (batch.length === 0)
746
893
  continue;
747
- const handled = await this.dispatchBatch(batch);
748
- if (!handled) {
749
- // Disposal still leaves the current lease untouched. A failed pull
750
- // gets a short visibility delay so this receipt can retry promptly;
751
- // an SQS visibility failure is contained to this receipt and falls
752
- // back to the queue's normal timeout.
753
- if (this.disposing || this.disposed)
754
- return;
755
- const retryReceipts = batch.flatMap((event) => receiptsByPath.get(event.relativePath) ?? []);
756
- const retryReceipt = async (msg) => {
757
- if (!msg.ReceiptHandle)
758
- return;
759
- retriedCount += 1;
760
- try {
761
- await this.sqs.changeMessageVisibility({
762
- queueUrl: this.queueUrl,
763
- receiptHandle: msg.ReceiptHandle,
764
- visibilityTimeoutSeconds: RECEIVER_RETRY_VISIBILITY_SECONDS,
765
- });
766
- }
767
- catch (err) {
768
- this.logger.warn({
769
- event: "receiver.retry.failed",
770
- tenantId: this.tenantId,
771
- messageId: msg.MessageId,
772
- err: { message: err?.message },
773
- }, "push receiver failed to shorten retry visibility");
774
- }
775
- };
776
- for (let offset = 0; offset < retryReceipts.length; offset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
777
- const retryBatch = retryReceipts.slice(offset, offset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
778
- await Promise.all(retryBatch.map(retryReceipt));
894
+ const successfulEvents = [];
895
+ const failedEvents = [];
896
+ const result = await this.dispatchBatchUntilAdmitted(batch, signal);
897
+ const recovery = await this.recoverFailedBatch(batch, result, signal);
898
+ if (recovery.interrupted)
899
+ return;
900
+ successfulEvents.push(...recovery.successfulEvents);
901
+ failedEvents.push(...recovery.failedEvents);
902
+ // Stop and drain the current heartbeat before changing failed receipts
903
+ // to their retry lease. Keep later partition receipts heartbeating.
904
+ await leaseHeartbeat.pause();
905
+ for (const event of failedEvents) {
906
+ for (const msg of receiptsByPath.get(event.relativePath) ?? []) {
907
+ leaseHeartbeat.remove(msg);
779
908
  }
780
- continue;
781
909
  }
782
- // Receipt acknowledgement follows the successful lane, not the original
783
- // aggregate delivery. A slow-lane failure therefore leaves only its own
784
- // events available for redelivery while dedupe absorbs the completed fast
785
- // lane if a transport delete races or fails.
786
- for (const event of batch) {
910
+ leaseHeartbeat.resume();
911
+ await this.retryFailedEvents(failedEvents, receiptsByPath, (count) => {
912
+ retriedCount += count;
913
+ });
914
+ // A slow lane failure leaves only its own events for redelivery; the
915
+ // successful lanes are acknowledged even when their original aggregate
916
+ // pull needed per-path recovery.
917
+ for (const event of successfulEvents) {
787
918
  processedCount += 1;
788
919
  for (const msg of receiptsByPath.get(event.relativePath) ?? []) {
789
920
  await acknowledge(msg);
@@ -793,6 +924,7 @@ export class SqsPushReceiver {
793
924
  }
794
925
  finally {
795
926
  completeIntake();
927
+ await leaseHeartbeat.stop();
796
928
  if (filteredEvents.length > 0)
797
929
  this.onFilteredEvents?.(filteredEvents);
798
930
  this.logger.info({
@@ -801,6 +933,7 @@ export class SqsPushReceiver {
801
933
  received: receivedCount,
802
934
  coalesced: selected.size,
803
935
  processed: processedCount,
936
+ skippedUnchanged: skippedUnchangedCount,
804
937
  deleted: deletedCount,
805
938
  released: releasedCount,
806
939
  retried: retriedCount,
@@ -822,18 +955,250 @@ export class SqsPushReceiver {
822
955
  }, "push receiver drain completed");
823
956
  }
824
957
  }
958
+ async dispatchBatchUntilAdmitted(events, signal) {
959
+ let result = await this.dispatchBatch(events);
960
+ let retryDelayMs = RECEIVER_ADMISSION_RETRY_INITIAL_MS;
961
+ while (!result.handled &&
962
+ isRealtimeAdmissionTimeout(result.error) &&
963
+ !signal.aborted &&
964
+ !this.disposing &&
965
+ !this.disposed) {
966
+ this.logger.info({
967
+ event: "receiver.sync.admission-timeout",
968
+ tenantId: this.tenantId,
969
+ pathCount: events.length,
970
+ retryDelayMs,
971
+ }, "push receiver retaining receipt lease while realtime admission is busy");
972
+ await this.sleep(retryDelayMs, signal);
973
+ if (signal.aborted || this.disposing || this.disposed)
974
+ return result;
975
+ result = await this.dispatchBatch(events);
976
+ retryDelayMs = Math.min(RECEIVER_ADMISSION_RETRY_MAX_MS, retryDelayMs * 2);
977
+ }
978
+ return result;
979
+ }
980
+ /**
981
+ * Isolate deterministic path failures by bisecting failed batches. A
982
+ * transport-class failure never fans out to singleton pulls; if both halves
983
+ * fail for transport reasons, the whole unresolved range returns to SQS.
984
+ */
985
+ async recoverFailedBatch(events, initialResult, signal) {
986
+ const successfulEvents = [];
987
+ const failedEvents = [];
988
+ let interrupted = false;
989
+ const recover = async (batch, result) => {
990
+ if (interrupted)
991
+ return;
992
+ if (result.handled) {
993
+ successfulEvents.push(...batch);
994
+ return;
995
+ }
996
+ if (signal.aborted || this.disposing || this.disposed) {
997
+ interrupted = true;
998
+ return;
999
+ }
1000
+ if (batch.length === 1 || isTransportClassError(result.error)) {
1001
+ failedEvents.push(...batch);
1002
+ return;
1003
+ }
1004
+ const middle = Math.floor(batch.length / 2);
1005
+ const left = batch.slice(0, middle);
1006
+ const right = batch.slice(middle);
1007
+ const [leftResult, rightResult] = await Promise.all([
1008
+ this.dispatchBatchUntilAdmitted(left, signal),
1009
+ this.dispatchBatchUntilAdmitted(right, signal),
1010
+ ]);
1011
+ if (signal.aborted || this.disposing || this.disposed) {
1012
+ interrupted = true;
1013
+ return;
1014
+ }
1015
+ if (!leftResult.handled &&
1016
+ !rightResult.handled &&
1017
+ isTransportClassError(leftResult.error) &&
1018
+ isTransportClassError(rightResult.error)) {
1019
+ failedEvents.push(...batch);
1020
+ return;
1021
+ }
1022
+ await recover(left, leftResult);
1023
+ await recover(right, rightResult);
1024
+ };
1025
+ await recover(events, initialResult);
1026
+ return { successfulEvents, failedEvents, interrupted };
1027
+ }
1028
+ async retryFailedEvents(events, receiptsByPath, onRetried) {
1029
+ const retryReceipts = events
1030
+ .flatMap((event) => receiptsByPath.get(event.relativePath) ?? [])
1031
+ .filter((message) => Boolean(message.ReceiptHandle));
1032
+ const updatedReceipts = await this.changeReceiptVisibilityBatches(retryReceipts, RECEIVER_RETRY_VISIBILITY_SECONDS, "receiver.retry.failed", "push receiver failed to shorten retry visibility");
1033
+ onRetried(updatedReceipts);
1034
+ }
1035
+ async changeReceiptVisibilityBatches(messages, visibilityTimeoutSeconds, failureEvent, failureDescription, onInvalidReceipt) {
1036
+ let updatedReceipts = 0;
1037
+ for (let waveOffset = 0; waveOffset < messages.length; waveOffset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
1038
+ const wave = messages.slice(waveOffset, waveOffset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
1039
+ const calls = [];
1040
+ for (let batchOffset = 0; batchOffset < wave.length; batchOffset += RECEIVER_VISIBILITY_BATCH_SIZE) {
1041
+ const batch = wave.slice(batchOffset, batchOffset + RECEIVER_VISIBILITY_BATCH_SIZE);
1042
+ const entries = batch.map((message, index) => ({
1043
+ id: `receipt-${index}`,
1044
+ receiptHandle: message.ReceiptHandle,
1045
+ }));
1046
+ const messageByEntryId = new Map(entries.map((entry, index) => [entry.id, batch[index]]));
1047
+ calls.push((async () => {
1048
+ try {
1049
+ const result = await this.sqs.changeMessageVisibilityBatch({
1050
+ queueUrl: this.queueUrl,
1051
+ entries,
1052
+ visibilityTimeoutSeconds,
1053
+ });
1054
+ const failuresById = new Map();
1055
+ for (const failure of result.failedEntries) {
1056
+ if (!failuresById.has(failure.id))
1057
+ failuresById.set(failure.id, failure);
1058
+ }
1059
+ for (const failure of failuresById.values()) {
1060
+ const message = messageByEntryId.get(failure.id);
1061
+ if (message) {
1062
+ if (failure.code === "ReceiptHandleIsInvalid") {
1063
+ onInvalidReceipt?.(message);
1064
+ }
1065
+ this.logVisibilityFailure(failureEvent, message, failureDescription, failure);
1066
+ }
1067
+ else {
1068
+ this.logger.warn({
1069
+ event: failureEvent,
1070
+ tenantId: this.tenantId,
1071
+ entryId: failure.id,
1072
+ ...(failure.code ? { code: failure.code } : {}),
1073
+ ...(failure.message ? { failureMessage: failure.message } : {}),
1074
+ ...(failure.senderFault === undefined
1075
+ ? {}
1076
+ : { senderFault: failure.senderFault }),
1077
+ failureReason: "SQS returned a failed visibility entry id not in the request",
1078
+ }, failureDescription);
1079
+ }
1080
+ }
1081
+ const knownFailedCount = [...failuresById.keys()].filter((id) => messageByEntryId.has(id)).length;
1082
+ // Entries absent from SQS's Failed list succeeded. Per-entry
1083
+ // failures remain on the original lease and are excluded from the
1084
+ // retried count; retrying them one by one could amplify a service
1085
+ // or throttling failure into a long serial tail.
1086
+ return batch.length - knownFailedCount;
1087
+ }
1088
+ catch (error) {
1089
+ for (const message of batch) {
1090
+ this.logVisibilityFailure(failureEvent, message, failureDescription, undefined, error);
1091
+ }
1092
+ return 0;
1093
+ }
1094
+ })());
1095
+ }
1096
+ const waveResults = await Promise.all(calls);
1097
+ updatedReceipts += waveResults.reduce((count, updated) => count + updated, 0);
1098
+ }
1099
+ return updatedReceipts;
1100
+ }
1101
+ logVisibilityFailure(event, message, description, failure, error) {
1102
+ const errno = error;
1103
+ this.logger.warn({
1104
+ event,
1105
+ tenantId: this.tenantId,
1106
+ ...(message.MessageId === undefined ? {} : { messageId: message.MessageId }),
1107
+ ...(failure?.id === undefined ? {} : { entryId: failure.id }),
1108
+ ...(failure?.code === undefined ? {} : { code: failure.code }),
1109
+ ...(failure?.message === undefined ? {} : { failureMessage: failure.message }),
1110
+ ...(failure?.senderFault === undefined
1111
+ ? {}
1112
+ : { senderFault: failure.senderFault }),
1113
+ ...(error === undefined
1114
+ ? {}
1115
+ : {
1116
+ err: {
1117
+ message: errno?.message ?? String(error),
1118
+ ...(errno?.code === undefined ? {} : { code: errno.code }),
1119
+ },
1120
+ }),
1121
+ }, description);
1122
+ }
1123
+ /**
1124
+ * Keep SQS receipts valid while a targeted sync operation is still running.
1125
+ * A pull may outlive the receive lease; extending the selected receipts
1126
+ * prevents the queue from issuing a new receipt handle before ack/retry.
1127
+ */
1128
+ startReceiptLeaseHeartbeat() {
1129
+ const receipts = new Set();
1130
+ let stopped = false;
1131
+ let paused = false;
1132
+ let inFlight = null;
1133
+ let timer = null;
1134
+ const heartbeat = () => {
1135
+ if (stopped || paused || inFlight !== null)
1136
+ return;
1137
+ const activeReceipts = [...receipts].filter((message) => Boolean(message.ReceiptHandle));
1138
+ if (activeReceipts.length === 0)
1139
+ return;
1140
+ inFlight = this.changeReceiptVisibilityBatches(activeReceipts, this.visibilityTimeoutSeconds, "receiver.lease.extend.failed", "push receiver failed to extend an in-flight receipt lease", (message) => receipts.delete(message)).then(() => undefined).finally(() => {
1141
+ inFlight = null;
1142
+ });
1143
+ };
1144
+ const clearTimer = () => {
1145
+ if (timer === null)
1146
+ return;
1147
+ clearInterval(timer);
1148
+ timer = null;
1149
+ };
1150
+ const armTimer = () => {
1151
+ if (stopped || paused || timer !== null)
1152
+ return;
1153
+ timer = setInterval(heartbeat, this.visibilityHeartbeatIntervalMs);
1154
+ timer.unref?.();
1155
+ };
1156
+ armTimer();
1157
+ return {
1158
+ add: (message) => {
1159
+ if (!message.ReceiptHandle)
1160
+ return;
1161
+ receipts.add(message);
1162
+ },
1163
+ remove: (message) => {
1164
+ receipts.delete(message);
1165
+ },
1166
+ pause: async () => {
1167
+ if (stopped)
1168
+ return;
1169
+ paused = true;
1170
+ clearTimer();
1171
+ await inFlight;
1172
+ },
1173
+ resume: () => {
1174
+ if (stopped)
1175
+ return;
1176
+ paused = false;
1177
+ armTimer();
1178
+ },
1179
+ stop: async () => {
1180
+ if (stopped)
1181
+ return;
1182
+ stopped = true;
1183
+ paused = true;
1184
+ clearTimer();
1185
+ await inFlight;
1186
+ },
1187
+ };
1188
+ }
825
1189
  /**
826
1190
  * Invoke one coalesced sync pass. The dedupe high-water marks advance only
827
1191
  * after it completes, preserving redelivery after any pass failure.
828
1192
  */
829
1193
  async dispatchBatch(events) {
830
1194
  if (this.disposing || this.disposed)
831
- return false;
1195
+ return { handled: false };
832
1196
  const controller = new AbortController();
833
1197
  this.inFlightAborts.add(controller);
834
1198
  const ctx = { events, signal: controller.signal };
835
1199
  const startMs = this.now();
836
1200
  let handled = false;
1201
+ let failure;
837
1202
  const p = (async () => {
838
1203
  try {
839
1204
  this.logger.debug({ event: "receiver.sync.start", tenantId: this.tenantId, pathCount: events.length }, "push receiver invoking coalesced sync engine");
@@ -886,6 +1251,7 @@ export class SqsPushReceiver {
886
1251
  // crash the receiver loop. Leaving the SQS message undeleted lets
887
1252
  // redelivery retry the failed targeted pull.
888
1253
  const e = err;
1254
+ failure = err;
889
1255
  this.logger.error({
890
1256
  event: "receiver.sync.failed",
891
1257
  tenantId: this.tenantId,
@@ -898,7 +1264,7 @@ export class SqsPushReceiver {
898
1264
  }
899
1265
  })();
900
1266
  await p;
901
- return handled;
1267
+ return { handled, ...(failure === undefined ? {} : { error: failure }) };
902
1268
  }
903
1269
  /** Delete a message, swallowing transport errors (redelivery is harmless). */
904
1270
  async safeDelete(msg) {