@indigoai-us/hq-cloud 6.18.11-alpha.1 → 6.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +1 -1
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +2 -2
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -31
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +51 -212
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +20 -72
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +15 -7
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +16 -15
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +1 -69
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +10 -50
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +40 -121
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +0 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +0 -76
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +8 -11
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +23 -92
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +182 -228
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +0 -3
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +0 -8
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal.d.ts +0 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +2 -66
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +1 -16
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -5
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +1 -58
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +0 -46
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/logger.test.js +0 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -52
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +50 -416
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +40 -673
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/targeted-upsert-exact-list.d.ts +33 -0
- package/dist/sync/targeted-upsert-exact-list.d.ts.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.js +137 -0
- package/dist/sync/targeted-upsert-exact-list.js.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.test.d.ts +2 -0
- package/dist/sync/targeted-upsert-exact-list.test.d.ts.map +1 -0
- package/dist/sync/targeted-upsert-exact-list.test.js +151 -0
- package/dist/sync/targeted-upsert-exact-list.test.js.map +1 -0
- package/dist/watcher.d.ts +0 -18
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +13 -78
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -72
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/bin/sync-runner-args.test.d.ts +0 -2
- package/dist/bin/sync-runner-args.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-args.test.js +0 -82
- package/dist/bin/sync-runner-args.test.js.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts +0 -2
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js +0 -366
- package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts +0 -2
- package/dist/bin/sync-runner-delete-snapshot.bench.test.d.ts.map +0 -1
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js +0 -317
- package/dist/bin/sync-runner-delete-snapshot.bench.test.js.map +0 -1
- package/dist/sync/content-hash.d.ts +0 -4
- package/dist/sync/content-hash.d.ts.map +0 -1
- package/dist/sync/content-hash.js +0 -11
- package/dist/sync/content-hash.js.map +0 -1
- package/dist/sync/content-hash.test.d.ts +0 -2
- package/dist/sync/content-hash.test.d.ts.map +0 -1
- package/dist/sync/content-hash.test.js +0 -16
- package/dist/sync/content-hash.test.js.map +0 -1
- package/dist/sync/unchanged-push-events.d.ts +0 -16
- package/dist/sync/unchanged-push-events.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.js +0 -51
- package/dist/sync/unchanged-push-events.js.map +0 -1
- package/dist/sync/unchanged-push-events.test.d.ts +0 -2
- package/dist/sync/unchanged-push-events.test.d.ts.map +0 -1
- package/dist/sync/unchanged-push-events.test.js +0 -76
- package/dist/sync/unchanged-push-events.test.js.map +0 -1
|
@@ -115,14 +115,8 @@ export const DEFAULT_MAX_MESSAGES = 10;
|
|
|
115
115
|
export const DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS = 600;
|
|
116
116
|
/** Retry a failed targeted pull promptly while preserving queue ownership. */
|
|
117
117
|
export const RECEIVER_RETRY_VISIBILITY_SECONDS = 20;
|
|
118
|
-
/** SQS ChangeMessageVisibilityBatch accepts at most ten receipt entries. */
|
|
119
|
-
const RECEIVER_VISIBILITY_BATCH_SIZE = 10;
|
|
120
118
|
/** Bound the retry fanout while resetting the default 1,000-receipt drain promptly. */
|
|
121
119
|
const RECEIVER_RETRY_VISIBILITY_CONCURRENCY = 256;
|
|
122
|
-
const RECEIVER_SKIP_ACK_CONCURRENCY = 64;
|
|
123
|
-
/** Keep an admitted realtime pull local when a full chore temporarily denies admission. */
|
|
124
|
-
const RECEIVER_ADMISSION_RETRY_INITIAL_MS = 5_000;
|
|
125
|
-
const RECEIVER_ADMISSION_RETRY_MAX_MS = 60_000;
|
|
126
120
|
/** Reconnect backoff defaults. */
|
|
127
121
|
export const DEFAULT_RECONNECT_INITIAL_MS = 250;
|
|
128
122
|
export const DEFAULT_RECONNECT_MAX_MS = 30_000;
|
|
@@ -176,86 +170,6 @@ class BoundedSequenceDedupe {
|
|
|
176
170
|
}
|
|
177
171
|
}
|
|
178
172
|
}
|
|
179
|
-
/**
|
|
180
|
-
* A production pull bridge can carry the runner's failure reason and class
|
|
181
|
-
* across its exit-code boundary so the receiver does not split a transport
|
|
182
|
-
* failure into per-path retries.
|
|
183
|
-
*/
|
|
184
|
-
export class ReceiverDispatchError extends Error {
|
|
185
|
-
failureClass;
|
|
186
|
-
failureReason;
|
|
187
|
-
constructor(failureClass, failureReason) {
|
|
188
|
-
super(failureReason);
|
|
189
|
-
this.failureClass = failureClass;
|
|
190
|
-
this.failureReason = failureReason;
|
|
191
|
-
this.name = "ReceiverDispatchError";
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
function isRealtimeAdmissionTimeout(error) {
|
|
195
|
-
if (typeof error !== "object" || error === null)
|
|
196
|
-
return false;
|
|
197
|
-
const candidate = error;
|
|
198
|
-
return candidate.name === "RealtimeAdmissionTimeout" ||
|
|
199
|
-
(typeof candidate.message === "string" &&
|
|
200
|
-
/^realtime admission timeout\b/i.test(candidate.message));
|
|
201
|
-
}
|
|
202
|
-
const TRANSPORT_FAILURE_CODES = new Set([
|
|
203
|
-
"ABORT_ERR",
|
|
204
|
-
"ACCESSDENIED",
|
|
205
|
-
"ACCESSDENIEDEXCEPTION",
|
|
206
|
-
"CREDENTIALSEXPIRED",
|
|
207
|
-
"ECONNABORTED",
|
|
208
|
-
"ECONNREFUSED",
|
|
209
|
-
"ECONNRESET",
|
|
210
|
-
"EDQUOT",
|
|
211
|
-
"EHOSTUNREACH",
|
|
212
|
-
"EAI_AGAIN",
|
|
213
|
-
"ENETDOWN",
|
|
214
|
-
"ENETUNREACH",
|
|
215
|
-
"ENOSPC",
|
|
216
|
-
"ENOTFOUND",
|
|
217
|
-
"ETIMEDOUT",
|
|
218
|
-
"EXPIREDTOKEN",
|
|
219
|
-
"EXPIREDTOKENEXCEPTION",
|
|
220
|
-
"FORBIDDEN",
|
|
221
|
-
"INVALIDCLIENTTOKENID",
|
|
222
|
-
"REQUESTEXPIRED",
|
|
223
|
-
"TOKENEXPIRED",
|
|
224
|
-
"UNAUTHORIZED",
|
|
225
|
-
"UND_ERR_CONNECT_TIMEOUT",
|
|
226
|
-
"UND_ERR_SOCKET",
|
|
227
|
-
]);
|
|
228
|
-
const TRANSPORT_FAILURE_MESSAGE = /(?:\b(?:ECONNABORTED|ECONNREFUSED|ECONNRESET|EDQUOT|EHOSTUNREACH|EAI_AGAIN|ENETDOWN|ENETUNREACH|ENOSPC|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_SOCKET)\b|no space left on device|disk full|network (?:error|failure|unavailable)|socket (?:hang up|closed)|connection (?:reset|timed? out|refused)|\babort(?:ed|ing)?\b|\btimed? ?out\b|\b(?:HTTP\s*)?(?:401|403|408|429|5\d\d)\b|\b(?:forbidden|unauthorized)\b|(?:access|credential|token).*(?:expired|expiry|invalid|forbidden|unauthorized))/i;
|
|
229
|
-
function isTransportClassError(error, visited = new Set()) {
|
|
230
|
-
if (typeof error === "string")
|
|
231
|
-
return TRANSPORT_FAILURE_MESSAGE.test(error);
|
|
232
|
-
if (typeof error !== "object" || error === null || visited.has(error))
|
|
233
|
-
return false;
|
|
234
|
-
visited.add(error);
|
|
235
|
-
if (error instanceof ReceiverDispatchError) {
|
|
236
|
-
return error.failureClass === "transport";
|
|
237
|
-
}
|
|
238
|
-
const candidate = error;
|
|
239
|
-
const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : "";
|
|
240
|
-
if (TRANSPORT_FAILURE_CODES.has(code) ||
|
|
241
|
-
/^(?:HTTP)?(?:401|403|408|429|5\d\d)$/.test(code) ||
|
|
242
|
-
(typeof candidate.name === "string" && /^(?:AbortError|TimeoutError|ExpiredTokenException)$/i.test(candidate.name))) {
|
|
243
|
-
return true;
|
|
244
|
-
}
|
|
245
|
-
const status = Number(candidate.status ?? candidate.statusCode ?? candidate.$metadata?.httpStatusCode);
|
|
246
|
-
if ([401, 403, 408, 429].includes(status) || status >= 500)
|
|
247
|
-
return true;
|
|
248
|
-
if (typeof candidate.message === "string" &&
|
|
249
|
-
TRANSPORT_FAILURE_MESSAGE.test(candidate.message)) {
|
|
250
|
-
return true;
|
|
251
|
-
}
|
|
252
|
-
return candidate.cause === undefined
|
|
253
|
-
? false
|
|
254
|
-
: isTransportClassError(candidate.cause, visited);
|
|
255
|
-
}
|
|
256
|
-
export function isReceiverTransportFailure(error) {
|
|
257
|
-
return isTransportClassError(error);
|
|
258
|
-
}
|
|
259
173
|
const NOOP_LOGGER = {
|
|
260
174
|
info: () => undefined,
|
|
261
175
|
warn: () => undefined,
|
|
@@ -397,7 +311,6 @@ export class SqsPushReceiver {
|
|
|
397
311
|
/** Legacy per-event seam, retained to preserve the old embedding contract. */
|
|
398
312
|
syncFn;
|
|
399
313
|
shouldProcess;
|
|
400
|
-
skipUnchangedFn;
|
|
401
314
|
onFilteredEvents;
|
|
402
315
|
onReceiveActivity;
|
|
403
316
|
logger;
|
|
@@ -405,7 +318,6 @@ export class SqsPushReceiver {
|
|
|
405
318
|
waitTimeSeconds;
|
|
406
319
|
maxMessages;
|
|
407
320
|
visibilityTimeoutSeconds;
|
|
408
|
-
visibilityHeartbeatIntervalMs;
|
|
409
321
|
disposeDrainMs;
|
|
410
322
|
reconnectInitialMs;
|
|
411
323
|
reconnectMaxMs;
|
|
@@ -457,7 +369,6 @@ export class SqsPushReceiver {
|
|
|
457
369
|
throw new Error("SqsPushReceiver: syncBatchFn is required");
|
|
458
370
|
}
|
|
459
371
|
this.shouldProcess = opts.shouldProcess ?? (() => true);
|
|
460
|
-
this.skipUnchangedFn = opts.skipUnchangedFn;
|
|
461
372
|
this.onFilteredEvents = opts.onFilteredEvents;
|
|
462
373
|
this.onReceiveActivity = opts.onReceiveActivity;
|
|
463
374
|
this.logger = opts.logger ?? NOOP_LOGGER;
|
|
@@ -465,7 +376,6 @@ export class SqsPushReceiver {
|
|
|
465
376
|
this.waitTimeSeconds = opts.waitTimeSeconds ?? DEFAULT_WAIT_TIME_SECONDS;
|
|
466
377
|
this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
|
|
467
378
|
this.visibilityTimeoutSeconds = Math.max(1, Math.min(43_200, Math.floor(opts.visibilityTimeoutSeconds ?? DEFAULT_RECEIVER_VISIBILITY_TIMEOUT_SECONDS)));
|
|
468
|
-
this.visibilityHeartbeatIntervalMs = Math.max(1, Math.min(2_147_483_647, Math.floor(opts.visibilityHeartbeatIntervalMs ?? this.visibilityTimeoutSeconds * 500)));
|
|
469
379
|
this.disposeDrainMs =
|
|
470
380
|
opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
|
|
471
381
|
this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
|
|
@@ -582,7 +492,7 @@ export class SqsPushReceiver {
|
|
|
582
492
|
try {
|
|
583
493
|
received = await this.sqs.receiveMessage({
|
|
584
494
|
queueUrl: this.queueUrl,
|
|
585
|
-
maxMessages:
|
|
495
|
+
maxMessages: this.maxMessages,
|
|
586
496
|
waitTimeSeconds: this.waitTimeSeconds,
|
|
587
497
|
visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
|
|
588
498
|
signal,
|
|
@@ -668,7 +578,6 @@ export class SqsPushReceiver {
|
|
|
668
578
|
let releasedCount = 0;
|
|
669
579
|
let retriedCount = 0;
|
|
670
580
|
let processedCount = 0;
|
|
671
|
-
let skippedUnchangedCount = 0;
|
|
672
581
|
let redeliveredCount = 0;
|
|
673
582
|
let maxReceiveCount = 0;
|
|
674
583
|
let oldestSentTimestampMs;
|
|
@@ -680,32 +589,15 @@ export class SqsPushReceiver {
|
|
|
680
589
|
intakeCompleted = true;
|
|
681
590
|
onIntakeComplete();
|
|
682
591
|
};
|
|
683
|
-
const leaseHeartbeat = this.startReceiptLeaseHeartbeat();
|
|
684
592
|
const acknowledge = async (msg) => {
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
if (await this.safeDelete(msg)) {
|
|
688
|
-
deletedCount += 1;
|
|
689
|
-
return true;
|
|
690
|
-
}
|
|
691
|
-
// Drain any extension already in flight and remove this receipt from
|
|
692
|
-
// future snapshots before releasing it. Otherwise a late heartbeat
|
|
693
|
-
// can restore the full lease after the zero-visibility retry release.
|
|
694
|
-
await leaseHeartbeat.pause();
|
|
695
|
-
heartbeatPaused = true;
|
|
696
|
-
leaseHeartbeat.remove(msg);
|
|
697
|
-
if (await this.releaseUnselected(msg)) {
|
|
698
|
-
// A failed delete is not allowed to strand a stale receipt until
|
|
699
|
-
// its normal lease expires. Releasing returns retry ownership to
|
|
700
|
-
// SQS immediately while successful receipts remain heartbeatable.
|
|
701
|
-
releasedCount += 1;
|
|
702
|
-
}
|
|
703
|
-
return false;
|
|
593
|
+
if (await this.safeDelete(msg)) {
|
|
594
|
+
deletedCount += 1;
|
|
704
595
|
}
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
596
|
+
else if (await this.releaseUnselected(msg)) {
|
|
597
|
+
// A failed delete is not allowed to strand a stale receipt until its
|
|
598
|
+
// normal lease expires. Releasing preserves delete-after-success while
|
|
599
|
+
// returning retry ownership to SQS immediately.
|
|
600
|
+
releasedCount += 1;
|
|
709
601
|
}
|
|
710
602
|
};
|
|
711
603
|
const countOrigin = (originDeviceId) => {
|
|
@@ -779,7 +671,6 @@ export class SqsPushReceiver {
|
|
|
779
671
|
const receipts = receiptsByPath.get(event.relativePath) ?? [];
|
|
780
672
|
receipts.push(msg);
|
|
781
673
|
receiptsByPath.set(event.relativePath, receipts);
|
|
782
|
-
leaseHeartbeat.add(msg);
|
|
783
674
|
if (!selectedEvent || event.sequenceNumber > selectedEvent.sequenceNumber) {
|
|
784
675
|
selected.set(event.relativePath, event);
|
|
785
676
|
}
|
|
@@ -802,7 +693,6 @@ export class SqsPushReceiver {
|
|
|
802
693
|
let emptyConfirmations = 0;
|
|
803
694
|
let waitTimeSeconds = 0;
|
|
804
695
|
while (!saturated &&
|
|
805
|
-
selected.size < this.batchMaxPaths &&
|
|
806
696
|
receivedCount < this.drainMaxMessages &&
|
|
807
697
|
!signal.aborted &&
|
|
808
698
|
this.now() < fillDeadline) {
|
|
@@ -816,7 +706,7 @@ export class SqsPushReceiver {
|
|
|
816
706
|
try {
|
|
817
707
|
next = await this.sqs.receiveMessage({
|
|
818
708
|
queueUrl: this.queueUrl,
|
|
819
|
-
maxMessages: Math.min(this.maxMessages, this.
|
|
709
|
+
maxMessages: Math.min(this.maxMessages, this.drainMaxMessages - receivedCount),
|
|
820
710
|
waitTimeSeconds,
|
|
821
711
|
visibilityTimeoutSeconds: this.visibilityTimeoutSeconds,
|
|
822
712
|
signal,
|
|
@@ -850,71 +740,50 @@ export class SqsPushReceiver {
|
|
|
850
740
|
}
|
|
851
741
|
completeIntake();
|
|
852
742
|
const events = [...selected.values()];
|
|
853
|
-
|
|
854
|
-
if (this.skipUnchangedFn && events.length > 0) {
|
|
855
|
-
let unchangedPaths = new Set();
|
|
856
|
-
try {
|
|
857
|
-
unchangedPaths = await this.skipUnchangedFn(events);
|
|
858
|
-
}
|
|
859
|
-
catch (err) {
|
|
860
|
-
// A journal/stat read is an optimization only. Preserve delivery by
|
|
861
|
-
// falling back to the ordinary targeted pull when the proof fails.
|
|
862
|
-
this.logger.warn({
|
|
863
|
-
event: "receiver.unchanged.check.failed",
|
|
864
|
-
tenantId: this.tenantId,
|
|
865
|
-
pathCount: events.length,
|
|
866
|
-
err: { message: err?.message },
|
|
867
|
-
}, "push receiver could not prove unchanged local content; dispatching pull");
|
|
868
|
-
}
|
|
869
|
-
const unchangedEvents = [];
|
|
870
|
-
eventsToDispatch = events.filter((event) => {
|
|
871
|
-
if (!unchangedPaths.has(event.relativePath))
|
|
872
|
-
return true;
|
|
873
|
-
unchangedEvents.push(event);
|
|
874
|
-
return false;
|
|
875
|
-
});
|
|
876
|
-
skippedUnchangedCount += unchangedEvents.length;
|
|
877
|
-
for (let offset = 0; offset < unchangedEvents.length; offset += RECEIVER_SKIP_ACK_CONCURRENCY) {
|
|
878
|
-
const acknowledgementBatch = unchangedEvents.slice(offset, offset + RECEIVER_SKIP_ACK_CONCURRENCY);
|
|
879
|
-
await Promise.all(acknowledgementBatch.map(async (event) => {
|
|
880
|
-
const receipts = receiptsByPath.get(event.relativePath) ?? [];
|
|
881
|
-
const acknowledgements = await Promise.all(receipts.map(acknowledge));
|
|
882
|
-
if (acknowledgements.every(Boolean)) {
|
|
883
|
-
this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
|
|
884
|
-
}
|
|
885
|
-
}));
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
const batches = eventsToDispatch.length === 0
|
|
889
|
-
? []
|
|
890
|
-
: this.syncBatchFn.partition?.(eventsToDispatch) ?? [eventsToDispatch];
|
|
743
|
+
const batches = this.syncBatchFn.partition?.(events) ?? [events];
|
|
891
744
|
for (const batch of batches) {
|
|
892
745
|
if (batch.length === 0)
|
|
893
746
|
continue;
|
|
894
|
-
const
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
747
|
+
const handled = await this.dispatchBatch(batch);
|
|
748
|
+
if (!handled) {
|
|
749
|
+
// Disposal still leaves the current lease untouched. A failed pull
|
|
750
|
+
// gets a short visibility delay so this receipt can retry promptly;
|
|
751
|
+
// an SQS visibility failure is contained to this receipt and falls
|
|
752
|
+
// back to the queue's normal timeout.
|
|
753
|
+
if (this.disposing || this.disposed)
|
|
754
|
+
return;
|
|
755
|
+
const retryReceipts = batch.flatMap((event) => receiptsByPath.get(event.relativePath) ?? []);
|
|
756
|
+
const retryReceipt = async (msg) => {
|
|
757
|
+
if (!msg.ReceiptHandle)
|
|
758
|
+
return;
|
|
759
|
+
retriedCount += 1;
|
|
760
|
+
try {
|
|
761
|
+
await this.sqs.changeMessageVisibility({
|
|
762
|
+
queueUrl: this.queueUrl,
|
|
763
|
+
receiptHandle: msg.ReceiptHandle,
|
|
764
|
+
visibilityTimeoutSeconds: RECEIVER_RETRY_VISIBILITY_SECONDS,
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
catch (err) {
|
|
768
|
+
this.logger.warn({
|
|
769
|
+
event: "receiver.retry.failed",
|
|
770
|
+
tenantId: this.tenantId,
|
|
771
|
+
messageId: msg.MessageId,
|
|
772
|
+
err: { message: err?.message },
|
|
773
|
+
}, "push receiver failed to shorten retry visibility");
|
|
774
|
+
}
|
|
775
|
+
};
|
|
776
|
+
for (let offset = 0; offset < retryReceipts.length; offset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
|
|
777
|
+
const retryBatch = retryReceipts.slice(offset, offset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
|
|
778
|
+
await Promise.all(retryBatch.map(retryReceipt));
|
|
908
779
|
}
|
|
780
|
+
continue;
|
|
909
781
|
}
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
// successful lanes are acknowledged even when their original aggregate
|
|
916
|
-
// pull needed per-path recovery.
|
|
917
|
-
for (const event of successfulEvents) {
|
|
782
|
+
// Receipt acknowledgement follows the successful lane, not the original
|
|
783
|
+
// aggregate delivery. A slow-lane failure therefore leaves only its own
|
|
784
|
+
// events available for redelivery while dedupe absorbs the completed fast
|
|
785
|
+
// lane if a transport delete races or fails.
|
|
786
|
+
for (const event of batch) {
|
|
918
787
|
processedCount += 1;
|
|
919
788
|
for (const msg of receiptsByPath.get(event.relativePath) ?? []) {
|
|
920
789
|
await acknowledge(msg);
|
|
@@ -924,7 +793,6 @@ export class SqsPushReceiver {
|
|
|
924
793
|
}
|
|
925
794
|
finally {
|
|
926
795
|
completeIntake();
|
|
927
|
-
await leaseHeartbeat.stop();
|
|
928
796
|
if (filteredEvents.length > 0)
|
|
929
797
|
this.onFilteredEvents?.(filteredEvents);
|
|
930
798
|
this.logger.info({
|
|
@@ -933,7 +801,6 @@ export class SqsPushReceiver {
|
|
|
933
801
|
received: receivedCount,
|
|
934
802
|
coalesced: selected.size,
|
|
935
803
|
processed: processedCount,
|
|
936
|
-
skippedUnchanged: skippedUnchangedCount,
|
|
937
804
|
deleted: deletedCount,
|
|
938
805
|
released: releasedCount,
|
|
939
806
|
retried: retriedCount,
|
|
@@ -955,250 +822,18 @@ export class SqsPushReceiver {
|
|
|
955
822
|
}, "push receiver drain completed");
|
|
956
823
|
}
|
|
957
824
|
}
|
|
958
|
-
async dispatchBatchUntilAdmitted(events, signal) {
|
|
959
|
-
let result = await this.dispatchBatch(events);
|
|
960
|
-
let retryDelayMs = RECEIVER_ADMISSION_RETRY_INITIAL_MS;
|
|
961
|
-
while (!result.handled &&
|
|
962
|
-
isRealtimeAdmissionTimeout(result.error) &&
|
|
963
|
-
!signal.aborted &&
|
|
964
|
-
!this.disposing &&
|
|
965
|
-
!this.disposed) {
|
|
966
|
-
this.logger.info({
|
|
967
|
-
event: "receiver.sync.admission-timeout",
|
|
968
|
-
tenantId: this.tenantId,
|
|
969
|
-
pathCount: events.length,
|
|
970
|
-
retryDelayMs,
|
|
971
|
-
}, "push receiver retaining receipt lease while realtime admission is busy");
|
|
972
|
-
await this.sleep(retryDelayMs, signal);
|
|
973
|
-
if (signal.aborted || this.disposing || this.disposed)
|
|
974
|
-
return result;
|
|
975
|
-
result = await this.dispatchBatch(events);
|
|
976
|
-
retryDelayMs = Math.min(RECEIVER_ADMISSION_RETRY_MAX_MS, retryDelayMs * 2);
|
|
977
|
-
}
|
|
978
|
-
return result;
|
|
979
|
-
}
|
|
980
|
-
/**
|
|
981
|
-
* Isolate deterministic path failures by bisecting failed batches. A
|
|
982
|
-
* transport-class failure never fans out to singleton pulls; if both halves
|
|
983
|
-
* fail for transport reasons, the whole unresolved range returns to SQS.
|
|
984
|
-
*/
|
|
985
|
-
async recoverFailedBatch(events, initialResult, signal) {
|
|
986
|
-
const successfulEvents = [];
|
|
987
|
-
const failedEvents = [];
|
|
988
|
-
let interrupted = false;
|
|
989
|
-
const recover = async (batch, result) => {
|
|
990
|
-
if (interrupted)
|
|
991
|
-
return;
|
|
992
|
-
if (result.handled) {
|
|
993
|
-
successfulEvents.push(...batch);
|
|
994
|
-
return;
|
|
995
|
-
}
|
|
996
|
-
if (signal.aborted || this.disposing || this.disposed) {
|
|
997
|
-
interrupted = true;
|
|
998
|
-
return;
|
|
999
|
-
}
|
|
1000
|
-
if (batch.length === 1 || isTransportClassError(result.error)) {
|
|
1001
|
-
failedEvents.push(...batch);
|
|
1002
|
-
return;
|
|
1003
|
-
}
|
|
1004
|
-
const middle = Math.floor(batch.length / 2);
|
|
1005
|
-
const left = batch.slice(0, middle);
|
|
1006
|
-
const right = batch.slice(middle);
|
|
1007
|
-
const [leftResult, rightResult] = await Promise.all([
|
|
1008
|
-
this.dispatchBatchUntilAdmitted(left, signal),
|
|
1009
|
-
this.dispatchBatchUntilAdmitted(right, signal),
|
|
1010
|
-
]);
|
|
1011
|
-
if (signal.aborted || this.disposing || this.disposed) {
|
|
1012
|
-
interrupted = true;
|
|
1013
|
-
return;
|
|
1014
|
-
}
|
|
1015
|
-
if (!leftResult.handled &&
|
|
1016
|
-
!rightResult.handled &&
|
|
1017
|
-
isTransportClassError(leftResult.error) &&
|
|
1018
|
-
isTransportClassError(rightResult.error)) {
|
|
1019
|
-
failedEvents.push(...batch);
|
|
1020
|
-
return;
|
|
1021
|
-
}
|
|
1022
|
-
await recover(left, leftResult);
|
|
1023
|
-
await recover(right, rightResult);
|
|
1024
|
-
};
|
|
1025
|
-
await recover(events, initialResult);
|
|
1026
|
-
return { successfulEvents, failedEvents, interrupted };
|
|
1027
|
-
}
|
|
1028
|
-
async retryFailedEvents(events, receiptsByPath, onRetried) {
|
|
1029
|
-
const retryReceipts = events
|
|
1030
|
-
.flatMap((event) => receiptsByPath.get(event.relativePath) ?? [])
|
|
1031
|
-
.filter((message) => Boolean(message.ReceiptHandle));
|
|
1032
|
-
const updatedReceipts = await this.changeReceiptVisibilityBatches(retryReceipts, RECEIVER_RETRY_VISIBILITY_SECONDS, "receiver.retry.failed", "push receiver failed to shorten retry visibility");
|
|
1033
|
-
onRetried(updatedReceipts);
|
|
1034
|
-
}
|
|
1035
|
-
async changeReceiptVisibilityBatches(messages, visibilityTimeoutSeconds, failureEvent, failureDescription, onInvalidReceipt) {
|
|
1036
|
-
let updatedReceipts = 0;
|
|
1037
|
-
for (let waveOffset = 0; waveOffset < messages.length; waveOffset += RECEIVER_RETRY_VISIBILITY_CONCURRENCY) {
|
|
1038
|
-
const wave = messages.slice(waveOffset, waveOffset + RECEIVER_RETRY_VISIBILITY_CONCURRENCY);
|
|
1039
|
-
const calls = [];
|
|
1040
|
-
for (let batchOffset = 0; batchOffset < wave.length; batchOffset += RECEIVER_VISIBILITY_BATCH_SIZE) {
|
|
1041
|
-
const batch = wave.slice(batchOffset, batchOffset + RECEIVER_VISIBILITY_BATCH_SIZE);
|
|
1042
|
-
const entries = batch.map((message, index) => ({
|
|
1043
|
-
id: `receipt-${index}`,
|
|
1044
|
-
receiptHandle: message.ReceiptHandle,
|
|
1045
|
-
}));
|
|
1046
|
-
const messageByEntryId = new Map(entries.map((entry, index) => [entry.id, batch[index]]));
|
|
1047
|
-
calls.push((async () => {
|
|
1048
|
-
try {
|
|
1049
|
-
const result = await this.sqs.changeMessageVisibilityBatch({
|
|
1050
|
-
queueUrl: this.queueUrl,
|
|
1051
|
-
entries,
|
|
1052
|
-
visibilityTimeoutSeconds,
|
|
1053
|
-
});
|
|
1054
|
-
const failuresById = new Map();
|
|
1055
|
-
for (const failure of result.failedEntries) {
|
|
1056
|
-
if (!failuresById.has(failure.id))
|
|
1057
|
-
failuresById.set(failure.id, failure);
|
|
1058
|
-
}
|
|
1059
|
-
for (const failure of failuresById.values()) {
|
|
1060
|
-
const message = messageByEntryId.get(failure.id);
|
|
1061
|
-
if (message) {
|
|
1062
|
-
if (failure.code === "ReceiptHandleIsInvalid") {
|
|
1063
|
-
onInvalidReceipt?.(message);
|
|
1064
|
-
}
|
|
1065
|
-
this.logVisibilityFailure(failureEvent, message, failureDescription, failure);
|
|
1066
|
-
}
|
|
1067
|
-
else {
|
|
1068
|
-
this.logger.warn({
|
|
1069
|
-
event: failureEvent,
|
|
1070
|
-
tenantId: this.tenantId,
|
|
1071
|
-
entryId: failure.id,
|
|
1072
|
-
...(failure.code ? { code: failure.code } : {}),
|
|
1073
|
-
...(failure.message ? { failureMessage: failure.message } : {}),
|
|
1074
|
-
...(failure.senderFault === undefined
|
|
1075
|
-
? {}
|
|
1076
|
-
: { senderFault: failure.senderFault }),
|
|
1077
|
-
failureReason: "SQS returned a failed visibility entry id not in the request",
|
|
1078
|
-
}, failureDescription);
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
const knownFailedCount = [...failuresById.keys()].filter((id) => messageByEntryId.has(id)).length;
|
|
1082
|
-
// Entries absent from SQS's Failed list succeeded. Per-entry
|
|
1083
|
-
// failures remain on the original lease and are excluded from the
|
|
1084
|
-
// retried count; retrying them one by one could amplify a service
|
|
1085
|
-
// or throttling failure into a long serial tail.
|
|
1086
|
-
return batch.length - knownFailedCount;
|
|
1087
|
-
}
|
|
1088
|
-
catch (error) {
|
|
1089
|
-
for (const message of batch) {
|
|
1090
|
-
this.logVisibilityFailure(failureEvent, message, failureDescription, undefined, error);
|
|
1091
|
-
}
|
|
1092
|
-
return 0;
|
|
1093
|
-
}
|
|
1094
|
-
})());
|
|
1095
|
-
}
|
|
1096
|
-
const waveResults = await Promise.all(calls);
|
|
1097
|
-
updatedReceipts += waveResults.reduce((count, updated) => count + updated, 0);
|
|
1098
|
-
}
|
|
1099
|
-
return updatedReceipts;
|
|
1100
|
-
}
|
|
1101
|
-
logVisibilityFailure(event, message, description, failure, error) {
|
|
1102
|
-
const errno = error;
|
|
1103
|
-
this.logger.warn({
|
|
1104
|
-
event,
|
|
1105
|
-
tenantId: this.tenantId,
|
|
1106
|
-
...(message.MessageId === undefined ? {} : { messageId: message.MessageId }),
|
|
1107
|
-
...(failure?.id === undefined ? {} : { entryId: failure.id }),
|
|
1108
|
-
...(failure?.code === undefined ? {} : { code: failure.code }),
|
|
1109
|
-
...(failure?.message === undefined ? {} : { failureMessage: failure.message }),
|
|
1110
|
-
...(failure?.senderFault === undefined
|
|
1111
|
-
? {}
|
|
1112
|
-
: { senderFault: failure.senderFault }),
|
|
1113
|
-
...(error === undefined
|
|
1114
|
-
? {}
|
|
1115
|
-
: {
|
|
1116
|
-
err: {
|
|
1117
|
-
message: errno?.message ?? String(error),
|
|
1118
|
-
...(errno?.code === undefined ? {} : { code: errno.code }),
|
|
1119
|
-
},
|
|
1120
|
-
}),
|
|
1121
|
-
}, description);
|
|
1122
|
-
}
|
|
1123
|
-
/**
|
|
1124
|
-
* Keep SQS receipts valid while a targeted sync operation is still running.
|
|
1125
|
-
* A pull may outlive the receive lease; extending the selected receipts
|
|
1126
|
-
* prevents the queue from issuing a new receipt handle before ack/retry.
|
|
1127
|
-
*/
|
|
1128
|
-
startReceiptLeaseHeartbeat() {
|
|
1129
|
-
const receipts = new Set();
|
|
1130
|
-
let stopped = false;
|
|
1131
|
-
let paused = false;
|
|
1132
|
-
let inFlight = null;
|
|
1133
|
-
let timer = null;
|
|
1134
|
-
const heartbeat = () => {
|
|
1135
|
-
if (stopped || paused || inFlight !== null)
|
|
1136
|
-
return;
|
|
1137
|
-
const activeReceipts = [...receipts].filter((message) => Boolean(message.ReceiptHandle));
|
|
1138
|
-
if (activeReceipts.length === 0)
|
|
1139
|
-
return;
|
|
1140
|
-
inFlight = this.changeReceiptVisibilityBatches(activeReceipts, this.visibilityTimeoutSeconds, "receiver.lease.extend.failed", "push receiver failed to extend an in-flight receipt lease", (message) => receipts.delete(message)).then(() => undefined).finally(() => {
|
|
1141
|
-
inFlight = null;
|
|
1142
|
-
});
|
|
1143
|
-
};
|
|
1144
|
-
const clearTimer = () => {
|
|
1145
|
-
if (timer === null)
|
|
1146
|
-
return;
|
|
1147
|
-
clearInterval(timer);
|
|
1148
|
-
timer = null;
|
|
1149
|
-
};
|
|
1150
|
-
const armTimer = () => {
|
|
1151
|
-
if (stopped || paused || timer !== null)
|
|
1152
|
-
return;
|
|
1153
|
-
timer = setInterval(heartbeat, this.visibilityHeartbeatIntervalMs);
|
|
1154
|
-
timer.unref?.();
|
|
1155
|
-
};
|
|
1156
|
-
armTimer();
|
|
1157
|
-
return {
|
|
1158
|
-
add: (message) => {
|
|
1159
|
-
if (!message.ReceiptHandle)
|
|
1160
|
-
return;
|
|
1161
|
-
receipts.add(message);
|
|
1162
|
-
},
|
|
1163
|
-
remove: (message) => {
|
|
1164
|
-
receipts.delete(message);
|
|
1165
|
-
},
|
|
1166
|
-
pause: async () => {
|
|
1167
|
-
if (stopped)
|
|
1168
|
-
return;
|
|
1169
|
-
paused = true;
|
|
1170
|
-
clearTimer();
|
|
1171
|
-
await inFlight;
|
|
1172
|
-
},
|
|
1173
|
-
resume: () => {
|
|
1174
|
-
if (stopped)
|
|
1175
|
-
return;
|
|
1176
|
-
paused = false;
|
|
1177
|
-
armTimer();
|
|
1178
|
-
},
|
|
1179
|
-
stop: async () => {
|
|
1180
|
-
if (stopped)
|
|
1181
|
-
return;
|
|
1182
|
-
stopped = true;
|
|
1183
|
-
paused = true;
|
|
1184
|
-
clearTimer();
|
|
1185
|
-
await inFlight;
|
|
1186
|
-
},
|
|
1187
|
-
};
|
|
1188
|
-
}
|
|
1189
825
|
/**
|
|
1190
826
|
* Invoke one coalesced sync pass. The dedupe high-water marks advance only
|
|
1191
827
|
* after it completes, preserving redelivery after any pass failure.
|
|
1192
828
|
*/
|
|
1193
829
|
async dispatchBatch(events) {
|
|
1194
830
|
if (this.disposing || this.disposed)
|
|
1195
|
-
return
|
|
831
|
+
return false;
|
|
1196
832
|
const controller = new AbortController();
|
|
1197
833
|
this.inFlightAborts.add(controller);
|
|
1198
834
|
const ctx = { events, signal: controller.signal };
|
|
1199
835
|
const startMs = this.now();
|
|
1200
836
|
let handled = false;
|
|
1201
|
-
let failure;
|
|
1202
837
|
const p = (async () => {
|
|
1203
838
|
try {
|
|
1204
839
|
this.logger.debug({ event: "receiver.sync.start", tenantId: this.tenantId, pathCount: events.length }, "push receiver invoking coalesced sync engine");
|
|
@@ -1251,7 +886,6 @@ export class SqsPushReceiver {
|
|
|
1251
886
|
// crash the receiver loop. Leaving the SQS message undeleted lets
|
|
1252
887
|
// redelivery retry the failed targeted pull.
|
|
1253
888
|
const e = err;
|
|
1254
|
-
failure = err;
|
|
1255
889
|
this.logger.error({
|
|
1256
890
|
event: "receiver.sync.failed",
|
|
1257
891
|
tenantId: this.tenantId,
|
|
@@ -1264,7 +898,7 @@ export class SqsPushReceiver {
|
|
|
1264
898
|
}
|
|
1265
899
|
})();
|
|
1266
900
|
await p;
|
|
1267
|
-
return
|
|
901
|
+
return handled;
|
|
1268
902
|
}
|
|
1269
903
|
/** Delete a message, swallowing transport errors (redelivery is harmless). */
|
|
1270
904
|
async safeDelete(msg) {
|