@beignet/core 0.0.50 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +53 -26
- package/dist/application/index.d.ts +5 -5
- package/dist/application/index.d.ts.map +1 -1
- package/dist/application/index.js +5 -4
- package/dist/application/index.js.map +1 -1
- package/dist/events/index.d.ts.map +1 -1
- package/dist/events/index.js +6 -3
- package/dist/events/index.js.map +1 -1
- package/dist/events/payload-state.d.ts +4 -0
- package/dist/events/payload-state.d.ts.map +1 -0
- package/dist/events/payload-state.js +11 -0
- package/dist/events/payload-state.js.map +1 -0
- package/dist/locks/index.d.ts.map +1 -1
- package/dist/locks/index.js +0 -4
- package/dist/locks/index.js.map +1 -1
- package/dist/outbox/index.d.ts +163 -15
- package/dist/outbox/index.d.ts.map +1 -1
- package/dist/outbox/index.js +1009 -152
- package/dist/outbox/index.js.map +1 -1
- package/dist/payments/index.d.ts.map +1 -1
- package/dist/payments/index.js +0 -4
- package/dist/payments/index.js.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/ports/testing.d.ts +13 -0
- package/dist/ports/testing.d.ts.map +1 -1
- package/dist/ports/testing.js +12 -0
- package/dist/ports/testing.js.map +1 -1
- package/dist/ports/unit-of-work.d.ts +9 -7
- package/dist/ports/unit-of-work.d.ts.map +1 -1
- package/dist/ports/unit-of-work.js +16 -6
- package/dist/ports/unit-of-work.js.map +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/provider.d.ts +0 -38
- package/dist/providers/provider.d.ts.map +1 -1
- package/dist/providers/provider.js.map +1 -1
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +0 -4
- package/dist/search/index.js.map +1 -1
- package/dist/server/hooks/cors.d.ts +5 -0
- package/dist/server/hooks/cors.d.ts.map +1 -1
- package/dist/server/hooks/cors.js +29 -1
- package/dist/server/hooks/cors.js.map +1 -1
- package/dist/server/server.d.ts.map +1 -1
- package/dist/server/server.js +34 -9
- package/dist/server/server.js.map +1 -1
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +12 -6
- package/dist/testing/index.js.map +1 -1
- package/package.json +1 -1
- package/skills/app-architecture/SKILL.md +14 -0
- package/src/application/index.ts +18 -7
- package/src/events/index.ts +9 -6
- package/src/events/payload-state.ts +24 -0
- package/src/locks/index.ts +0 -4
- package/src/outbox/index.ts +1382 -175
- package/src/payments/index.ts +0 -4
- package/src/ports/index.ts +3 -0
- package/src/ports/testing.ts +31 -0
- package/src/ports/unit-of-work.ts +28 -14
- package/src/providers/index.ts +0 -1
- package/src/providers/provider.ts +0 -40
- package/src/search/index.ts +0 -4
- package/src/server/hooks/cors.ts +42 -0
- package/src/server/server.ts +52 -19
- package/src/testing/index.ts +20 -9
package/src/outbox/index.ts
CHANGED
|
@@ -11,6 +11,10 @@ import {
|
|
|
11
11
|
type InferEventPayload,
|
|
12
12
|
parseEventPayload,
|
|
13
13
|
} from "../events/index.js";
|
|
14
|
+
import {
|
|
15
|
+
isEventPayloadParsed,
|
|
16
|
+
markEventPayloadParsed,
|
|
17
|
+
} from "../events/payload-state.js";
|
|
14
18
|
import {
|
|
15
19
|
getJobRetryDelayMs,
|
|
16
20
|
getJobRetryMaxAttempts,
|
|
@@ -46,6 +50,18 @@ export type MaybePromise<T> = T | Promise<T>;
|
|
|
46
50
|
* Default lease duration for claimed outbox messages.
|
|
47
51
|
*/
|
|
48
52
|
export const DEFAULT_OUTBOX_LEASE_MS = 30_000;
|
|
53
|
+
/**
|
|
54
|
+
* Default maximum messages handled by one bounded drain pass.
|
|
55
|
+
*/
|
|
56
|
+
export const DEFAULT_OUTBOX_BATCH_SIZE = 100;
|
|
57
|
+
/**
|
|
58
|
+
* Default number of outbox messages delivered concurrently.
|
|
59
|
+
*/
|
|
60
|
+
export const DEFAULT_OUTBOX_CONCURRENCY = 1;
|
|
61
|
+
/**
|
|
62
|
+
* Default maximum time Beignet renews a claim for one delivery.
|
|
63
|
+
*/
|
|
64
|
+
export const DEFAULT_OUTBOX_MAX_ACTIVE_MS = 300_000;
|
|
49
65
|
/**
|
|
50
66
|
* Default maximum delivery attempts before a message is dead-lettered.
|
|
51
67
|
*/
|
|
@@ -218,7 +234,7 @@ export interface ClaimedOutboxMessage
|
|
|
218
234
|
*/
|
|
219
235
|
export interface OutboxClaimBatchOptions {
|
|
220
236
|
/**
|
|
221
|
-
* Maximum messages to claim in one batch.
|
|
237
|
+
* Maximum eligible messages to claim or reconcile in one batch.
|
|
222
238
|
*/
|
|
223
239
|
limit: number;
|
|
224
240
|
/**
|
|
@@ -231,6 +247,41 @@ export interface OutboxClaimBatchOptions {
|
|
|
231
247
|
leaseMs?: number;
|
|
232
248
|
}
|
|
233
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Result of atomically selecting one bounded set of eligible messages.
|
|
252
|
+
*/
|
|
253
|
+
export interface OutboxClaimBatchResult {
|
|
254
|
+
/** Messages claimed for delivery by the current worker. */
|
|
255
|
+
claimed: readonly ClaimedOutboxMessage[];
|
|
256
|
+
/**
|
|
257
|
+
* Eligible messages moved directly to dead letter because their claim
|
|
258
|
+
* attempt budget was already exhausted.
|
|
259
|
+
*/
|
|
260
|
+
deadLettered: readonly OutboxMessage[];
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Input for extending one active outbox claim.
|
|
265
|
+
*/
|
|
266
|
+
export interface OutboxRenewClaimInput {
|
|
267
|
+
/** Claimed message ID. */
|
|
268
|
+
id: string;
|
|
269
|
+
/** Claim token returned by `claimBatch(...)`. */
|
|
270
|
+
claimToken: string;
|
|
271
|
+
/** Renewal timestamp. */
|
|
272
|
+
now?: Date;
|
|
273
|
+
/** New lease duration measured from `now`. */
|
|
274
|
+
leaseMs?: number;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Confirmed result of extending one active outbox claim.
|
|
279
|
+
*/
|
|
280
|
+
export interface OutboxRenewClaimResult {
|
|
281
|
+
/** New confirmed lease expiration timestamp. */
|
|
282
|
+
lockedUntil: Date;
|
|
283
|
+
}
|
|
284
|
+
|
|
234
285
|
/**
|
|
235
286
|
* Input for marking a claimed message delivered.
|
|
236
287
|
*/
|
|
@@ -403,7 +454,11 @@ export interface OutboxPort {
|
|
|
403
454
|
/**
|
|
404
455
|
* Atomically claim eligible messages for one worker.
|
|
405
456
|
*/
|
|
406
|
-
claimBatch(options: OutboxClaimBatchOptions): Promise<
|
|
457
|
+
claimBatch(options: OutboxClaimBatchOptions): Promise<OutboxClaimBatchResult>;
|
|
458
|
+
/**
|
|
459
|
+
* Extend an unexpired claim owned by the supplied claim token.
|
|
460
|
+
*/
|
|
461
|
+
renewClaim(input: OutboxRenewClaimInput): Promise<OutboxRenewClaimResult>;
|
|
407
462
|
/**
|
|
408
463
|
* Mark a claimed message delivered.
|
|
409
464
|
*/
|
|
@@ -542,6 +597,40 @@ export type OutboxInstrumentationContext = Pick<
|
|
|
542
597
|
"requestId" | "traceId" | "spanId" | "parentSpanId" | "traceparent"
|
|
543
598
|
>;
|
|
544
599
|
|
|
600
|
+
/** Wait primitive used by outbox heartbeats and bounded settlement retries. */
|
|
601
|
+
export type OutboxDrainWait = (
|
|
602
|
+
delayMs: number,
|
|
603
|
+
signal: AbortSignal,
|
|
604
|
+
) => Promise<void>;
|
|
605
|
+
|
|
606
|
+
/** Structured failure from a claim heartbeat or active-delivery boundary. */
|
|
607
|
+
export interface OutboxLeaseFailure {
|
|
608
|
+
/** Underlying renewal, ownership, or duration error. */
|
|
609
|
+
error: unknown;
|
|
610
|
+
/** Message whose claim could not be kept active. */
|
|
611
|
+
message: ClaimedOutboxMessage;
|
|
612
|
+
/** Lease phase that surfaced the failure. */
|
|
613
|
+
operation: "renewClaim" | "maxActiveDuration";
|
|
614
|
+
/** Current lease state after handling the failure. */
|
|
615
|
+
state: "recovered" | "degraded" | "lost";
|
|
616
|
+
/** Whether the worker no longer has confirmed ownership. */
|
|
617
|
+
confirmedLost: boolean;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/** Structured failure from a token-guarded outbox settlement. */
|
|
621
|
+
export interface OutboxSettlementFailure {
|
|
622
|
+
/** Storage failure returned by the settlement operation. */
|
|
623
|
+
error: unknown;
|
|
624
|
+
/** Message whose final storage state is unknown. */
|
|
625
|
+
message: ClaimedOutboxMessage;
|
|
626
|
+
/** Settlement operation that failed. */
|
|
627
|
+
operation: "markDelivered" | "markFailed";
|
|
628
|
+
/** Whether the external delivery completed successfully. */
|
|
629
|
+
deliverySucceeded: boolean;
|
|
630
|
+
/** Original delivery error when `deliverySucceeded` is false. */
|
|
631
|
+
deliveryError?: unknown;
|
|
632
|
+
}
|
|
633
|
+
|
|
545
634
|
/**
|
|
546
635
|
* Options for draining one outbox batch.
|
|
547
636
|
*/
|
|
@@ -571,17 +660,38 @@ export interface DrainOutboxOptions {
|
|
|
571
660
|
*/
|
|
572
661
|
jobs?: JobDispatcherPort;
|
|
573
662
|
/**
|
|
574
|
-
* Maximum messages to
|
|
663
|
+
* Maximum eligible messages to handle in one drain pass.
|
|
575
664
|
*/
|
|
576
665
|
batchSize?: number;
|
|
577
666
|
/**
|
|
578
|
-
*
|
|
667
|
+
* Maximum messages delivered concurrently. Defaults to serial delivery.
|
|
668
|
+
* Values greater than one do not preserve delivery order.
|
|
579
669
|
*/
|
|
580
|
-
|
|
670
|
+
concurrency?: number;
|
|
671
|
+
/**
|
|
672
|
+
* Clock used independently for claiming, renewal, settlement, and retry
|
|
673
|
+
* scheduling. Defaults to the system clock.
|
|
674
|
+
*/
|
|
675
|
+
now?: () => Date;
|
|
581
676
|
/**
|
|
582
677
|
* Claim lease duration in milliseconds.
|
|
583
678
|
*/
|
|
584
679
|
leaseMs?: number;
|
|
680
|
+
/**
|
|
681
|
+
* Interval between serialized claim renewals. Defaults to one third of the
|
|
682
|
+
* lease duration and must remain shorter than the lease.
|
|
683
|
+
*/
|
|
684
|
+
heartbeatMs?: number;
|
|
685
|
+
/**
|
|
686
|
+
* Maximum time Beignet renews a claim for one delivery. When exceeded, the
|
|
687
|
+
* drain stops renewing and leaves final recovery to the last lease expiry.
|
|
688
|
+
*/
|
|
689
|
+
maxActiveMs?: number;
|
|
690
|
+
/**
|
|
691
|
+
* Abort-aware wait implementation. Inject a deterministic implementation in
|
|
692
|
+
* tests; production callers normally use the default timer.
|
|
693
|
+
*/
|
|
694
|
+
wait?: OutboxDrainWait;
|
|
585
695
|
/**
|
|
586
696
|
* Retry delay in milliseconds or function for per-message delay.
|
|
587
697
|
*/
|
|
@@ -613,19 +723,17 @@ export interface DrainOutboxOptions {
|
|
|
613
723
|
* Observer called after a failed delivery is successfully moved to the dead
|
|
614
724
|
* letter state. Observer failures are ignored.
|
|
615
725
|
*/
|
|
616
|
-
onDeadLetter?: (
|
|
617
|
-
error: unknown,
|
|
618
|
-
message: ClaimedOutboxMessage,
|
|
619
|
-
) => MaybePromise<void>;
|
|
726
|
+
onDeadLetter?: (error: unknown, message: OutboxMessage) => MaybePromise<void>;
|
|
620
727
|
/**
|
|
621
|
-
* Observer called when
|
|
622
|
-
*
|
|
728
|
+
* Observer called when claim renewal degrades or ownership is lost.
|
|
729
|
+
* Observer failures are ignored.
|
|
623
730
|
*/
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
731
|
+
onLeaseError?: (failure: OutboxLeaseFailure) => MaybePromise<void>;
|
|
732
|
+
/**
|
|
733
|
+
* Observer called when a delivery outcome cannot be settled durably.
|
|
734
|
+
* Observer failures are ignored.
|
|
735
|
+
*/
|
|
736
|
+
onSettlementError?: (failure: OutboxSettlementFailure) => MaybePromise<void>;
|
|
629
737
|
}
|
|
630
738
|
|
|
631
739
|
/**
|
|
@@ -648,6 +756,15 @@ export interface DrainOutboxResult {
|
|
|
648
756
|
* Messages moved to dead letter state.
|
|
649
757
|
*/
|
|
650
758
|
deadLettered: number;
|
|
759
|
+
/**
|
|
760
|
+
* Dead-lettered messages whose claim attempt budget was already exhausted.
|
|
761
|
+
* This is a subset of `deadLettered`.
|
|
762
|
+
*/
|
|
763
|
+
abandonedDeadLettered: number;
|
|
764
|
+
/** Messages delivered or failed whose final storage state is unknown. */
|
|
765
|
+
settlementFailed: number;
|
|
766
|
+
/** Messages whose active claim could no longer be confirmed. */
|
|
767
|
+
leaseLost: number;
|
|
651
768
|
}
|
|
652
769
|
|
|
653
770
|
/**
|
|
@@ -686,6 +803,62 @@ export class OutboxClaimError extends Error {
|
|
|
686
803
|
}
|
|
687
804
|
}
|
|
688
805
|
|
|
806
|
+
/**
|
|
807
|
+
* Error persisted when an eligible message has exhausted its claim attempts
|
|
808
|
+
* without reaching a terminal settlement.
|
|
809
|
+
*/
|
|
810
|
+
export class OutboxAbandonedClaimError extends Error {
|
|
811
|
+
/** Message ID whose attempt budget was exhausted. */
|
|
812
|
+
readonly id: string;
|
|
813
|
+
/** Number of claims already made. */
|
|
814
|
+
readonly attempts: number;
|
|
815
|
+
/** Maximum permitted claim attempts. */
|
|
816
|
+
readonly maxAttempts: number;
|
|
817
|
+
|
|
818
|
+
constructor(args: { id: string; attempts: number; maxAttempts: number }) {
|
|
819
|
+
super(
|
|
820
|
+
`Outbox message "${args.id}" exhausted ${args.maxAttempts} claim attempts without a terminal settlement.`,
|
|
821
|
+
);
|
|
822
|
+
this.name = "OutboxAbandonedClaimError";
|
|
823
|
+
this.id = args.id;
|
|
824
|
+
this.attempts = args.attempts;
|
|
825
|
+
this.maxAttempts = args.maxAttempts;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/** Error surfaced when an outbox worker no longer has a confirmed claim. */
|
|
830
|
+
export class OutboxLeaseLostError extends Error {
|
|
831
|
+
/** Message ID whose ownership was lost. */
|
|
832
|
+
readonly id: string;
|
|
833
|
+
|
|
834
|
+
constructor(args: { id: string; message?: string; cause?: unknown }) {
|
|
835
|
+
super(
|
|
836
|
+
args.message ??
|
|
837
|
+
`Outbox claim for message "${args.id}" is no longer active.`,
|
|
838
|
+
args.cause === undefined ? undefined : { cause: args.cause },
|
|
839
|
+
);
|
|
840
|
+
this.name = "OutboxLeaseLostError";
|
|
841
|
+
this.id = args.id;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** Error surfaced when delivery exceeds the configured active-claim window. */
|
|
846
|
+
export class OutboxMaxActiveDurationError extends Error {
|
|
847
|
+
/** Message ID whose active window elapsed. */
|
|
848
|
+
readonly id: string;
|
|
849
|
+
/** Configured maximum active duration. */
|
|
850
|
+
readonly maxActiveMs: number;
|
|
851
|
+
|
|
852
|
+
constructor(args: { id: string; maxActiveMs: number }) {
|
|
853
|
+
super(
|
|
854
|
+
`Outbox delivery for message "${args.id}" exceeded the ${args.maxActiveMs}ms active-claim limit.`,
|
|
855
|
+
);
|
|
856
|
+
this.name = "OutboxMaxActiveDurationError";
|
|
857
|
+
this.id = args.id;
|
|
858
|
+
this.maxActiveMs = args.maxActiveMs;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
|
|
689
862
|
/**
|
|
690
863
|
* Error thrown when an outbox admin operation cannot be completed safely.
|
|
691
864
|
*/
|
|
@@ -1169,9 +1342,11 @@ export function createMemoryOutbox(
|
|
|
1169
1342
|
assertPositiveInteger("limit", options.limit);
|
|
1170
1343
|
const now = options.now ?? storeNow();
|
|
1171
1344
|
const leaseMs = options.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1345
|
+
assertValidDate("now", now);
|
|
1172
1346
|
assertPositiveInteger("leaseMs", leaseMs);
|
|
1173
1347
|
const lockedUntil = new Date(now.getTime() + leaseMs);
|
|
1174
1348
|
const claimed: ClaimedOutboxMessage[] = [];
|
|
1349
|
+
const deadLettered: OutboxMessage[] = [];
|
|
1175
1350
|
|
|
1176
1351
|
const eligible = [...messages.values()]
|
|
1177
1352
|
.filter((message) => isEligible(message, now))
|
|
@@ -1183,6 +1358,23 @@ export function createMemoryOutbox(
|
|
|
1183
1358
|
.slice(0, options.limit);
|
|
1184
1359
|
|
|
1185
1360
|
for (const message of eligible) {
|
|
1361
|
+
if (message.attempts >= message.maxAttempts) {
|
|
1362
|
+
message.status = "deadLettered";
|
|
1363
|
+
message.lastError = serializeOutboxError(
|
|
1364
|
+
new OutboxAbandonedClaimError({
|
|
1365
|
+
id: message.id,
|
|
1366
|
+
attempts: message.attempts,
|
|
1367
|
+
maxAttempts: message.maxAttempts,
|
|
1368
|
+
}),
|
|
1369
|
+
);
|
|
1370
|
+
message.claimToken = null;
|
|
1371
|
+
message.claimedAt = null;
|
|
1372
|
+
message.lockedUntil = null;
|
|
1373
|
+
message.updatedAt = cloneDate(now);
|
|
1374
|
+
deadLettered.push(copyMessage(message));
|
|
1375
|
+
continue;
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1186
1378
|
message.status = "claimed";
|
|
1187
1379
|
message.attempts += 1;
|
|
1188
1380
|
message.claimToken = createStoreId();
|
|
@@ -1192,7 +1384,31 @@ export function createMemoryOutbox(
|
|
|
1192
1384
|
claimed.push(toClaimedMessage(message));
|
|
1193
1385
|
}
|
|
1194
1386
|
|
|
1195
|
-
return claimed;
|
|
1387
|
+
return { claimed, deadLettered };
|
|
1388
|
+
},
|
|
1389
|
+
|
|
1390
|
+
async renewClaim(input) {
|
|
1391
|
+
assertNonEmptyString("id", input.id);
|
|
1392
|
+
assertNonEmptyString("claimToken", input.claimToken);
|
|
1393
|
+
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1394
|
+
const now = input.now ?? storeNow();
|
|
1395
|
+
const leaseMs = input.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1396
|
+
assertValidDate("now", now);
|
|
1397
|
+
assertPositiveInteger("leaseMs", leaseMs);
|
|
1398
|
+
if (
|
|
1399
|
+
message.lockedUntil === null ||
|
|
1400
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1401
|
+
) {
|
|
1402
|
+
throw new OutboxClaimError({
|
|
1403
|
+
id: input.id,
|
|
1404
|
+
message: `Outbox message "${input.id}" no longer has an active claim to renew.`,
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
const lockedUntil = new Date(now.getTime() + leaseMs);
|
|
1409
|
+
message.lockedUntil = cloneDate(lockedUntil);
|
|
1410
|
+
message.updatedAt = cloneDate(now);
|
|
1411
|
+
return { lockedUntil: cloneDate(lockedUntil) };
|
|
1196
1412
|
},
|
|
1197
1413
|
|
|
1198
1414
|
async markDelivered(input) {
|
|
@@ -1200,6 +1416,16 @@ export function createMemoryOutbox(
|
|
|
1200
1416
|
assertNonEmptyString("claimToken", input.claimToken);
|
|
1201
1417
|
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1202
1418
|
const now = input.now ?? storeNow();
|
|
1419
|
+
assertValidDate("now", now);
|
|
1420
|
+
if (
|
|
1421
|
+
message.lockedUntil === null ||
|
|
1422
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1423
|
+
) {
|
|
1424
|
+
throw new OutboxClaimError({
|
|
1425
|
+
id: input.id,
|
|
1426
|
+
message: `Outbox message "${input.id}" no longer has an active claim to settle.`,
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1203
1429
|
|
|
1204
1430
|
message.status = "delivered";
|
|
1205
1431
|
message.deliveredAt = cloneDate(now);
|
|
@@ -1214,6 +1440,17 @@ export function createMemoryOutbox(
|
|
|
1214
1440
|
assertNonEmptyString("claimToken", input.claimToken);
|
|
1215
1441
|
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1216
1442
|
const now = input.now ?? storeNow();
|
|
1443
|
+
assertValidDate("now", now);
|
|
1444
|
+
if (input.retryAt) assertValidDate("retryAt", input.retryAt);
|
|
1445
|
+
if (
|
|
1446
|
+
message.lockedUntil === null ||
|
|
1447
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1448
|
+
) {
|
|
1449
|
+
throw new OutboxClaimError({
|
|
1450
|
+
id: input.id,
|
|
1451
|
+
message: `Outbox message "${input.id}" no longer has an active claim to settle.`,
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1217
1454
|
|
|
1218
1455
|
message.status = input.deadLetter ? "deadLettered" : "pending";
|
|
1219
1456
|
message.lastError = serializeOutboxError(input.error);
|
|
@@ -1271,7 +1508,16 @@ export async function enqueueEvent<E extends EventPayloadDef>(
|
|
|
1271
1508
|
payload: InferEventPayload<E>,
|
|
1272
1509
|
options: EnqueueTypedOutboxOptions = {},
|
|
1273
1510
|
): Promise<OutboxMessage> {
|
|
1274
|
-
await parseEventPayload(event, payload);
|
|
1511
|
+
const parsed = await parseEventPayload(event, payload);
|
|
1512
|
+
return await enqueueParsedEvent(outbox, event, parsed, options);
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
async function enqueueParsedEvent<E extends EventPayloadDef>(
|
|
1516
|
+
outbox: OutboxPort,
|
|
1517
|
+
event: E,
|
|
1518
|
+
payload: InferEventPayload<E>,
|
|
1519
|
+
options: EnqueueTypedOutboxOptions,
|
|
1520
|
+
): Promise<OutboxMessage> {
|
|
1275
1521
|
const trace =
|
|
1276
1522
|
parseTraceCarrier(options.trace) ?? captureTraceCarrier(options.tracing);
|
|
1277
1523
|
return outbox.enqueue({
|
|
@@ -1317,7 +1563,10 @@ export function createOutboxEventRecorder(
|
|
|
1317
1563
|
): DomainEventRecorderPort {
|
|
1318
1564
|
return {
|
|
1319
1565
|
async record(event, payload, publishOptions) {
|
|
1320
|
-
|
|
1566
|
+
const parsed = isEventPayloadParsed(publishOptions)
|
|
1567
|
+
? payload
|
|
1568
|
+
: await parseEventPayload(event, payload);
|
|
1569
|
+
await enqueueParsedEvent(outbox, event, parsed, {
|
|
1321
1570
|
...options,
|
|
1322
1571
|
trace: publishOptions?.trace ?? options.trace,
|
|
1323
1572
|
});
|
|
@@ -1392,7 +1641,7 @@ function shouldRetryOutboxMessage(
|
|
|
1392
1641
|
}
|
|
1393
1642
|
|
|
1394
1643
|
function outboxInstrumentationDetails(
|
|
1395
|
-
message:
|
|
1644
|
+
message: OutboxMessage,
|
|
1396
1645
|
details?: Record<string, unknown>,
|
|
1397
1646
|
): Record<string, unknown> {
|
|
1398
1647
|
return {
|
|
@@ -1424,11 +1673,11 @@ async function deliverOutboxMessage(
|
|
|
1424
1673
|
);
|
|
1425
1674
|
}
|
|
1426
1675
|
|
|
1427
|
-
await parseEventPayload(event, message.payload);
|
|
1676
|
+
const payload = await parseEventPayload(event, message.payload);
|
|
1428
1677
|
await options.eventBus.publish(
|
|
1429
1678
|
event,
|
|
1430
|
-
|
|
1431
|
-
trace ? { trace } : undefined,
|
|
1679
|
+
payload,
|
|
1680
|
+
markEventPayloadParsed(trace ? { trace } : undefined),
|
|
1432
1681
|
);
|
|
1433
1682
|
return;
|
|
1434
1683
|
}
|
|
@@ -1474,20 +1723,1033 @@ async function deliverOutboxMessage(
|
|
|
1474
1723
|
);
|
|
1475
1724
|
}
|
|
1476
1725
|
|
|
1726
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
1727
|
+
const OUTBOX_SETTLEMENT_ATTEMPTS = 3;
|
|
1728
|
+
const OUTBOX_SETTLEMENT_RETRY_DELAY_MS = 100;
|
|
1729
|
+
|
|
1730
|
+
type ResolvedDrainRuntime = {
|
|
1731
|
+
batchSize: number;
|
|
1732
|
+
concurrency: number;
|
|
1733
|
+
leaseMs: number;
|
|
1734
|
+
heartbeatMs: number;
|
|
1735
|
+
maxActiveMs: number;
|
|
1736
|
+
now: () => Date;
|
|
1737
|
+
wait: OutboxDrainWait;
|
|
1738
|
+
};
|
|
1739
|
+
|
|
1740
|
+
type MessageDrainOutcome = {
|
|
1741
|
+
delivered: number;
|
|
1742
|
+
retried: number;
|
|
1743
|
+
deadLettered: number;
|
|
1744
|
+
settlementFailed: number;
|
|
1745
|
+
leaseLost: number;
|
|
1746
|
+
};
|
|
1747
|
+
|
|
1748
|
+
type ClaimHeartbeat = {
|
|
1749
|
+
lost: Promise<OutboxLeaseFailure>;
|
|
1750
|
+
stopAndExtend(): Promise<{
|
|
1751
|
+
lockedUntil: Date;
|
|
1752
|
+
renewalError?: unknown;
|
|
1753
|
+
renewalRecovered?: boolean;
|
|
1754
|
+
failure?: OutboxLeaseFailure;
|
|
1755
|
+
}>;
|
|
1756
|
+
stop(): Promise<void>;
|
|
1757
|
+
};
|
|
1758
|
+
|
|
1759
|
+
type BoundedRenewalResult =
|
|
1760
|
+
| { kind: "succeeded"; lockedUntil: Date }
|
|
1761
|
+
| { kind: "failed"; error: unknown }
|
|
1762
|
+
| { kind: "deadline" }
|
|
1763
|
+
| { kind: "stopped" }
|
|
1764
|
+
| { kind: "waitFailed"; error: unknown };
|
|
1765
|
+
|
|
1766
|
+
type BoundedSettlementResult =
|
|
1767
|
+
| { kind: "succeeded" }
|
|
1768
|
+
| { kind: "failed"; error: unknown }
|
|
1769
|
+
| { kind: "deadline" }
|
|
1770
|
+
| { kind: "waitFailed"; error: unknown };
|
|
1771
|
+
|
|
1772
|
+
function defaultOutboxWait(
|
|
1773
|
+
delayMs: number,
|
|
1774
|
+
signal: AbortSignal,
|
|
1775
|
+
): Promise<void> {
|
|
1776
|
+
return new Promise((resolve) => {
|
|
1777
|
+
if (signal.aborted) {
|
|
1778
|
+
resolve();
|
|
1779
|
+
return;
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1783
|
+
const finish = () => {
|
|
1784
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1785
|
+
signal.removeEventListener("abort", finish);
|
|
1786
|
+
resolve();
|
|
1787
|
+
};
|
|
1788
|
+
timer = setTimeout(finish, delayMs);
|
|
1789
|
+
signal.addEventListener("abort", finish, { once: true });
|
|
1790
|
+
});
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
function readOutboxNow(now: () => Date): Date {
|
|
1794
|
+
const value = now();
|
|
1795
|
+
assertValidDate("now", value);
|
|
1796
|
+
return value;
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
function assertTimerDuration(name: string, value: number): void {
|
|
1800
|
+
assertPositiveInteger(name, value);
|
|
1801
|
+
if (value > MAX_TIMER_DELAY_MS) {
|
|
1802
|
+
throw new Error(`${name} must be at most ${MAX_TIMER_DELAY_MS}`);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
function resolveDrainRuntime(
|
|
1807
|
+
options: DrainOutboxOptions,
|
|
1808
|
+
): ResolvedDrainRuntime {
|
|
1809
|
+
const batchSize = options.batchSize ?? DEFAULT_OUTBOX_BATCH_SIZE;
|
|
1810
|
+
const concurrency = options.concurrency ?? DEFAULT_OUTBOX_CONCURRENCY;
|
|
1811
|
+
const leaseMs = options.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1812
|
+
const heartbeatMs =
|
|
1813
|
+
options.heartbeatMs ?? Math.max(1, Math.floor(leaseMs / 3));
|
|
1814
|
+
const maxActiveMs = options.maxActiveMs ?? DEFAULT_OUTBOX_MAX_ACTIVE_MS;
|
|
1815
|
+
|
|
1816
|
+
assertPositiveInteger("batchSize", batchSize);
|
|
1817
|
+
assertPositiveInteger("concurrency", concurrency);
|
|
1818
|
+
if (concurrency > batchSize) {
|
|
1819
|
+
throw new Error("concurrency must be less than or equal to batchSize");
|
|
1820
|
+
}
|
|
1821
|
+
assertTimerDuration("leaseMs", leaseMs);
|
|
1822
|
+
if (leaseMs < 2) throw new Error("leaseMs must be at least 2");
|
|
1823
|
+
assertTimerDuration("heartbeatMs", heartbeatMs);
|
|
1824
|
+
if (heartbeatMs >= leaseMs) {
|
|
1825
|
+
throw new Error("heartbeatMs must be shorter than leaseMs");
|
|
1826
|
+
}
|
|
1827
|
+
assertTimerDuration("maxActiveMs", maxActiveMs);
|
|
1828
|
+
|
|
1829
|
+
return {
|
|
1830
|
+
batchSize,
|
|
1831
|
+
concurrency,
|
|
1832
|
+
leaseMs,
|
|
1833
|
+
heartbeatMs,
|
|
1834
|
+
maxActiveMs,
|
|
1835
|
+
now: options.now ?? (() => new Date()),
|
|
1836
|
+
wait: options.wait ?? defaultOutboxWait,
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
async function notifyLeaseFailure(
|
|
1841
|
+
options: DrainOutboxOptions,
|
|
1842
|
+
failure: OutboxLeaseFailure,
|
|
1843
|
+
): Promise<void> {
|
|
1844
|
+
try {
|
|
1845
|
+
await options.onLeaseError?.(failure);
|
|
1846
|
+
} catch {
|
|
1847
|
+
// Lease observers cannot change delivery ownership or recovery behavior.
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
async function notifySettlementFailure(
|
|
1852
|
+
options: DrainOutboxOptions,
|
|
1853
|
+
failure: OutboxSettlementFailure,
|
|
1854
|
+
): Promise<void> {
|
|
1855
|
+
try {
|
|
1856
|
+
await options.onSettlementError?.(failure);
|
|
1857
|
+
} catch {
|
|
1858
|
+
// Settlement observers cannot replace the unknown storage outcome.
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
async function runBoundedClaimRenewal(options: {
|
|
1863
|
+
outbox: OutboxPort;
|
|
1864
|
+
message: ClaimedOutboxMessage;
|
|
1865
|
+
runtime: ResolvedDrainRuntime;
|
|
1866
|
+
renewalAt: Date;
|
|
1867
|
+
deadline: Date;
|
|
1868
|
+
signal: AbortSignal;
|
|
1869
|
+
}): Promise<BoundedRenewalResult> {
|
|
1870
|
+
const remainingMs = options.deadline.getTime() - options.renewalAt.getTime();
|
|
1871
|
+
if (remainingMs <= 0) return { kind: "deadline" };
|
|
1872
|
+
if (options.signal.aborted) return { kind: "stopped" };
|
|
1873
|
+
|
|
1874
|
+
const waitController = new AbortController();
|
|
1875
|
+
const stopWaiting = () => waitController.abort();
|
|
1876
|
+
options.signal.addEventListener("abort", stopWaiting, { once: true });
|
|
1877
|
+
|
|
1878
|
+
const renewal = Promise.resolve()
|
|
1879
|
+
.then(() =>
|
|
1880
|
+
options.outbox.renewClaim({
|
|
1881
|
+
id: options.message.id,
|
|
1882
|
+
claimToken: options.message.claimToken,
|
|
1883
|
+
now: options.renewalAt,
|
|
1884
|
+
leaseMs: options.runtime.leaseMs,
|
|
1885
|
+
}),
|
|
1886
|
+
)
|
|
1887
|
+
.then(
|
|
1888
|
+
(result) => ({ kind: "succeeded" as const, result }),
|
|
1889
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
1890
|
+
);
|
|
1891
|
+
const deadline = Promise.resolve()
|
|
1892
|
+
.then(() => options.runtime.wait(remainingMs, waitController.signal))
|
|
1893
|
+
.then(
|
|
1894
|
+
() =>
|
|
1895
|
+
options.signal.aborted
|
|
1896
|
+
? { kind: "stopped" as const }
|
|
1897
|
+
: { kind: "deadline" as const },
|
|
1898
|
+
(error: unknown) =>
|
|
1899
|
+
options.signal.aborted
|
|
1900
|
+
? { kind: "stopped" as const }
|
|
1901
|
+
: { kind: "waitFailed" as const, error },
|
|
1902
|
+
);
|
|
1903
|
+
|
|
1904
|
+
const result = await Promise.race([renewal, deadline]);
|
|
1905
|
+
options.signal.removeEventListener("abort", stopWaiting);
|
|
1906
|
+
waitController.abort();
|
|
1907
|
+
|
|
1908
|
+
if (result.kind !== "succeeded") return result;
|
|
1909
|
+
try {
|
|
1910
|
+
const completedAt = readOutboxNow(options.runtime.now);
|
|
1911
|
+
if (completedAt.getTime() >= options.deadline.getTime()) {
|
|
1912
|
+
return { kind: "deadline" };
|
|
1913
|
+
}
|
|
1914
|
+
assertValidDate("lockedUntil", result.result.lockedUntil);
|
|
1915
|
+
if (result.result.lockedUntil.getTime() <= completedAt.getTime()) {
|
|
1916
|
+
return {
|
|
1917
|
+
kind: "failed",
|
|
1918
|
+
error: new Error(
|
|
1919
|
+
`Outbox claim renewal for message "${options.message.id}" returned an expired lease.`,
|
|
1920
|
+
),
|
|
1921
|
+
};
|
|
1922
|
+
}
|
|
1923
|
+
return {
|
|
1924
|
+
kind: "succeeded",
|
|
1925
|
+
lockedUntil: cloneDate(result.result.lockedUntil),
|
|
1926
|
+
};
|
|
1927
|
+
} catch (error) {
|
|
1928
|
+
return { kind: "failed", error };
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
function createClaimHeartbeat(
|
|
1933
|
+
options: DrainOutboxOptions,
|
|
1934
|
+
runtime: ResolvedDrainRuntime,
|
|
1935
|
+
message: ClaimedOutboxMessage,
|
|
1936
|
+
activeUntil: Date,
|
|
1937
|
+
): ClaimHeartbeat {
|
|
1938
|
+
const controller = new AbortController();
|
|
1939
|
+
let stopped = false;
|
|
1940
|
+
let lockedUntil = cloneDate(message.lockedUntil);
|
|
1941
|
+
let renewing = false;
|
|
1942
|
+
let firstRenewalError: unknown;
|
|
1943
|
+
let terminalFailure: OutboxLeaseFailure | undefined;
|
|
1944
|
+
let resolveLost: (failure: OutboxLeaseFailure) => void = () => {};
|
|
1945
|
+
const lost = new Promise<OutboxLeaseFailure>((resolve) => {
|
|
1946
|
+
resolveLost = resolve;
|
|
1947
|
+
});
|
|
1948
|
+
|
|
1949
|
+
const fail = (failure: OutboxLeaseFailure) => {
|
|
1950
|
+
if (terminalFailure) return;
|
|
1951
|
+
terminalFailure = failure;
|
|
1952
|
+
stopped = true;
|
|
1953
|
+
controller.abort();
|
|
1954
|
+
resolveLost(failure);
|
|
1955
|
+
};
|
|
1956
|
+
|
|
1957
|
+
const run = async () => {
|
|
1958
|
+
try {
|
|
1959
|
+
let delayMs = runtime.heartbeatMs;
|
|
1960
|
+
while (!stopped) {
|
|
1961
|
+
try {
|
|
1962
|
+
await runtime.wait(delayMs, controller.signal);
|
|
1963
|
+
} catch (error) {
|
|
1964
|
+
if (stopped || controller.signal.aborted) return;
|
|
1965
|
+
fail({
|
|
1966
|
+
error: new OutboxLeaseLostError({
|
|
1967
|
+
id: message.id,
|
|
1968
|
+
message: `Outbox heartbeat scheduling failed for message "${message.id}".`,
|
|
1969
|
+
cause: error,
|
|
1970
|
+
}),
|
|
1971
|
+
message,
|
|
1972
|
+
operation: "renewClaim",
|
|
1973
|
+
state: "lost",
|
|
1974
|
+
confirmedLost: false,
|
|
1975
|
+
});
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
if (stopped || controller.signal.aborted) return;
|
|
1979
|
+
|
|
1980
|
+
const renewalAt = readOutboxNow(runtime.now);
|
|
1981
|
+
const renewalDeadline = new Date(
|
|
1982
|
+
Math.min(lockedUntil.getTime(), activeUntil.getTime()),
|
|
1983
|
+
);
|
|
1984
|
+
const activeDeadlineEndsFirst =
|
|
1985
|
+
activeUntil.getTime() <= lockedUntil.getTime();
|
|
1986
|
+
if (renewalAt.getTime() >= renewalDeadline.getTime()) {
|
|
1987
|
+
fail({
|
|
1988
|
+
error: activeDeadlineEndsFirst
|
|
1989
|
+
? new OutboxMaxActiveDurationError({
|
|
1990
|
+
id: message.id,
|
|
1991
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
1992
|
+
})
|
|
1993
|
+
: new OutboxLeaseLostError({ id: message.id }),
|
|
1994
|
+
message,
|
|
1995
|
+
operation: activeDeadlineEndsFirst
|
|
1996
|
+
? "maxActiveDuration"
|
|
1997
|
+
: "renewClaim",
|
|
1998
|
+
state: "lost",
|
|
1999
|
+
confirmedLost: !activeDeadlineEndsFirst,
|
|
2000
|
+
});
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
renewing = true;
|
|
2005
|
+
let renewal: BoundedRenewalResult;
|
|
2006
|
+
try {
|
|
2007
|
+
renewal = await runBoundedClaimRenewal({
|
|
2008
|
+
outbox: options.outbox,
|
|
2009
|
+
message,
|
|
2010
|
+
runtime,
|
|
2011
|
+
renewalAt,
|
|
2012
|
+
deadline: renewalDeadline,
|
|
2013
|
+
signal: controller.signal,
|
|
2014
|
+
});
|
|
2015
|
+
} catch (error) {
|
|
2016
|
+
renewal = { kind: "failed", error };
|
|
2017
|
+
} finally {
|
|
2018
|
+
renewing = false;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
if (renewal.kind === "stopped") return;
|
|
2022
|
+
if (renewal.kind === "waitFailed") {
|
|
2023
|
+
fail({
|
|
2024
|
+
error: new OutboxLeaseLostError({
|
|
2025
|
+
id: message.id,
|
|
2026
|
+
message: `Could not enforce the renewal deadline for outbox message "${message.id}".`,
|
|
2027
|
+
cause: renewal.error,
|
|
2028
|
+
}),
|
|
2029
|
+
message,
|
|
2030
|
+
operation: "renewClaim",
|
|
2031
|
+
state: "lost",
|
|
2032
|
+
confirmedLost: false,
|
|
2033
|
+
});
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
if (renewal.kind === "deadline") {
|
|
2037
|
+
fail({
|
|
2038
|
+
error: activeDeadlineEndsFirst
|
|
2039
|
+
? new OutboxMaxActiveDurationError({
|
|
2040
|
+
id: message.id,
|
|
2041
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2042
|
+
})
|
|
2043
|
+
: new OutboxLeaseLostError({
|
|
2044
|
+
id: message.id,
|
|
2045
|
+
message: `Outbox claim renewal for message "${message.id}" did not complete before the lease deadline.`,
|
|
2046
|
+
}),
|
|
2047
|
+
message,
|
|
2048
|
+
operation: activeDeadlineEndsFirst
|
|
2049
|
+
? "maxActiveDuration"
|
|
2050
|
+
: "renewClaim",
|
|
2051
|
+
state: "lost",
|
|
2052
|
+
confirmedLost: false,
|
|
2053
|
+
});
|
|
2054
|
+
return;
|
|
2055
|
+
}
|
|
2056
|
+
if (renewal.kind === "succeeded") {
|
|
2057
|
+
lockedUntil = renewal.lockedUntil;
|
|
2058
|
+
delayMs = runtime.heartbeatMs;
|
|
2059
|
+
continue;
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
const error = renewal.error;
|
|
2063
|
+
if (error instanceof OutboxClaimError) {
|
|
2064
|
+
fail({
|
|
2065
|
+
error,
|
|
2066
|
+
message,
|
|
2067
|
+
operation: "renewClaim",
|
|
2068
|
+
state: "lost",
|
|
2069
|
+
confirmedLost: true,
|
|
2070
|
+
});
|
|
2071
|
+
return;
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
firstRenewalError ??= error;
|
|
2075
|
+
if (stopped) return;
|
|
2076
|
+
const retryAt = readOutboxNow(runtime.now);
|
|
2077
|
+
const remainingMs = renewalDeadline.getTime() - retryAt.getTime();
|
|
2078
|
+
if (remainingMs <= 0) {
|
|
2079
|
+
fail({
|
|
2080
|
+
error: activeDeadlineEndsFirst
|
|
2081
|
+
? new OutboxMaxActiveDurationError({
|
|
2082
|
+
id: message.id,
|
|
2083
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2084
|
+
})
|
|
2085
|
+
: new OutboxLeaseLostError({
|
|
2086
|
+
id: message.id,
|
|
2087
|
+
message: `Outbox claim renewal for message "${message.id}" did not recover before the lease expired.`,
|
|
2088
|
+
cause: error,
|
|
2089
|
+
}),
|
|
2090
|
+
message,
|
|
2091
|
+
operation: activeDeadlineEndsFirst
|
|
2092
|
+
? "maxActiveDuration"
|
|
2093
|
+
: "renewClaim",
|
|
2094
|
+
state: "lost",
|
|
2095
|
+
confirmedLost: false,
|
|
2096
|
+
});
|
|
2097
|
+
return;
|
|
2098
|
+
}
|
|
2099
|
+
delayMs = Math.max(
|
|
2100
|
+
1,
|
|
2101
|
+
Math.min(runtime.heartbeatMs, Math.floor(remainingMs / 3)),
|
|
2102
|
+
);
|
|
2103
|
+
}
|
|
2104
|
+
} catch (error) {
|
|
2105
|
+
if (stopped || controller.signal.aborted) return;
|
|
2106
|
+
fail({
|
|
2107
|
+
error: new OutboxLeaseLostError({
|
|
2108
|
+
id: message.id,
|
|
2109
|
+
message: `Outbox claim renewal failed unexpectedly for message "${message.id}".`,
|
|
2110
|
+
cause: error,
|
|
2111
|
+
}),
|
|
2112
|
+
message,
|
|
2113
|
+
operation: "renewClaim",
|
|
2114
|
+
state: "lost",
|
|
2115
|
+
confirmedLost: false,
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
};
|
|
2119
|
+
|
|
2120
|
+
const running = run();
|
|
2121
|
+
|
|
2122
|
+
const stop = async (abortRenewal = true) => {
|
|
2123
|
+
stopped = true;
|
|
2124
|
+
if (abortRenewal || !renewing) controller.abort();
|
|
2125
|
+
await running;
|
|
2126
|
+
controller.abort();
|
|
2127
|
+
};
|
|
2128
|
+
|
|
2129
|
+
return {
|
|
2130
|
+
lost,
|
|
2131
|
+
stop: () => stop(true),
|
|
2132
|
+
async stopAndExtend() {
|
|
2133
|
+
await stop(false);
|
|
2134
|
+
if (terminalFailure) {
|
|
2135
|
+
return { lockedUntil, failure: terminalFailure };
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
const renewalAt = readOutboxNow(runtime.now);
|
|
2139
|
+
const activeDeadlineEndsFirst =
|
|
2140
|
+
activeUntil.getTime() <= lockedUntil.getTime();
|
|
2141
|
+
const finalRenewalDeadline = new Date(
|
|
2142
|
+
Math.min(lockedUntil.getTime(), activeUntil.getTime()),
|
|
2143
|
+
);
|
|
2144
|
+
if (renewalAt.getTime() >= finalRenewalDeadline.getTime()) {
|
|
2145
|
+
return {
|
|
2146
|
+
lockedUntil,
|
|
2147
|
+
failure: {
|
|
2148
|
+
error: activeDeadlineEndsFirst
|
|
2149
|
+
? new OutboxMaxActiveDurationError({
|
|
2150
|
+
id: message.id,
|
|
2151
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2152
|
+
})
|
|
2153
|
+
: new OutboxLeaseLostError({ id: message.id }),
|
|
2154
|
+
message,
|
|
2155
|
+
operation: activeDeadlineEndsFirst
|
|
2156
|
+
? "maxActiveDuration"
|
|
2157
|
+
: "renewClaim",
|
|
2158
|
+
state: "lost",
|
|
2159
|
+
confirmedLost: !activeDeadlineEndsFirst,
|
|
2160
|
+
},
|
|
2161
|
+
};
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
const finalController = new AbortController();
|
|
2165
|
+
const renewal = await runBoundedClaimRenewal({
|
|
2166
|
+
outbox: options.outbox,
|
|
2167
|
+
message,
|
|
2168
|
+
runtime,
|
|
2169
|
+
renewalAt,
|
|
2170
|
+
deadline: finalRenewalDeadline,
|
|
2171
|
+
signal: finalController.signal,
|
|
2172
|
+
});
|
|
2173
|
+
finalController.abort();
|
|
2174
|
+
|
|
2175
|
+
if (renewal.kind === "succeeded") {
|
|
2176
|
+
return {
|
|
2177
|
+
lockedUntil: renewal.lockedUntil,
|
|
2178
|
+
renewalError: firstRenewalError,
|
|
2179
|
+
renewalRecovered: firstRenewalError === undefined ? undefined : true,
|
|
2180
|
+
};
|
|
2181
|
+
}
|
|
2182
|
+
if (renewal.kind === "failed") {
|
|
2183
|
+
if (!(renewal.error instanceof OutboxClaimError)) {
|
|
2184
|
+
const failedAt = readOutboxNow(runtime.now);
|
|
2185
|
+
if (
|
|
2186
|
+
failedAt.getTime() < lockedUntil.getTime() &&
|
|
2187
|
+
failedAt.getTime() < activeUntil.getTime()
|
|
2188
|
+
) {
|
|
2189
|
+
return {
|
|
2190
|
+
lockedUntil,
|
|
2191
|
+
renewalError: renewal.error,
|
|
2192
|
+
renewalRecovered: false,
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
if (
|
|
2196
|
+
activeDeadlineEndsFirst &&
|
|
2197
|
+
failedAt.getTime() >= activeUntil.getTime()
|
|
2198
|
+
) {
|
|
2199
|
+
return {
|
|
2200
|
+
lockedUntil,
|
|
2201
|
+
failure: {
|
|
2202
|
+
error: new OutboxMaxActiveDurationError({
|
|
2203
|
+
id: message.id,
|
|
2204
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2205
|
+
}),
|
|
2206
|
+
message,
|
|
2207
|
+
operation: "maxActiveDuration",
|
|
2208
|
+
state: "lost",
|
|
2209
|
+
confirmedLost: false,
|
|
2210
|
+
},
|
|
2211
|
+
};
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
return {
|
|
2215
|
+
lockedUntil,
|
|
2216
|
+
failure: {
|
|
2217
|
+
error: renewal.error,
|
|
2218
|
+
message,
|
|
2219
|
+
operation: "renewClaim",
|
|
2220
|
+
state: "lost",
|
|
2221
|
+
confirmedLost: renewal.error instanceof OutboxClaimError,
|
|
2222
|
+
},
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
|
|
2226
|
+
return {
|
|
2227
|
+
lockedUntil,
|
|
2228
|
+
failure: {
|
|
2229
|
+
error:
|
|
2230
|
+
renewal.kind === "waitFailed"
|
|
2231
|
+
? new OutboxLeaseLostError({
|
|
2232
|
+
id: message.id,
|
|
2233
|
+
message: activeDeadlineEndsFirst
|
|
2234
|
+
? `Could not enforce the maximum active duration for outbox message "${message.id}".`
|
|
2235
|
+
: `Could not enforce the final renewal deadline for outbox message "${message.id}".`,
|
|
2236
|
+
cause: renewal.error,
|
|
2237
|
+
})
|
|
2238
|
+
: activeDeadlineEndsFirst
|
|
2239
|
+
? new OutboxMaxActiveDurationError({
|
|
2240
|
+
id: message.id,
|
|
2241
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2242
|
+
})
|
|
2243
|
+
: new OutboxLeaseLostError({
|
|
2244
|
+
id: message.id,
|
|
2245
|
+
message: `Outbox claim renewal for message "${message.id}" did not complete before settlement.`,
|
|
2246
|
+
}),
|
|
2247
|
+
message,
|
|
2248
|
+
operation: activeDeadlineEndsFirst
|
|
2249
|
+
? "maxActiveDuration"
|
|
2250
|
+
: "renewClaim",
|
|
2251
|
+
state: "lost",
|
|
2252
|
+
confirmedLost: false,
|
|
2253
|
+
},
|
|
2254
|
+
};
|
|
2255
|
+
},
|
|
2256
|
+
};
|
|
2257
|
+
}
|
|
2258
|
+
|
|
2259
|
+
async function settleClaim(options: {
|
|
2260
|
+
operation: OutboxSettlementFailure["operation"];
|
|
2261
|
+
message: ClaimedOutboxMessage;
|
|
2262
|
+
lockedUntil: Date;
|
|
2263
|
+
activeUntil: Date;
|
|
2264
|
+
runtime: ResolvedDrainRuntime;
|
|
2265
|
+
settle(now: Date): Promise<void>;
|
|
2266
|
+
}): Promise<{ ok: true } | { ok: false; error: unknown; claimLost: boolean }> {
|
|
2267
|
+
let lastError: unknown;
|
|
2268
|
+
|
|
2269
|
+
for (let attempt = 1; attempt <= OUTBOX_SETTLEMENT_ATTEMPTS; attempt += 1) {
|
|
2270
|
+
const settlementAt = readOutboxNow(options.runtime.now);
|
|
2271
|
+
const settlementDeadline = new Date(
|
|
2272
|
+
Math.min(options.lockedUntil.getTime(), options.activeUntil.getTime()),
|
|
2273
|
+
);
|
|
2274
|
+
const settlement = await runBoundedSettlement({
|
|
2275
|
+
runtime: options.runtime,
|
|
2276
|
+
settlementAt,
|
|
2277
|
+
deadline: settlementDeadline,
|
|
2278
|
+
settle: options.settle,
|
|
2279
|
+
});
|
|
2280
|
+
if (settlement.kind === "succeeded") {
|
|
2281
|
+
return { ok: true };
|
|
2282
|
+
}
|
|
2283
|
+
if (settlement.kind === "deadline") {
|
|
2284
|
+
return {
|
|
2285
|
+
ok: false,
|
|
2286
|
+
error: new OutboxLeaseLostError({
|
|
2287
|
+
id: options.message.id,
|
|
2288
|
+
message: `Outbox ${options.operation} for message "${options.message.id}" did not complete before its settlement deadline.`,
|
|
2289
|
+
}),
|
|
2290
|
+
claimLost: true,
|
|
2291
|
+
};
|
|
2292
|
+
}
|
|
2293
|
+
if (settlement.kind === "waitFailed") {
|
|
2294
|
+
return { ok: false, error: settlement.error, claimLost: false };
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const error = settlement.error;
|
|
2298
|
+
lastError = error;
|
|
2299
|
+
if (error instanceof OutboxClaimError) {
|
|
2300
|
+
return { ok: false, error, claimLost: true };
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
const remainingMs =
|
|
2304
|
+
settlementDeadline.getTime() -
|
|
2305
|
+
readOutboxNow(options.runtime.now).getTime();
|
|
2306
|
+
if (attempt >= OUTBOX_SETTLEMENT_ATTEMPTS || remainingMs <= 0) break;
|
|
2307
|
+
|
|
2308
|
+
const waitController = new AbortController();
|
|
2309
|
+
try {
|
|
2310
|
+
await options.runtime.wait(
|
|
2311
|
+
Math.max(
|
|
2312
|
+
1,
|
|
2313
|
+
Math.min(
|
|
2314
|
+
OUTBOX_SETTLEMENT_RETRY_DELAY_MS * 2 ** (attempt - 1),
|
|
2315
|
+
remainingMs,
|
|
2316
|
+
),
|
|
2317
|
+
),
|
|
2318
|
+
waitController.signal,
|
|
2319
|
+
);
|
|
2320
|
+
} catch (error) {
|
|
2321
|
+
return { ok: false, error, claimLost: false };
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
return { ok: false, error: lastError, claimLost: false };
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
async function runBoundedSettlement(options: {
|
|
2329
|
+
runtime: ResolvedDrainRuntime;
|
|
2330
|
+
settlementAt: Date;
|
|
2331
|
+
deadline: Date;
|
|
2332
|
+
settle(now: Date): Promise<void>;
|
|
2333
|
+
}): Promise<BoundedSettlementResult> {
|
|
2334
|
+
const remainingMs =
|
|
2335
|
+
options.deadline.getTime() - options.settlementAt.getTime();
|
|
2336
|
+
if (remainingMs <= 0) return { kind: "deadline" };
|
|
2337
|
+
|
|
2338
|
+
const waitController = new AbortController();
|
|
2339
|
+
const settlement = Promise.resolve()
|
|
2340
|
+
.then(() => options.settle(options.settlementAt))
|
|
2341
|
+
.then(
|
|
2342
|
+
() => ({ kind: "succeeded" as const }),
|
|
2343
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
2344
|
+
);
|
|
2345
|
+
const deadline = Promise.resolve()
|
|
2346
|
+
.then(() => options.runtime.wait(remainingMs, waitController.signal))
|
|
2347
|
+
.then(
|
|
2348
|
+
() => ({ kind: "deadline" as const }),
|
|
2349
|
+
(error: unknown) => ({ kind: "waitFailed" as const, error }),
|
|
2350
|
+
);
|
|
2351
|
+
|
|
2352
|
+
const result = await Promise.race([settlement, deadline]);
|
|
2353
|
+
waitController.abort();
|
|
2354
|
+
return result;
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
async function recordDeadLetter(
|
|
2358
|
+
options: DrainOutboxOptions,
|
|
2359
|
+
instrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2360
|
+
jobInstrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2361
|
+
message: OutboxMessage,
|
|
2362
|
+
error: unknown,
|
|
2363
|
+
details: Record<string, unknown> = {},
|
|
2364
|
+
): Promise<void> {
|
|
2365
|
+
try {
|
|
2366
|
+
await options.onDeadLetter?.(error, message);
|
|
2367
|
+
} catch {
|
|
2368
|
+
// Dead-letter observers must not change the settled message state.
|
|
2369
|
+
}
|
|
2370
|
+
instrumentation.record({
|
|
2371
|
+
type: "outbox",
|
|
2372
|
+
...options.instrumentationContext,
|
|
2373
|
+
messageId: message.id,
|
|
2374
|
+
messageKind: message.kind,
|
|
2375
|
+
messageName: message.name,
|
|
2376
|
+
status: "deadLettered",
|
|
2377
|
+
details: outboxInstrumentationDetails(message, {
|
|
2378
|
+
...details,
|
|
2379
|
+
error: serializeOutboxError(error),
|
|
2380
|
+
}),
|
|
2381
|
+
});
|
|
2382
|
+
if (message.kind === "job") {
|
|
2383
|
+
jobInstrumentation.record({
|
|
2384
|
+
type: "job",
|
|
2385
|
+
...options.instrumentationContext,
|
|
2386
|
+
jobName: message.name,
|
|
2387
|
+
status: "deadLettered",
|
|
2388
|
+
details: outboxInstrumentationDetails(message, {
|
|
2389
|
+
...details,
|
|
2390
|
+
error: serializeOutboxError(error),
|
|
2391
|
+
}),
|
|
2392
|
+
});
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
async function processClaimedMessage(
|
|
2397
|
+
options: DrainOutboxOptions,
|
|
2398
|
+
runtime: ResolvedDrainRuntime,
|
|
2399
|
+
instrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2400
|
+
jobInstrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2401
|
+
tracing: TracingPort | undefined,
|
|
2402
|
+
message: ClaimedOutboxMessage,
|
|
2403
|
+
): Promise<MessageDrainOutcome> {
|
|
2404
|
+
const emptyOutcome = (): MessageDrainOutcome => ({
|
|
2405
|
+
delivered: 0,
|
|
2406
|
+
retried: 0,
|
|
2407
|
+
deadLettered: 0,
|
|
2408
|
+
settlementFailed: 0,
|
|
2409
|
+
leaseLost: 0,
|
|
2410
|
+
});
|
|
2411
|
+
const outcome = emptyOutcome();
|
|
2412
|
+
const startedAt = readOutboxNow(runtime.now);
|
|
2413
|
+
const activeUntil = new Date(startedAt.getTime() + runtime.maxActiveMs);
|
|
2414
|
+
const heartbeat = createClaimHeartbeat(
|
|
2415
|
+
options,
|
|
2416
|
+
runtime,
|
|
2417
|
+
message,
|
|
2418
|
+
activeUntil,
|
|
2419
|
+
);
|
|
2420
|
+
const lifetimeController = new AbortController();
|
|
2421
|
+
|
|
2422
|
+
const parentTrace = parseTraceCarrier(message.trace);
|
|
2423
|
+
const traceAttributes = {
|
|
2424
|
+
"beignet.outbox.message_kind": message.kind,
|
|
2425
|
+
"beignet.outbox.message_name": message.name,
|
|
2426
|
+
} as const;
|
|
2427
|
+
const delivery = Promise.resolve()
|
|
2428
|
+
.then(() =>
|
|
2429
|
+
runWithTracing(
|
|
2430
|
+
tracing,
|
|
2431
|
+
{
|
|
2432
|
+
name: `beignet.outbox deliver ${message.name}`,
|
|
2433
|
+
type: "outbox",
|
|
2434
|
+
kind: "consumer",
|
|
2435
|
+
parent: parentTrace,
|
|
2436
|
+
attributes: traceAttributes,
|
|
2437
|
+
metricAttributes: traceAttributes,
|
|
2438
|
+
},
|
|
2439
|
+
(span) =>
|
|
2440
|
+
deliverOutboxMessage(
|
|
2441
|
+
options,
|
|
2442
|
+
message,
|
|
2443
|
+
captureTraceCarrier(span?.context ?? parentTrace),
|
|
2444
|
+
),
|
|
2445
|
+
),
|
|
2446
|
+
)
|
|
2447
|
+
.then(
|
|
2448
|
+
() => ({ kind: "succeeded" as const }),
|
|
2449
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
2450
|
+
);
|
|
2451
|
+
const maximumActive = Promise.resolve()
|
|
2452
|
+
.then(() => runtime.wait(runtime.maxActiveMs, lifetimeController.signal))
|
|
2453
|
+
.then(
|
|
2454
|
+
() => ({ kind: "maxActive" as const }),
|
|
2455
|
+
(error: unknown) => ({ kind: "waitFailed" as const, error }),
|
|
2456
|
+
);
|
|
2457
|
+
const leaseLost = heartbeat.lost.then((failure) => ({
|
|
2458
|
+
kind: "leaseLost" as const,
|
|
2459
|
+
failure,
|
|
2460
|
+
}));
|
|
2461
|
+
|
|
2462
|
+
const deliveryOutcome = await Promise.race([
|
|
2463
|
+
delivery,
|
|
2464
|
+
maximumActive,
|
|
2465
|
+
leaseLost,
|
|
2466
|
+
]);
|
|
2467
|
+
lifetimeController.abort();
|
|
2468
|
+
|
|
2469
|
+
if (deliveryOutcome.kind === "maxActive") {
|
|
2470
|
+
await heartbeat.stop();
|
|
2471
|
+
const failure: OutboxLeaseFailure = {
|
|
2472
|
+
error: new OutboxMaxActiveDurationError({
|
|
2473
|
+
id: message.id,
|
|
2474
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2475
|
+
}),
|
|
2476
|
+
message,
|
|
2477
|
+
operation: "maxActiveDuration",
|
|
2478
|
+
state: "lost",
|
|
2479
|
+
confirmedLost: false,
|
|
2480
|
+
};
|
|
2481
|
+
await notifyLeaseFailure(options, failure);
|
|
2482
|
+
instrumentation.custom({
|
|
2483
|
+
name: "outbox.lease.lost",
|
|
2484
|
+
label: "Outbox claim no longer confirmed",
|
|
2485
|
+
summary: `Stopped renewing ${message.kind} "${message.name}" after its active-delivery limit`,
|
|
2486
|
+
details: outboxInstrumentationDetails(message, {
|
|
2487
|
+
operation: failure.operation,
|
|
2488
|
+
state: failure.state,
|
|
2489
|
+
error: serializeOutboxError(failure.error),
|
|
2490
|
+
}),
|
|
2491
|
+
});
|
|
2492
|
+
outcome.leaseLost = 1;
|
|
2493
|
+
return outcome;
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
if (deliveryOutcome.kind === "waitFailed") {
|
|
2497
|
+
await heartbeat.stop();
|
|
2498
|
+
const failure: OutboxLeaseFailure = {
|
|
2499
|
+
error: new OutboxLeaseLostError({
|
|
2500
|
+
id: message.id,
|
|
2501
|
+
message: `Could not enforce the active-delivery limit for outbox message "${message.id}".`,
|
|
2502
|
+
cause: deliveryOutcome.error,
|
|
2503
|
+
}),
|
|
2504
|
+
message,
|
|
2505
|
+
operation: "maxActiveDuration",
|
|
2506
|
+
state: "lost",
|
|
2507
|
+
confirmedLost: false,
|
|
2508
|
+
};
|
|
2509
|
+
await notifyLeaseFailure(options, failure);
|
|
2510
|
+
instrumentation.custom({
|
|
2511
|
+
name: "outbox.lease.lost",
|
|
2512
|
+
label: "Outbox claim no longer confirmed",
|
|
2513
|
+
summary: `Could not enforce the active-delivery limit for ${message.kind} "${message.name}"`,
|
|
2514
|
+
details: outboxInstrumentationDetails(message, {
|
|
2515
|
+
operation: failure.operation,
|
|
2516
|
+
state: failure.state,
|
|
2517
|
+
error: serializeOutboxError(failure.error),
|
|
2518
|
+
}),
|
|
2519
|
+
});
|
|
2520
|
+
outcome.leaseLost = 1;
|
|
2521
|
+
return outcome;
|
|
2522
|
+
}
|
|
2523
|
+
|
|
2524
|
+
if (deliveryOutcome.kind === "leaseLost") {
|
|
2525
|
+
await heartbeat.stop();
|
|
2526
|
+
await notifyLeaseFailure(options, deliveryOutcome.failure);
|
|
2527
|
+
instrumentation.custom({
|
|
2528
|
+
name: "outbox.lease.lost",
|
|
2529
|
+
label: "Outbox claim lost",
|
|
2530
|
+
summary: `Could not keep the claim for ${message.kind} "${message.name}" active`,
|
|
2531
|
+
details: outboxInstrumentationDetails(message, {
|
|
2532
|
+
operation: deliveryOutcome.failure.operation,
|
|
2533
|
+
state: deliveryOutcome.failure.state,
|
|
2534
|
+
confirmedLost: deliveryOutcome.failure.confirmedLost,
|
|
2535
|
+
error: serializeOutboxError(deliveryOutcome.failure.error),
|
|
2536
|
+
}),
|
|
2537
|
+
});
|
|
2538
|
+
outcome.leaseLost = 1;
|
|
2539
|
+
return outcome;
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
const lease = await heartbeat.stopAndExtend();
|
|
2543
|
+
if (lease.failure) {
|
|
2544
|
+
await notifyLeaseFailure(options, lease.failure);
|
|
2545
|
+
instrumentation.custom({
|
|
2546
|
+
name: "outbox.lease.lost",
|
|
2547
|
+
label: "Outbox claim lost",
|
|
2548
|
+
summary: `Could not confirm the claim for ${message.kind} "${message.name}" before settlement`,
|
|
2549
|
+
details: outboxInstrumentationDetails(message, {
|
|
2550
|
+
operation: lease.failure.operation,
|
|
2551
|
+
state: lease.failure.state,
|
|
2552
|
+
confirmedLost: lease.failure.confirmedLost,
|
|
2553
|
+
error: serializeOutboxError(lease.failure.error),
|
|
2554
|
+
}),
|
|
2555
|
+
});
|
|
2556
|
+
outcome.leaseLost = 1;
|
|
2557
|
+
return outcome;
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
const reportRenewalOutcome = async () => {
|
|
2561
|
+
if (lease.renewalError === undefined) return;
|
|
2562
|
+
const failure: OutboxLeaseFailure = {
|
|
2563
|
+
error: lease.renewalError,
|
|
2564
|
+
message,
|
|
2565
|
+
operation: "renewClaim",
|
|
2566
|
+
state: lease.renewalRecovered ? "recovered" : "degraded",
|
|
2567
|
+
confirmedLost: false,
|
|
2568
|
+
};
|
|
2569
|
+
await notifyLeaseFailure(options, failure);
|
|
2570
|
+
instrumentation.custom({
|
|
2571
|
+
name: lease.renewalRecovered
|
|
2572
|
+
? "outbox.lease.renewal.recovered"
|
|
2573
|
+
: "outbox.lease.renewal.degraded",
|
|
2574
|
+
label: lease.renewalRecovered
|
|
2575
|
+
? "Outbox claim renewal recovered"
|
|
2576
|
+
: "Outbox claim renewal degraded",
|
|
2577
|
+
summary: lease.renewalRecovered
|
|
2578
|
+
? `Recovered claim renewal for ${message.kind} "${message.name}"`
|
|
2579
|
+
: `Continued ${message.kind} "${message.name}" settlement under its last confirmed lease`,
|
|
2580
|
+
details: outboxInstrumentationDetails(message, {
|
|
2581
|
+
state: failure.state,
|
|
2582
|
+
error: serializeOutboxError(lease.renewalError),
|
|
2583
|
+
}),
|
|
2584
|
+
});
|
|
2585
|
+
};
|
|
2586
|
+
|
|
2587
|
+
if (deliveryOutcome.kind === "succeeded") {
|
|
2588
|
+
const settlement = await settleClaim({
|
|
2589
|
+
operation: "markDelivered",
|
|
2590
|
+
message,
|
|
2591
|
+
lockedUntil: lease.lockedUntil,
|
|
2592
|
+
activeUntil,
|
|
2593
|
+
runtime,
|
|
2594
|
+
settle: (now) =>
|
|
2595
|
+
options.outbox.markDelivered({
|
|
2596
|
+
id: message.id,
|
|
2597
|
+
claimToken: message.claimToken,
|
|
2598
|
+
now,
|
|
2599
|
+
}),
|
|
2600
|
+
});
|
|
2601
|
+
await reportRenewalOutcome();
|
|
2602
|
+
if (!settlement.ok) {
|
|
2603
|
+
const failure: OutboxSettlementFailure = {
|
|
2604
|
+
error: settlement.error,
|
|
2605
|
+
message,
|
|
2606
|
+
operation: "markDelivered",
|
|
2607
|
+
deliverySucceeded: true,
|
|
2608
|
+
};
|
|
2609
|
+
await notifySettlementFailure(options, failure);
|
|
2610
|
+
instrumentation.custom({
|
|
2611
|
+
name: "outbox.settlement.failed",
|
|
2612
|
+
label: "Outbox settlement failed",
|
|
2613
|
+
summary: `Delivered ${message.kind} "${message.name}", but could not confirm its durable acknowledgement`,
|
|
2614
|
+
details: outboxInstrumentationDetails(message, {
|
|
2615
|
+
operation: failure.operation,
|
|
2616
|
+
deliverySucceeded: true,
|
|
2617
|
+
settlementError: serializeOutboxError(settlement.error),
|
|
2618
|
+
}),
|
|
2619
|
+
});
|
|
2620
|
+
outcome.settlementFailed = 1;
|
|
2621
|
+
if (settlement.claimLost) outcome.leaseLost = 1;
|
|
2622
|
+
return outcome;
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
instrumentation.record({
|
|
2626
|
+
type: "outbox",
|
|
2627
|
+
...options.instrumentationContext,
|
|
2628
|
+
messageId: message.id,
|
|
2629
|
+
messageKind: message.kind,
|
|
2630
|
+
messageName: message.name,
|
|
2631
|
+
status: "delivered",
|
|
2632
|
+
details: outboxInstrumentationDetails(message),
|
|
2633
|
+
});
|
|
2634
|
+
outcome.delivered = 1;
|
|
2635
|
+
return outcome;
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
const deliveryError = deliveryOutcome.error;
|
|
2639
|
+
const failedAt = readOutboxNow(runtime.now);
|
|
2640
|
+
const shouldRetry = shouldRetryOutboxMessage(options, message, deliveryError);
|
|
2641
|
+
const deadLetter = !shouldRetry;
|
|
2642
|
+
const retryDelayMs = deadLetter
|
|
2643
|
+
? 0
|
|
2644
|
+
: resolveRetryDelayMs(options, message, deliveryError, failedAt);
|
|
2645
|
+
const retryAt = deadLetter
|
|
2646
|
+
? undefined
|
|
2647
|
+
: new Date(failedAt.getTime() + retryDelayMs);
|
|
2648
|
+
const settlement = await settleClaim({
|
|
2649
|
+
operation: "markFailed",
|
|
2650
|
+
message,
|
|
2651
|
+
lockedUntil: lease.lockedUntil,
|
|
2652
|
+
activeUntil,
|
|
2653
|
+
runtime,
|
|
2654
|
+
settle: (now) =>
|
|
2655
|
+
options.outbox.markFailed({
|
|
2656
|
+
id: message.id,
|
|
2657
|
+
claimToken: message.claimToken,
|
|
2658
|
+
error: deliveryError,
|
|
2659
|
+
deadLetter,
|
|
2660
|
+
now,
|
|
2661
|
+
retryAt,
|
|
2662
|
+
}),
|
|
2663
|
+
});
|
|
2664
|
+
|
|
2665
|
+
await reportRenewalOutcome();
|
|
2666
|
+
|
|
2667
|
+
try {
|
|
2668
|
+
await options.onError?.(deliveryError, message);
|
|
2669
|
+
} catch {
|
|
2670
|
+
// Delivery observers cannot change retry, dead-letter, or recovery state.
|
|
2671
|
+
}
|
|
2672
|
+
|
|
2673
|
+
if (!settlement.ok) {
|
|
2674
|
+
const failure: OutboxSettlementFailure = {
|
|
2675
|
+
error: settlement.error,
|
|
2676
|
+
message,
|
|
2677
|
+
operation: "markFailed",
|
|
2678
|
+
deliverySucceeded: false,
|
|
2679
|
+
deliveryError,
|
|
2680
|
+
};
|
|
2681
|
+
await notifySettlementFailure(options, failure);
|
|
2682
|
+
instrumentation.custom({
|
|
2683
|
+
name: "outbox.settlement.failed",
|
|
2684
|
+
label: "Outbox settlement failed",
|
|
2685
|
+
summary: `Could not settle failed ${message.kind} "${message.name}"`,
|
|
2686
|
+
details: outboxInstrumentationDetails(message, {
|
|
2687
|
+
operation: failure.operation,
|
|
2688
|
+
deliverySucceeded: false,
|
|
2689
|
+
deliveryError: serializeOutboxError(deliveryError),
|
|
2690
|
+
settlementError: serializeOutboxError(settlement.error),
|
|
2691
|
+
}),
|
|
2692
|
+
});
|
|
2693
|
+
outcome.settlementFailed = 1;
|
|
2694
|
+
if (settlement.claimLost) outcome.leaseLost = 1;
|
|
2695
|
+
return outcome;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
if (deadLetter) {
|
|
2699
|
+
await recordDeadLetter(
|
|
2700
|
+
options,
|
|
2701
|
+
instrumentation,
|
|
2702
|
+
jobInstrumentation,
|
|
2703
|
+
message,
|
|
2704
|
+
deliveryError,
|
|
2705
|
+
);
|
|
2706
|
+
outcome.deadLettered = 1;
|
|
2707
|
+
return outcome;
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
instrumentation.record({
|
|
2711
|
+
type: "outbox",
|
|
2712
|
+
...options.instrumentationContext,
|
|
2713
|
+
messageId: message.id,
|
|
2714
|
+
messageKind: message.kind,
|
|
2715
|
+
messageName: message.name,
|
|
2716
|
+
status: "retryScheduled",
|
|
2717
|
+
details: outboxInstrumentationDetails(message, {
|
|
2718
|
+
retryDelayMs,
|
|
2719
|
+
retryAt: retryAt?.toISOString(),
|
|
2720
|
+
error: serializeOutboxError(deliveryError),
|
|
2721
|
+
}),
|
|
2722
|
+
});
|
|
2723
|
+
if (message.kind === "job") {
|
|
2724
|
+
jobInstrumentation.record({
|
|
2725
|
+
type: "job",
|
|
2726
|
+
...options.instrumentationContext,
|
|
2727
|
+
jobName: message.name,
|
|
2728
|
+
status: "retryScheduled",
|
|
2729
|
+
details: outboxInstrumentationDetails(message, {
|
|
2730
|
+
retryDelayMs,
|
|
2731
|
+
retryAt: retryAt?.toISOString(),
|
|
2732
|
+
error: serializeOutboxError(deliveryError),
|
|
2733
|
+
}),
|
|
2734
|
+
});
|
|
2735
|
+
}
|
|
2736
|
+
outcome.retried = 1;
|
|
2737
|
+
return outcome;
|
|
2738
|
+
}
|
|
2739
|
+
|
|
1477
2740
|
/**
|
|
1478
|
-
* Claim and deliver one
|
|
2741
|
+
* Claim and deliver one bounded set of outbox messages.
|
|
1479
2742
|
*
|
|
1480
|
-
*
|
|
1481
|
-
*
|
|
1482
|
-
*
|
|
1483
|
-
*
|
|
1484
|
-
* then dead-lettered.
|
|
2743
|
+
* The drain claims only enough messages to fill active delivery slots and
|
|
2744
|
+
* renews each active claim until delivery settles or reaches its configured
|
|
2745
|
+
* maximum duration. It remains an at-least-once transport: a process can still
|
|
2746
|
+
* terminate after the external effect succeeds but before acknowledgement.
|
|
1485
2747
|
*/
|
|
1486
2748
|
export async function drainOutbox(
|
|
1487
2749
|
options: DrainOutboxOptions,
|
|
1488
2750
|
): Promise<DrainOutboxResult> {
|
|
1489
|
-
const
|
|
1490
|
-
|
|
2751
|
+
const runtime = resolveDrainRuntime(options);
|
|
2752
|
+
assertOutboxDrainPort(options.outbox);
|
|
1491
2753
|
assertOutboxDeliveryCapabilities(options);
|
|
1492
2754
|
const instrumentation = createProviderInstrumentation(
|
|
1493
2755
|
options.instrumentation,
|
|
@@ -1504,165 +2766,110 @@ export async function drainOutbox(
|
|
|
1504
2766
|
},
|
|
1505
2767
|
);
|
|
1506
2768
|
const tracing = resolveTracingPort(options.instrumentation);
|
|
1507
|
-
|
|
1508
|
-
const now = options.now ?? new Date();
|
|
1509
|
-
const messages = await options.outbox.claimBatch({
|
|
1510
|
-
limit: batchSize,
|
|
1511
|
-
now,
|
|
1512
|
-
leaseMs: options.leaseMs,
|
|
1513
|
-
});
|
|
1514
2769
|
const result: DrainOutboxResult = {
|
|
1515
|
-
claimed:
|
|
2770
|
+
claimed: 0,
|
|
1516
2771
|
delivered: 0,
|
|
1517
2772
|
retried: 0,
|
|
1518
2773
|
deadLettered: 0,
|
|
2774
|
+
abandonedDeadLettered: 0,
|
|
2775
|
+
settlementFailed: 0,
|
|
2776
|
+
leaseLost: 0,
|
|
1519
2777
|
};
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
kind: "consumer",
|
|
1534
|
-
parent: parentTrace,
|
|
1535
|
-
attributes: traceAttributes,
|
|
1536
|
-
metricAttributes: traceAttributes,
|
|
1537
|
-
},
|
|
1538
|
-
(span) =>
|
|
1539
|
-
deliverOutboxMessage(
|
|
1540
|
-
options,
|
|
1541
|
-
message,
|
|
1542
|
-
captureTraceCarrier(span?.context ?? parentTrace),
|
|
1543
|
-
),
|
|
1544
|
-
);
|
|
1545
|
-
await options.outbox.markDelivered({
|
|
1546
|
-
id: message.id,
|
|
1547
|
-
claimToken: message.claimToken,
|
|
1548
|
-
now,
|
|
2778
|
+
let remaining = runtime.batchSize;
|
|
2779
|
+
let sourceExhausted = false;
|
|
2780
|
+
|
|
2781
|
+
while (remaining > 0 && !sourceExhausted) {
|
|
2782
|
+
const active: ClaimedOutboxMessage[] = [];
|
|
2783
|
+
const abandoned: OutboxMessage[] = [];
|
|
2784
|
+
|
|
2785
|
+
while (active.length < runtime.concurrency && remaining > 0) {
|
|
2786
|
+
const limit = Math.min(runtime.concurrency - active.length, remaining);
|
|
2787
|
+
const selected = await options.outbox.claimBatch({
|
|
2788
|
+
limit,
|
|
2789
|
+
now: readOutboxNow(runtime.now),
|
|
2790
|
+
leaseMs: runtime.leaseMs,
|
|
1549
2791
|
});
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
messageName: message.name,
|
|
1556
|
-
status: "delivered",
|
|
1557
|
-
details: outboxInstrumentationDetails(message),
|
|
1558
|
-
});
|
|
1559
|
-
result.delivered += 1;
|
|
1560
|
-
} catch (error) {
|
|
1561
|
-
try {
|
|
1562
|
-
await options.onError?.(error, message);
|
|
1563
|
-
} catch {
|
|
1564
|
-
// Preserve the delivery failure path so the message is retried or
|
|
1565
|
-
// dead-lettered even if the observer fails.
|
|
2792
|
+
const selectedCount =
|
|
2793
|
+
selected.claimed.length + selected.deadLettered.length;
|
|
2794
|
+
if (selectedCount === 0) {
|
|
2795
|
+
sourceExhausted = true;
|
|
2796
|
+
break;
|
|
1566
2797
|
}
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
: resolveRetryDelayMs(options, message, error, now);
|
|
1572
|
-
try {
|
|
1573
|
-
await options.outbox.markFailed({
|
|
1574
|
-
id: message.id,
|
|
1575
|
-
claimToken: message.claimToken,
|
|
1576
|
-
error,
|
|
1577
|
-
deadLetter,
|
|
1578
|
-
now,
|
|
1579
|
-
retryAt: deadLetter
|
|
1580
|
-
? undefined
|
|
1581
|
-
: new Date(now.getTime() + retryDelayMs),
|
|
1582
|
-
});
|
|
1583
|
-
} catch (settlementError) {
|
|
1584
|
-
try {
|
|
1585
|
-
await options.onSettlementError?.(settlementError, message, error);
|
|
1586
|
-
} catch {
|
|
1587
|
-
// Preserve the settlement failure when its observer also fails.
|
|
1588
|
-
}
|
|
1589
|
-
instrumentation.custom({
|
|
1590
|
-
name: "outbox.settlement.failed",
|
|
1591
|
-
label: "Outbox settlement failed",
|
|
1592
|
-
summary: `Could not settle failed ${message.kind} "${message.name}"`,
|
|
1593
|
-
details: outboxInstrumentationDetails(message, {
|
|
1594
|
-
deliveryError: serializeOutboxError(error),
|
|
1595
|
-
settlementError: serializeOutboxError(settlementError),
|
|
1596
|
-
}),
|
|
1597
|
-
});
|
|
1598
|
-
continue;
|
|
2798
|
+
if (selectedCount > limit) {
|
|
2799
|
+
throw new Error(
|
|
2800
|
+
`Outbox claimBatch returned ${selectedCount} messages for a limit of ${limit}.`,
|
|
2801
|
+
);
|
|
1599
2802
|
}
|
|
1600
2803
|
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
type: "job",
|
|
1648
|
-
...options.instrumentationContext,
|
|
1649
|
-
jobName: message.name,
|
|
1650
|
-
status: "retryScheduled",
|
|
1651
|
-
details: outboxInstrumentationDetails(message, {
|
|
1652
|
-
retryDelayMs,
|
|
1653
|
-
retryAt,
|
|
1654
|
-
error: serializeOutboxError(error),
|
|
1655
|
-
}),
|
|
1656
|
-
});
|
|
1657
|
-
}
|
|
1658
|
-
result.retried += 1;
|
|
1659
|
-
}
|
|
2804
|
+
remaining -= selectedCount;
|
|
2805
|
+
result.claimed += selected.claimed.length;
|
|
2806
|
+
active.push(...selected.claimed);
|
|
2807
|
+
abandoned.push(...selected.deadLettered);
|
|
2808
|
+
}
|
|
2809
|
+
|
|
2810
|
+
// Construct active delivery promises first so their heartbeats protect
|
|
2811
|
+
// freshly claimed rows while abandoned-message observers run.
|
|
2812
|
+
const outcomePromises = active.map((message) =>
|
|
2813
|
+
processClaimedMessage(
|
|
2814
|
+
options,
|
|
2815
|
+
runtime,
|
|
2816
|
+
instrumentation,
|
|
2817
|
+
jobInstrumentation,
|
|
2818
|
+
tracing,
|
|
2819
|
+
message,
|
|
2820
|
+
),
|
|
2821
|
+
);
|
|
2822
|
+
const abandonedPromises = abandoned.map(async (message) => {
|
|
2823
|
+
const error = new OutboxAbandonedClaimError({
|
|
2824
|
+
id: message.id,
|
|
2825
|
+
attempts: message.attempts,
|
|
2826
|
+
maxAttempts: message.maxAttempts,
|
|
2827
|
+
});
|
|
2828
|
+
await recordDeadLetter(
|
|
2829
|
+
options,
|
|
2830
|
+
instrumentation,
|
|
2831
|
+
jobInstrumentation,
|
|
2832
|
+
message,
|
|
2833
|
+
error,
|
|
2834
|
+
{ abandoned: true },
|
|
2835
|
+
);
|
|
2836
|
+
});
|
|
2837
|
+
const [outcomes] = await Promise.all([
|
|
2838
|
+
Promise.all(outcomePromises),
|
|
2839
|
+
Promise.all(abandonedPromises),
|
|
2840
|
+
]);
|
|
2841
|
+
result.deadLettered += abandoned.length;
|
|
2842
|
+
result.abandonedDeadLettered += abandoned.length;
|
|
2843
|
+
|
|
2844
|
+
for (const outcome of outcomes) {
|
|
2845
|
+
result.delivered += outcome.delivered;
|
|
2846
|
+
result.retried += outcome.retried;
|
|
2847
|
+
result.deadLettered += outcome.deadLettered;
|
|
2848
|
+
result.settlementFailed += outcome.settlementFailed;
|
|
2849
|
+
result.leaseLost += outcome.leaseLost;
|
|
1660
2850
|
}
|
|
1661
2851
|
}
|
|
1662
2852
|
|
|
1663
2853
|
return result;
|
|
1664
2854
|
}
|
|
1665
2855
|
|
|
2856
|
+
function assertOutboxDrainPort(outbox: OutboxPort): void {
|
|
2857
|
+
const candidate = outbox as unknown as Record<string, unknown>;
|
|
2858
|
+
const missing = [
|
|
2859
|
+
"claimBatch",
|
|
2860
|
+
"renewClaim",
|
|
2861
|
+
"markDelivered",
|
|
2862
|
+
"markFailed",
|
|
2863
|
+
].filter((method) => typeof candidate[method] !== "function");
|
|
2864
|
+
if (missing.length > 0) {
|
|
2865
|
+
throw new Error(
|
|
2866
|
+
`Cannot drain this outbox: the outbox port is missing ${missing
|
|
2867
|
+
.map((method) => `${method}()`)
|
|
2868
|
+
.join(", ")}.`,
|
|
2869
|
+
);
|
|
2870
|
+
}
|
|
2871
|
+
}
|
|
2872
|
+
|
|
1666
2873
|
function assertOutboxDeliveryCapabilities(options: DrainOutboxOptions): void {
|
|
1667
2874
|
const missing: string[] = [];
|
|
1668
2875
|
|