@beignet/core 0.0.50 → 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +99 -32
- package/dist/application/index.d.ts +5 -5
- package/dist/application/index.d.ts.map +1 -1
- package/dist/application/index.js +8 -4
- package/dist/application/index.js.map +1 -1
- package/dist/events/index.d.ts +26 -2
- package/dist/events/index.d.ts.map +1 -1
- package/dist/events/index.js +84 -7
- package/dist/events/index.js.map +1 -1
- package/dist/events/payload-state.d.ts +16 -0
- package/dist/events/payload-state.d.ts.map +1 -0
- package/dist/events/payload-state.js +123 -0
- package/dist/events/payload-state.js.map +1 -0
- package/dist/events/transport.d.ts +25 -0
- package/dist/events/transport.d.ts.map +1 -0
- package/dist/events/transport.js +192 -0
- package/dist/events/transport.js.map +1 -0
- package/dist/locks/index.d.ts.map +1 -1
- package/dist/locks/index.js +0 -4
- package/dist/locks/index.js.map +1 -1
- package/dist/openapi/index.d.ts.map +1 -1
- package/dist/openapi/index.js +27 -4
- package/dist/openapi/index.js.map +1 -1
- package/dist/outbox/index.d.ts +163 -15
- package/dist/outbox/index.d.ts.map +1 -1
- package/dist/outbox/index.js +1009 -155
- package/dist/outbox/index.js.map +1 -1
- package/dist/payments/index.d.ts.map +1 -1
- package/dist/payments/index.js +0 -4
- package/dist/payments/index.js.map +1 -1
- package/dist/ports/events.d.ts +4 -1
- package/dist/ports/events.d.ts.map +1 -1
- package/dist/ports/index.d.ts +1 -1
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/ports/storage.d.ts +7 -0
- package/dist/ports/storage.d.ts.map +1 -1
- package/dist/ports/storage.js +4 -0
- package/dist/ports/storage.js.map +1 -1
- package/dist/ports/testing.d.ts +16 -2
- package/dist/ports/testing.d.ts.map +1 -1
- package/dist/ports/testing.js +19 -4
- package/dist/ports/testing.js.map +1 -1
- package/dist/ports/unit-of-work.d.ts +9 -7
- package/dist/ports/unit-of-work.d.ts.map +1 -1
- package/dist/ports/unit-of-work.js +16 -7
- package/dist/ports/unit-of-work.js.map +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/provider.d.ts +0 -38
- package/dist/providers/provider.d.ts.map +1 -1
- package/dist/providers/provider.js.map +1 -1
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +2 -6
- package/dist/search/index.js.map +1 -1
- package/dist/server/hooks/cors.d.ts +5 -0
- package/dist/server/hooks/cors.d.ts.map +1 -1
- package/dist/server/hooks/cors.js +29 -1
- package/dist/server/hooks/cors.js.map +1 -1
- package/dist/server/instrumentation.d.ts +5 -5
- package/dist/server/instrumentation.d.ts.map +1 -1
- package/dist/server/instrumentation.js +3 -4
- package/dist/server/instrumentation.js.map +1 -1
- package/dist/server/response-finalization.d.ts.map +1 -1
- package/dist/server/response-finalization.js +25 -13
- package/dist/server/response-finalization.js.map +1 -1
- package/dist/server/server.d.ts +8 -5
- package/dist/server/server.d.ts.map +1 -1
- package/dist/server/server.js +35 -12
- package/dist/server/server.js.map +1 -1
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +12 -6
- package/dist/testing/index.js.map +1 -1
- package/dist/uploads/index.d.ts.map +1 -1
- package/dist/uploads/index.js +37 -16
- package/dist/uploads/index.js.map +1 -1
- package/package.json +2 -2
- package/skills/app-architecture/SKILL.md +16 -0
- package/src/application/index.ts +39 -7
- package/src/events/index.ts +137 -7
- package/src/events/payload-state.ts +223 -0
- package/src/events/transport.ts +242 -0
- package/src/locks/index.ts +0 -4
- package/src/openapi/index.ts +41 -3
- package/src/outbox/index.ts +1393 -178
- package/src/payments/index.ts +0 -4
- package/src/ports/events.ts +4 -1
- package/src/ports/index.ts +3 -0
- package/src/ports/storage.ts +10 -0
- package/src/ports/testing.ts +42 -4
- package/src/ports/unit-of-work.ts +33 -15
- package/src/providers/index.ts +0 -1
- package/src/providers/provider.ts +0 -40
- package/src/search/index.ts +2 -6
- package/src/server/hooks/cors.ts +42 -0
- package/src/server/instrumentation.ts +10 -8
- package/src/server/response-finalization.ts +33 -15
- package/src/server/server.ts +61 -27
- package/src/testing/index.ts +20 -9
- package/src/uploads/index.ts +34 -16
package/src/outbox/index.ts
CHANGED
|
@@ -8,8 +8,9 @@
|
|
|
8
8
|
import {
|
|
9
9
|
type EventPayloadDef,
|
|
10
10
|
type EventPublishOptions,
|
|
11
|
+
type EventTransportValue,
|
|
11
12
|
type InferEventPayload,
|
|
12
|
-
|
|
13
|
+
prepareEventPayloadForTransport,
|
|
13
14
|
} from "../events/index.js";
|
|
14
15
|
import {
|
|
15
16
|
getJobRetryDelayMs,
|
|
@@ -46,6 +47,18 @@ export type MaybePromise<T> = T | Promise<T>;
|
|
|
46
47
|
* Default lease duration for claimed outbox messages.
|
|
47
48
|
*/
|
|
48
49
|
export const DEFAULT_OUTBOX_LEASE_MS = 30_000;
|
|
50
|
+
/**
|
|
51
|
+
* Default maximum messages handled by one bounded drain pass.
|
|
52
|
+
*/
|
|
53
|
+
export const DEFAULT_OUTBOX_BATCH_SIZE = 100;
|
|
54
|
+
/**
|
|
55
|
+
* Default number of outbox messages delivered concurrently.
|
|
56
|
+
*/
|
|
57
|
+
export const DEFAULT_OUTBOX_CONCURRENCY = 1;
|
|
58
|
+
/**
|
|
59
|
+
* Default maximum time Beignet renews a claim for one delivery.
|
|
60
|
+
*/
|
|
61
|
+
export const DEFAULT_OUTBOX_MAX_ACTIVE_MS = 300_000;
|
|
49
62
|
/**
|
|
50
63
|
* Default maximum delivery attempts before a message is dead-lettered.
|
|
51
64
|
*/
|
|
@@ -218,7 +231,7 @@ export interface ClaimedOutboxMessage
|
|
|
218
231
|
*/
|
|
219
232
|
export interface OutboxClaimBatchOptions {
|
|
220
233
|
/**
|
|
221
|
-
* Maximum messages to claim in one batch.
|
|
234
|
+
* Maximum eligible messages to claim or reconcile in one batch.
|
|
222
235
|
*/
|
|
223
236
|
limit: number;
|
|
224
237
|
/**
|
|
@@ -231,6 +244,41 @@ export interface OutboxClaimBatchOptions {
|
|
|
231
244
|
leaseMs?: number;
|
|
232
245
|
}
|
|
233
246
|
|
|
247
|
+
/**
|
|
248
|
+
* Result of atomically selecting one bounded set of eligible messages.
|
|
249
|
+
*/
|
|
250
|
+
export interface OutboxClaimBatchResult {
|
|
251
|
+
/** Messages claimed for delivery by the current worker. */
|
|
252
|
+
claimed: readonly ClaimedOutboxMessage[];
|
|
253
|
+
/**
|
|
254
|
+
* Eligible messages moved directly to dead letter because their claim
|
|
255
|
+
* attempt budget was already exhausted.
|
|
256
|
+
*/
|
|
257
|
+
deadLettered: readonly OutboxMessage[];
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Input for extending one active outbox claim.
|
|
262
|
+
*/
|
|
263
|
+
export interface OutboxRenewClaimInput {
|
|
264
|
+
/** Claimed message ID. */
|
|
265
|
+
id: string;
|
|
266
|
+
/** Claim token returned by `claimBatch(...)`. */
|
|
267
|
+
claimToken: string;
|
|
268
|
+
/** Renewal timestamp. */
|
|
269
|
+
now?: Date;
|
|
270
|
+
/** New lease duration measured from `now`. */
|
|
271
|
+
leaseMs?: number;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Confirmed result of extending one active outbox claim.
|
|
276
|
+
*/
|
|
277
|
+
export interface OutboxRenewClaimResult {
|
|
278
|
+
/** New confirmed lease expiration timestamp. */
|
|
279
|
+
lockedUntil: Date;
|
|
280
|
+
}
|
|
281
|
+
|
|
234
282
|
/**
|
|
235
283
|
* Input for marking a claimed message delivered.
|
|
236
284
|
*/
|
|
@@ -403,7 +451,11 @@ export interface OutboxPort {
|
|
|
403
451
|
/**
|
|
404
452
|
* Atomically claim eligible messages for one worker.
|
|
405
453
|
*/
|
|
406
|
-
claimBatch(options: OutboxClaimBatchOptions): Promise<
|
|
454
|
+
claimBatch(options: OutboxClaimBatchOptions): Promise<OutboxClaimBatchResult>;
|
|
455
|
+
/**
|
|
456
|
+
* Extend an unexpired claim owned by the supplied claim token.
|
|
457
|
+
*/
|
|
458
|
+
renewClaim(input: OutboxRenewClaimInput): Promise<OutboxRenewClaimResult>;
|
|
407
459
|
/**
|
|
408
460
|
* Mark a claimed message delivered.
|
|
409
461
|
*/
|
|
@@ -542,6 +594,40 @@ export type OutboxInstrumentationContext = Pick<
|
|
|
542
594
|
"requestId" | "traceId" | "spanId" | "parentSpanId" | "traceparent"
|
|
543
595
|
>;
|
|
544
596
|
|
|
597
|
+
/** Wait primitive used by outbox heartbeats and bounded settlement retries. */
|
|
598
|
+
export type OutboxDrainWait = (
|
|
599
|
+
delayMs: number,
|
|
600
|
+
signal: AbortSignal,
|
|
601
|
+
) => Promise<void>;
|
|
602
|
+
|
|
603
|
+
/** Structured failure from a claim heartbeat or active-delivery boundary. */
|
|
604
|
+
export interface OutboxLeaseFailure {
|
|
605
|
+
/** Underlying renewal, ownership, or duration error. */
|
|
606
|
+
error: unknown;
|
|
607
|
+
/** Message whose claim could not be kept active. */
|
|
608
|
+
message: ClaimedOutboxMessage;
|
|
609
|
+
/** Lease phase that surfaced the failure. */
|
|
610
|
+
operation: "renewClaim" | "maxActiveDuration";
|
|
611
|
+
/** Current lease state after handling the failure. */
|
|
612
|
+
state: "recovered" | "degraded" | "lost";
|
|
613
|
+
/** Whether the worker no longer has confirmed ownership. */
|
|
614
|
+
confirmedLost: boolean;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/** Structured failure from a token-guarded outbox settlement. */
|
|
618
|
+
export interface OutboxSettlementFailure {
|
|
619
|
+
/** Storage failure returned by the settlement operation. */
|
|
620
|
+
error: unknown;
|
|
621
|
+
/** Message whose final storage state is unknown. */
|
|
622
|
+
message: ClaimedOutboxMessage;
|
|
623
|
+
/** Settlement operation that failed. */
|
|
624
|
+
operation: "markDelivered" | "markFailed";
|
|
625
|
+
/** Whether the external delivery completed successfully. */
|
|
626
|
+
deliverySucceeded: boolean;
|
|
627
|
+
/** Original delivery error when `deliverySucceeded` is false. */
|
|
628
|
+
deliveryError?: unknown;
|
|
629
|
+
}
|
|
630
|
+
|
|
545
631
|
/**
|
|
546
632
|
* Options for draining one outbox batch.
|
|
547
633
|
*/
|
|
@@ -571,17 +657,38 @@ export interface DrainOutboxOptions {
|
|
|
571
657
|
*/
|
|
572
658
|
jobs?: JobDispatcherPort;
|
|
573
659
|
/**
|
|
574
|
-
* Maximum messages to
|
|
660
|
+
* Maximum eligible messages to handle in one drain pass.
|
|
575
661
|
*/
|
|
576
662
|
batchSize?: number;
|
|
577
663
|
/**
|
|
578
|
-
*
|
|
664
|
+
* Maximum messages delivered concurrently. Defaults to serial delivery.
|
|
665
|
+
* Values greater than one do not preserve delivery order.
|
|
579
666
|
*/
|
|
580
|
-
|
|
667
|
+
concurrency?: number;
|
|
668
|
+
/**
|
|
669
|
+
* Clock used independently for claiming, renewal, settlement, and retry
|
|
670
|
+
* scheduling. Defaults to the system clock.
|
|
671
|
+
*/
|
|
672
|
+
now?: () => Date;
|
|
581
673
|
/**
|
|
582
674
|
* Claim lease duration in milliseconds.
|
|
583
675
|
*/
|
|
584
676
|
leaseMs?: number;
|
|
677
|
+
/**
|
|
678
|
+
* Interval between serialized claim renewals. Defaults to one third of the
|
|
679
|
+
* lease duration and must remain shorter than the lease.
|
|
680
|
+
*/
|
|
681
|
+
heartbeatMs?: number;
|
|
682
|
+
/**
|
|
683
|
+
* Maximum time Beignet renews a claim for one delivery. When exceeded, the
|
|
684
|
+
* drain stops renewing and leaves final recovery to the last lease expiry.
|
|
685
|
+
*/
|
|
686
|
+
maxActiveMs?: number;
|
|
687
|
+
/**
|
|
688
|
+
* Abort-aware wait implementation. Inject a deterministic implementation in
|
|
689
|
+
* tests; production callers normally use the default timer.
|
|
690
|
+
*/
|
|
691
|
+
wait?: OutboxDrainWait;
|
|
585
692
|
/**
|
|
586
693
|
* Retry delay in milliseconds or function for per-message delay.
|
|
587
694
|
*/
|
|
@@ -613,19 +720,17 @@ export interface DrainOutboxOptions {
|
|
|
613
720
|
* Observer called after a failed delivery is successfully moved to the dead
|
|
614
721
|
* letter state. Observer failures are ignored.
|
|
615
722
|
*/
|
|
616
|
-
onDeadLetter?: (
|
|
617
|
-
error: unknown,
|
|
618
|
-
message: ClaimedOutboxMessage,
|
|
619
|
-
) => MaybePromise<void>;
|
|
723
|
+
onDeadLetter?: (error: unknown, message: OutboxMessage) => MaybePromise<void>;
|
|
620
724
|
/**
|
|
621
|
-
* Observer called when
|
|
622
|
-
*
|
|
725
|
+
* Observer called when claim renewal degrades or ownership is lost.
|
|
726
|
+
* Observer failures are ignored.
|
|
623
727
|
*/
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
728
|
+
onLeaseError?: (failure: OutboxLeaseFailure) => MaybePromise<void>;
|
|
729
|
+
/**
|
|
730
|
+
* Observer called when a delivery outcome cannot be settled durably.
|
|
731
|
+
* Observer failures are ignored.
|
|
732
|
+
*/
|
|
733
|
+
onSettlementError?: (failure: OutboxSettlementFailure) => MaybePromise<void>;
|
|
629
734
|
}
|
|
630
735
|
|
|
631
736
|
/**
|
|
@@ -648,6 +753,15 @@ export interface DrainOutboxResult {
|
|
|
648
753
|
* Messages moved to dead letter state.
|
|
649
754
|
*/
|
|
650
755
|
deadLettered: number;
|
|
756
|
+
/**
|
|
757
|
+
* Dead-lettered messages whose claim attempt budget was already exhausted.
|
|
758
|
+
* This is a subset of `deadLettered`.
|
|
759
|
+
*/
|
|
760
|
+
abandonedDeadLettered: number;
|
|
761
|
+
/** Messages delivered or failed whose final storage state is unknown. */
|
|
762
|
+
settlementFailed: number;
|
|
763
|
+
/** Messages whose active claim could no longer be confirmed. */
|
|
764
|
+
leaseLost: number;
|
|
651
765
|
}
|
|
652
766
|
|
|
653
767
|
/**
|
|
@@ -686,6 +800,62 @@ export class OutboxClaimError extends Error {
|
|
|
686
800
|
}
|
|
687
801
|
}
|
|
688
802
|
|
|
803
|
+
/**
|
|
804
|
+
* Error persisted when an eligible message has exhausted its claim attempts
|
|
805
|
+
* without reaching a terminal settlement.
|
|
806
|
+
*/
|
|
807
|
+
export class OutboxAbandonedClaimError extends Error {
|
|
808
|
+
/** Message ID whose attempt budget was exhausted. */
|
|
809
|
+
readonly id: string;
|
|
810
|
+
/** Number of claims already made. */
|
|
811
|
+
readonly attempts: number;
|
|
812
|
+
/** Maximum permitted claim attempts. */
|
|
813
|
+
readonly maxAttempts: number;
|
|
814
|
+
|
|
815
|
+
constructor(args: { id: string; attempts: number; maxAttempts: number }) {
|
|
816
|
+
super(
|
|
817
|
+
`Outbox message "${args.id}" exhausted ${args.maxAttempts} claim attempts without a terminal settlement.`,
|
|
818
|
+
);
|
|
819
|
+
this.name = "OutboxAbandonedClaimError";
|
|
820
|
+
this.id = args.id;
|
|
821
|
+
this.attempts = args.attempts;
|
|
822
|
+
this.maxAttempts = args.maxAttempts;
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/** Error surfaced when an outbox worker no longer has a confirmed claim. */
|
|
827
|
+
export class OutboxLeaseLostError extends Error {
|
|
828
|
+
/** Message ID whose ownership was lost. */
|
|
829
|
+
readonly id: string;
|
|
830
|
+
|
|
831
|
+
constructor(args: { id: string; message?: string; cause?: unknown }) {
|
|
832
|
+
super(
|
|
833
|
+
args.message ??
|
|
834
|
+
`Outbox claim for message "${args.id}" is no longer active.`,
|
|
835
|
+
args.cause === undefined ? undefined : { cause: args.cause },
|
|
836
|
+
);
|
|
837
|
+
this.name = "OutboxLeaseLostError";
|
|
838
|
+
this.id = args.id;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Error surfaced when delivery exceeds the configured active-claim window. */
|
|
843
|
+
export class OutboxMaxActiveDurationError extends Error {
|
|
844
|
+
/** Message ID whose active window elapsed. */
|
|
845
|
+
readonly id: string;
|
|
846
|
+
/** Configured maximum active duration. */
|
|
847
|
+
readonly maxActiveMs: number;
|
|
848
|
+
|
|
849
|
+
constructor(args: { id: string; maxActiveMs: number }) {
|
|
850
|
+
super(
|
|
851
|
+
`Outbox delivery for message "${args.id}" exceeded the ${args.maxActiveMs}ms active-claim limit.`,
|
|
852
|
+
);
|
|
853
|
+
this.name = "OutboxMaxActiveDurationError";
|
|
854
|
+
this.id = args.id;
|
|
855
|
+
this.maxActiveMs = args.maxActiveMs;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
|
|
689
859
|
/**
|
|
690
860
|
* Error thrown when an outbox admin operation cannot be completed safely.
|
|
691
861
|
*/
|
|
@@ -1169,9 +1339,11 @@ export function createMemoryOutbox(
|
|
|
1169
1339
|
assertPositiveInteger("limit", options.limit);
|
|
1170
1340
|
const now = options.now ?? storeNow();
|
|
1171
1341
|
const leaseMs = options.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1342
|
+
assertValidDate("now", now);
|
|
1172
1343
|
assertPositiveInteger("leaseMs", leaseMs);
|
|
1173
1344
|
const lockedUntil = new Date(now.getTime() + leaseMs);
|
|
1174
1345
|
const claimed: ClaimedOutboxMessage[] = [];
|
|
1346
|
+
const deadLettered: OutboxMessage[] = [];
|
|
1175
1347
|
|
|
1176
1348
|
const eligible = [...messages.values()]
|
|
1177
1349
|
.filter((message) => isEligible(message, now))
|
|
@@ -1183,6 +1355,23 @@ export function createMemoryOutbox(
|
|
|
1183
1355
|
.slice(0, options.limit);
|
|
1184
1356
|
|
|
1185
1357
|
for (const message of eligible) {
|
|
1358
|
+
if (message.attempts >= message.maxAttempts) {
|
|
1359
|
+
message.status = "deadLettered";
|
|
1360
|
+
message.lastError = serializeOutboxError(
|
|
1361
|
+
new OutboxAbandonedClaimError({
|
|
1362
|
+
id: message.id,
|
|
1363
|
+
attempts: message.attempts,
|
|
1364
|
+
maxAttempts: message.maxAttempts,
|
|
1365
|
+
}),
|
|
1366
|
+
);
|
|
1367
|
+
message.claimToken = null;
|
|
1368
|
+
message.claimedAt = null;
|
|
1369
|
+
message.lockedUntil = null;
|
|
1370
|
+
message.updatedAt = cloneDate(now);
|
|
1371
|
+
deadLettered.push(copyMessage(message));
|
|
1372
|
+
continue;
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1186
1375
|
message.status = "claimed";
|
|
1187
1376
|
message.attempts += 1;
|
|
1188
1377
|
message.claimToken = createStoreId();
|
|
@@ -1192,7 +1381,31 @@ export function createMemoryOutbox(
|
|
|
1192
1381
|
claimed.push(toClaimedMessage(message));
|
|
1193
1382
|
}
|
|
1194
1383
|
|
|
1195
|
-
return claimed;
|
|
1384
|
+
return { claimed, deadLettered };
|
|
1385
|
+
},
|
|
1386
|
+
|
|
1387
|
+
async renewClaim(input) {
|
|
1388
|
+
assertNonEmptyString("id", input.id);
|
|
1389
|
+
assertNonEmptyString("claimToken", input.claimToken);
|
|
1390
|
+
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1391
|
+
const now = input.now ?? storeNow();
|
|
1392
|
+
const leaseMs = input.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1393
|
+
assertValidDate("now", now);
|
|
1394
|
+
assertPositiveInteger("leaseMs", leaseMs);
|
|
1395
|
+
if (
|
|
1396
|
+
message.lockedUntil === null ||
|
|
1397
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1398
|
+
) {
|
|
1399
|
+
throw new OutboxClaimError({
|
|
1400
|
+
id: input.id,
|
|
1401
|
+
message: `Outbox message "${input.id}" no longer has an active claim to renew.`,
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
const lockedUntil = new Date(now.getTime() + leaseMs);
|
|
1406
|
+
message.lockedUntil = cloneDate(lockedUntil);
|
|
1407
|
+
message.updatedAt = cloneDate(now);
|
|
1408
|
+
return { lockedUntil: cloneDate(lockedUntil) };
|
|
1196
1409
|
},
|
|
1197
1410
|
|
|
1198
1411
|
async markDelivered(input) {
|
|
@@ -1200,6 +1413,16 @@ export function createMemoryOutbox(
|
|
|
1200
1413
|
assertNonEmptyString("claimToken", input.claimToken);
|
|
1201
1414
|
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1202
1415
|
const now = input.now ?? storeNow();
|
|
1416
|
+
assertValidDate("now", now);
|
|
1417
|
+
if (
|
|
1418
|
+
message.lockedUntil === null ||
|
|
1419
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1420
|
+
) {
|
|
1421
|
+
throw new OutboxClaimError({
|
|
1422
|
+
id: input.id,
|
|
1423
|
+
message: `Outbox message "${input.id}" no longer has an active claim to settle.`,
|
|
1424
|
+
});
|
|
1425
|
+
}
|
|
1203
1426
|
|
|
1204
1427
|
message.status = "delivered";
|
|
1205
1428
|
message.deliveredAt = cloneDate(now);
|
|
@@ -1214,6 +1437,17 @@ export function createMemoryOutbox(
|
|
|
1214
1437
|
assertNonEmptyString("claimToken", input.claimToken);
|
|
1215
1438
|
const message = getClaimedOrThrow(input.id, input.claimToken);
|
|
1216
1439
|
const now = input.now ?? storeNow();
|
|
1440
|
+
assertValidDate("now", now);
|
|
1441
|
+
if (input.retryAt) assertValidDate("retryAt", input.retryAt);
|
|
1442
|
+
if (
|
|
1443
|
+
message.lockedUntil === null ||
|
|
1444
|
+
message.lockedUntil.getTime() <= now.getTime()
|
|
1445
|
+
) {
|
|
1446
|
+
throw new OutboxClaimError({
|
|
1447
|
+
id: input.id,
|
|
1448
|
+
message: `Outbox message "${input.id}" no longer has an active claim to settle.`,
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1217
1451
|
|
|
1218
1452
|
message.status = input.deadLetter ? "deadLettered" : "pending";
|
|
1219
1453
|
message.lastError = serializeOutboxError(input.error);
|
|
@@ -1271,14 +1505,28 @@ export async function enqueueEvent<E extends EventPayloadDef>(
|
|
|
1271
1505
|
payload: InferEventPayload<E>,
|
|
1272
1506
|
options: EnqueueTypedOutboxOptions = {},
|
|
1273
1507
|
): Promise<OutboxMessage> {
|
|
1274
|
-
await
|
|
1508
|
+
const prepared = await prepareEventPayloadForTransport(event, payload);
|
|
1509
|
+
return await enqueueTransportEvent(
|
|
1510
|
+
outbox,
|
|
1511
|
+
event,
|
|
1512
|
+
prepared.transportValue,
|
|
1513
|
+
options,
|
|
1514
|
+
);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
async function enqueueTransportEvent<E extends EventPayloadDef>(
|
|
1518
|
+
outbox: OutboxPort,
|
|
1519
|
+
event: E,
|
|
1520
|
+
payload: EventTransportValue,
|
|
1521
|
+
options: EnqueueTypedOutboxOptions,
|
|
1522
|
+
): Promise<OutboxMessage> {
|
|
1275
1523
|
const trace =
|
|
1276
1524
|
parseTraceCarrier(options.trace) ?? captureTraceCarrier(options.tracing);
|
|
1277
1525
|
return outbox.enqueue({
|
|
1278
1526
|
id: options.id,
|
|
1279
1527
|
kind: "event",
|
|
1280
1528
|
name: event.name,
|
|
1281
|
-
payload
|
|
1529
|
+
payload,
|
|
1282
1530
|
trace,
|
|
1283
1531
|
availableAt: options.availableAt,
|
|
1284
1532
|
maxAttempts: options.maxAttempts,
|
|
@@ -1317,9 +1565,14 @@ export function createOutboxEventRecorder(
|
|
|
1317
1565
|
): DomainEventRecorderPort {
|
|
1318
1566
|
return {
|
|
1319
1567
|
async record(event, payload, publishOptions) {
|
|
1320
|
-
|
|
1568
|
+
const prepared = await prepareEventPayloadForTransport(
|
|
1569
|
+
event,
|
|
1570
|
+
payload,
|
|
1571
|
+
publishOptions,
|
|
1572
|
+
);
|
|
1573
|
+
await enqueueTransportEvent(outbox, event, prepared.transportValue, {
|
|
1321
1574
|
...options,
|
|
1322
|
-
trace: publishOptions
|
|
1575
|
+
trace: prepared.publishOptions.trace ?? options.trace,
|
|
1323
1576
|
});
|
|
1324
1577
|
},
|
|
1325
1578
|
};
|
|
@@ -1392,7 +1645,7 @@ function shouldRetryOutboxMessage(
|
|
|
1392
1645
|
}
|
|
1393
1646
|
|
|
1394
1647
|
function outboxInstrumentationDetails(
|
|
1395
|
-
message:
|
|
1648
|
+
message: OutboxMessage,
|
|
1396
1649
|
details?: Record<string, unknown>,
|
|
1397
1650
|
): Record<string, unknown> {
|
|
1398
1651
|
return {
|
|
@@ -1424,12 +1677,16 @@ async function deliverOutboxMessage(
|
|
|
1424
1677
|
);
|
|
1425
1678
|
}
|
|
1426
1679
|
|
|
1427
|
-
await
|
|
1428
|
-
await options.eventBus.publish(
|
|
1680
|
+
const prepared = await prepareEventPayloadForTransport(
|
|
1429
1681
|
event,
|
|
1430
|
-
message.payload
|
|
1682
|
+
message.payload,
|
|
1431
1683
|
trace ? { trace } : undefined,
|
|
1432
1684
|
);
|
|
1685
|
+
await options.eventBus.publish(
|
|
1686
|
+
event,
|
|
1687
|
+
prepared.payload,
|
|
1688
|
+
prepared.publishOptions,
|
|
1689
|
+
);
|
|
1433
1690
|
return;
|
|
1434
1691
|
}
|
|
1435
1692
|
|
|
@@ -1474,20 +1731,1033 @@ async function deliverOutboxMessage(
|
|
|
1474
1731
|
);
|
|
1475
1732
|
}
|
|
1476
1733
|
|
|
1734
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
1735
|
+
const OUTBOX_SETTLEMENT_ATTEMPTS = 3;
|
|
1736
|
+
const OUTBOX_SETTLEMENT_RETRY_DELAY_MS = 100;
|
|
1737
|
+
|
|
1738
|
+
type ResolvedDrainRuntime = {
|
|
1739
|
+
batchSize: number;
|
|
1740
|
+
concurrency: number;
|
|
1741
|
+
leaseMs: number;
|
|
1742
|
+
heartbeatMs: number;
|
|
1743
|
+
maxActiveMs: number;
|
|
1744
|
+
now: () => Date;
|
|
1745
|
+
wait: OutboxDrainWait;
|
|
1746
|
+
};
|
|
1747
|
+
|
|
1748
|
+
type MessageDrainOutcome = {
|
|
1749
|
+
delivered: number;
|
|
1750
|
+
retried: number;
|
|
1751
|
+
deadLettered: number;
|
|
1752
|
+
settlementFailed: number;
|
|
1753
|
+
leaseLost: number;
|
|
1754
|
+
};
|
|
1755
|
+
|
|
1756
|
+
type ClaimHeartbeat = {
|
|
1757
|
+
lost: Promise<OutboxLeaseFailure>;
|
|
1758
|
+
stopAndExtend(): Promise<{
|
|
1759
|
+
lockedUntil: Date;
|
|
1760
|
+
renewalError?: unknown;
|
|
1761
|
+
renewalRecovered?: boolean;
|
|
1762
|
+
failure?: OutboxLeaseFailure;
|
|
1763
|
+
}>;
|
|
1764
|
+
stop(): Promise<void>;
|
|
1765
|
+
};
|
|
1766
|
+
|
|
1767
|
+
type BoundedRenewalResult =
|
|
1768
|
+
| { kind: "succeeded"; lockedUntil: Date }
|
|
1769
|
+
| { kind: "failed"; error: unknown }
|
|
1770
|
+
| { kind: "deadline" }
|
|
1771
|
+
| { kind: "stopped" }
|
|
1772
|
+
| { kind: "waitFailed"; error: unknown };
|
|
1773
|
+
|
|
1774
|
+
type BoundedSettlementResult =
|
|
1775
|
+
| { kind: "succeeded" }
|
|
1776
|
+
| { kind: "failed"; error: unknown }
|
|
1777
|
+
| { kind: "deadline" }
|
|
1778
|
+
| { kind: "waitFailed"; error: unknown };
|
|
1779
|
+
|
|
1780
|
+
function defaultOutboxWait(
|
|
1781
|
+
delayMs: number,
|
|
1782
|
+
signal: AbortSignal,
|
|
1783
|
+
): Promise<void> {
|
|
1784
|
+
return new Promise((resolve) => {
|
|
1785
|
+
if (signal.aborted) {
|
|
1786
|
+
resolve();
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1791
|
+
const finish = () => {
|
|
1792
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1793
|
+
signal.removeEventListener("abort", finish);
|
|
1794
|
+
resolve();
|
|
1795
|
+
};
|
|
1796
|
+
timer = setTimeout(finish, delayMs);
|
|
1797
|
+
signal.addEventListener("abort", finish, { once: true });
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
function readOutboxNow(now: () => Date): Date {
|
|
1802
|
+
const value = now();
|
|
1803
|
+
assertValidDate("now", value);
|
|
1804
|
+
return value;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
function assertTimerDuration(name: string, value: number): void {
|
|
1808
|
+
assertPositiveInteger(name, value);
|
|
1809
|
+
if (value > MAX_TIMER_DELAY_MS) {
|
|
1810
|
+
throw new Error(`${name} must be at most ${MAX_TIMER_DELAY_MS}`);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
function resolveDrainRuntime(
|
|
1815
|
+
options: DrainOutboxOptions,
|
|
1816
|
+
): ResolvedDrainRuntime {
|
|
1817
|
+
const batchSize = options.batchSize ?? DEFAULT_OUTBOX_BATCH_SIZE;
|
|
1818
|
+
const concurrency = options.concurrency ?? DEFAULT_OUTBOX_CONCURRENCY;
|
|
1819
|
+
const leaseMs = options.leaseMs ?? DEFAULT_OUTBOX_LEASE_MS;
|
|
1820
|
+
const heartbeatMs =
|
|
1821
|
+
options.heartbeatMs ?? Math.max(1, Math.floor(leaseMs / 3));
|
|
1822
|
+
const maxActiveMs = options.maxActiveMs ?? DEFAULT_OUTBOX_MAX_ACTIVE_MS;
|
|
1823
|
+
|
|
1824
|
+
assertPositiveInteger("batchSize", batchSize);
|
|
1825
|
+
assertPositiveInteger("concurrency", concurrency);
|
|
1826
|
+
if (concurrency > batchSize) {
|
|
1827
|
+
throw new Error("concurrency must be less than or equal to batchSize");
|
|
1828
|
+
}
|
|
1829
|
+
assertTimerDuration("leaseMs", leaseMs);
|
|
1830
|
+
if (leaseMs < 2) throw new Error("leaseMs must be at least 2");
|
|
1831
|
+
assertTimerDuration("heartbeatMs", heartbeatMs);
|
|
1832
|
+
if (heartbeatMs >= leaseMs) {
|
|
1833
|
+
throw new Error("heartbeatMs must be shorter than leaseMs");
|
|
1834
|
+
}
|
|
1835
|
+
assertTimerDuration("maxActiveMs", maxActiveMs);
|
|
1836
|
+
|
|
1837
|
+
return {
|
|
1838
|
+
batchSize,
|
|
1839
|
+
concurrency,
|
|
1840
|
+
leaseMs,
|
|
1841
|
+
heartbeatMs,
|
|
1842
|
+
maxActiveMs,
|
|
1843
|
+
now: options.now ?? (() => new Date()),
|
|
1844
|
+
wait: options.wait ?? defaultOutboxWait,
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
async function notifyLeaseFailure(
|
|
1849
|
+
options: DrainOutboxOptions,
|
|
1850
|
+
failure: OutboxLeaseFailure,
|
|
1851
|
+
): Promise<void> {
|
|
1852
|
+
try {
|
|
1853
|
+
await options.onLeaseError?.(failure);
|
|
1854
|
+
} catch {
|
|
1855
|
+
// Lease observers cannot change delivery ownership or recovery behavior.
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
async function notifySettlementFailure(
|
|
1860
|
+
options: DrainOutboxOptions,
|
|
1861
|
+
failure: OutboxSettlementFailure,
|
|
1862
|
+
): Promise<void> {
|
|
1863
|
+
try {
|
|
1864
|
+
await options.onSettlementError?.(failure);
|
|
1865
|
+
} catch {
|
|
1866
|
+
// Settlement observers cannot replace the unknown storage outcome.
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
|
|
1870
|
+
async function runBoundedClaimRenewal(options: {
|
|
1871
|
+
outbox: OutboxPort;
|
|
1872
|
+
message: ClaimedOutboxMessage;
|
|
1873
|
+
runtime: ResolvedDrainRuntime;
|
|
1874
|
+
renewalAt: Date;
|
|
1875
|
+
deadline: Date;
|
|
1876
|
+
signal: AbortSignal;
|
|
1877
|
+
}): Promise<BoundedRenewalResult> {
|
|
1878
|
+
const remainingMs = options.deadline.getTime() - options.renewalAt.getTime();
|
|
1879
|
+
if (remainingMs <= 0) return { kind: "deadline" };
|
|
1880
|
+
if (options.signal.aborted) return { kind: "stopped" };
|
|
1881
|
+
|
|
1882
|
+
const waitController = new AbortController();
|
|
1883
|
+
const stopWaiting = () => waitController.abort();
|
|
1884
|
+
options.signal.addEventListener("abort", stopWaiting, { once: true });
|
|
1885
|
+
|
|
1886
|
+
const renewal = Promise.resolve()
|
|
1887
|
+
.then(() =>
|
|
1888
|
+
options.outbox.renewClaim({
|
|
1889
|
+
id: options.message.id,
|
|
1890
|
+
claimToken: options.message.claimToken,
|
|
1891
|
+
now: options.renewalAt,
|
|
1892
|
+
leaseMs: options.runtime.leaseMs,
|
|
1893
|
+
}),
|
|
1894
|
+
)
|
|
1895
|
+
.then(
|
|
1896
|
+
(result) => ({ kind: "succeeded" as const, result }),
|
|
1897
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
1898
|
+
);
|
|
1899
|
+
const deadline = Promise.resolve()
|
|
1900
|
+
.then(() => options.runtime.wait(remainingMs, waitController.signal))
|
|
1901
|
+
.then(
|
|
1902
|
+
() =>
|
|
1903
|
+
options.signal.aborted
|
|
1904
|
+
? { kind: "stopped" as const }
|
|
1905
|
+
: { kind: "deadline" as const },
|
|
1906
|
+
(error: unknown) =>
|
|
1907
|
+
options.signal.aborted
|
|
1908
|
+
? { kind: "stopped" as const }
|
|
1909
|
+
: { kind: "waitFailed" as const, error },
|
|
1910
|
+
);
|
|
1911
|
+
|
|
1912
|
+
const result = await Promise.race([renewal, deadline]);
|
|
1913
|
+
options.signal.removeEventListener("abort", stopWaiting);
|
|
1914
|
+
waitController.abort();
|
|
1915
|
+
|
|
1916
|
+
if (result.kind !== "succeeded") return result;
|
|
1917
|
+
try {
|
|
1918
|
+
const completedAt = readOutboxNow(options.runtime.now);
|
|
1919
|
+
if (completedAt.getTime() >= options.deadline.getTime()) {
|
|
1920
|
+
return { kind: "deadline" };
|
|
1921
|
+
}
|
|
1922
|
+
assertValidDate("lockedUntil", result.result.lockedUntil);
|
|
1923
|
+
if (result.result.lockedUntil.getTime() <= completedAt.getTime()) {
|
|
1924
|
+
return {
|
|
1925
|
+
kind: "failed",
|
|
1926
|
+
error: new Error(
|
|
1927
|
+
`Outbox claim renewal for message "${options.message.id}" returned an expired lease.`,
|
|
1928
|
+
),
|
|
1929
|
+
};
|
|
1930
|
+
}
|
|
1931
|
+
return {
|
|
1932
|
+
kind: "succeeded",
|
|
1933
|
+
lockedUntil: cloneDate(result.result.lockedUntil),
|
|
1934
|
+
};
|
|
1935
|
+
} catch (error) {
|
|
1936
|
+
return { kind: "failed", error };
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
function createClaimHeartbeat(
|
|
1941
|
+
options: DrainOutboxOptions,
|
|
1942
|
+
runtime: ResolvedDrainRuntime,
|
|
1943
|
+
message: ClaimedOutboxMessage,
|
|
1944
|
+
activeUntil: Date,
|
|
1945
|
+
): ClaimHeartbeat {
|
|
1946
|
+
const controller = new AbortController();
|
|
1947
|
+
let stopped = false;
|
|
1948
|
+
let lockedUntil = cloneDate(message.lockedUntil);
|
|
1949
|
+
let renewing = false;
|
|
1950
|
+
let firstRenewalError: unknown;
|
|
1951
|
+
let terminalFailure: OutboxLeaseFailure | undefined;
|
|
1952
|
+
let resolveLost: (failure: OutboxLeaseFailure) => void = () => {};
|
|
1953
|
+
const lost = new Promise<OutboxLeaseFailure>((resolve) => {
|
|
1954
|
+
resolveLost = resolve;
|
|
1955
|
+
});
|
|
1956
|
+
|
|
1957
|
+
const fail = (failure: OutboxLeaseFailure) => {
|
|
1958
|
+
if (terminalFailure) return;
|
|
1959
|
+
terminalFailure = failure;
|
|
1960
|
+
stopped = true;
|
|
1961
|
+
controller.abort();
|
|
1962
|
+
resolveLost(failure);
|
|
1963
|
+
};
|
|
1964
|
+
|
|
1965
|
+
const run = async () => {
|
|
1966
|
+
try {
|
|
1967
|
+
let delayMs = runtime.heartbeatMs;
|
|
1968
|
+
while (!stopped) {
|
|
1969
|
+
try {
|
|
1970
|
+
await runtime.wait(delayMs, controller.signal);
|
|
1971
|
+
} catch (error) {
|
|
1972
|
+
if (stopped || controller.signal.aborted) return;
|
|
1973
|
+
fail({
|
|
1974
|
+
error: new OutboxLeaseLostError({
|
|
1975
|
+
id: message.id,
|
|
1976
|
+
message: `Outbox heartbeat scheduling failed for message "${message.id}".`,
|
|
1977
|
+
cause: error,
|
|
1978
|
+
}),
|
|
1979
|
+
message,
|
|
1980
|
+
operation: "renewClaim",
|
|
1981
|
+
state: "lost",
|
|
1982
|
+
confirmedLost: false,
|
|
1983
|
+
});
|
|
1984
|
+
return;
|
|
1985
|
+
}
|
|
1986
|
+
if (stopped || controller.signal.aborted) return;
|
|
1987
|
+
|
|
1988
|
+
const renewalAt = readOutboxNow(runtime.now);
|
|
1989
|
+
const renewalDeadline = new Date(
|
|
1990
|
+
Math.min(lockedUntil.getTime(), activeUntil.getTime()),
|
|
1991
|
+
);
|
|
1992
|
+
const activeDeadlineEndsFirst =
|
|
1993
|
+
activeUntil.getTime() <= lockedUntil.getTime();
|
|
1994
|
+
if (renewalAt.getTime() >= renewalDeadline.getTime()) {
|
|
1995
|
+
fail({
|
|
1996
|
+
error: activeDeadlineEndsFirst
|
|
1997
|
+
? new OutboxMaxActiveDurationError({
|
|
1998
|
+
id: message.id,
|
|
1999
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2000
|
+
})
|
|
2001
|
+
: new OutboxLeaseLostError({ id: message.id }),
|
|
2002
|
+
message,
|
|
2003
|
+
operation: activeDeadlineEndsFirst
|
|
2004
|
+
? "maxActiveDuration"
|
|
2005
|
+
: "renewClaim",
|
|
2006
|
+
state: "lost",
|
|
2007
|
+
confirmedLost: !activeDeadlineEndsFirst,
|
|
2008
|
+
});
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
renewing = true;
|
|
2013
|
+
let renewal: BoundedRenewalResult;
|
|
2014
|
+
try {
|
|
2015
|
+
renewal = await runBoundedClaimRenewal({
|
|
2016
|
+
outbox: options.outbox,
|
|
2017
|
+
message,
|
|
2018
|
+
runtime,
|
|
2019
|
+
renewalAt,
|
|
2020
|
+
deadline: renewalDeadline,
|
|
2021
|
+
signal: controller.signal,
|
|
2022
|
+
});
|
|
2023
|
+
} catch (error) {
|
|
2024
|
+
renewal = { kind: "failed", error };
|
|
2025
|
+
} finally {
|
|
2026
|
+
renewing = false;
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
if (renewal.kind === "stopped") return;
|
|
2030
|
+
if (renewal.kind === "waitFailed") {
|
|
2031
|
+
fail({
|
|
2032
|
+
error: new OutboxLeaseLostError({
|
|
2033
|
+
id: message.id,
|
|
2034
|
+
message: `Could not enforce the renewal deadline for outbox message "${message.id}".`,
|
|
2035
|
+
cause: renewal.error,
|
|
2036
|
+
}),
|
|
2037
|
+
message,
|
|
2038
|
+
operation: "renewClaim",
|
|
2039
|
+
state: "lost",
|
|
2040
|
+
confirmedLost: false,
|
|
2041
|
+
});
|
|
2042
|
+
return;
|
|
2043
|
+
}
|
|
2044
|
+
if (renewal.kind === "deadline") {
|
|
2045
|
+
fail({
|
|
2046
|
+
error: activeDeadlineEndsFirst
|
|
2047
|
+
? new OutboxMaxActiveDurationError({
|
|
2048
|
+
id: message.id,
|
|
2049
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2050
|
+
})
|
|
2051
|
+
: new OutboxLeaseLostError({
|
|
2052
|
+
id: message.id,
|
|
2053
|
+
message: `Outbox claim renewal for message "${message.id}" did not complete before the lease deadline.`,
|
|
2054
|
+
}),
|
|
2055
|
+
message,
|
|
2056
|
+
operation: activeDeadlineEndsFirst
|
|
2057
|
+
? "maxActiveDuration"
|
|
2058
|
+
: "renewClaim",
|
|
2059
|
+
state: "lost",
|
|
2060
|
+
confirmedLost: false,
|
|
2061
|
+
});
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
if (renewal.kind === "succeeded") {
|
|
2065
|
+
lockedUntil = renewal.lockedUntil;
|
|
2066
|
+
delayMs = runtime.heartbeatMs;
|
|
2067
|
+
continue;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
const error = renewal.error;
|
|
2071
|
+
if (error instanceof OutboxClaimError) {
|
|
2072
|
+
fail({
|
|
2073
|
+
error,
|
|
2074
|
+
message,
|
|
2075
|
+
operation: "renewClaim",
|
|
2076
|
+
state: "lost",
|
|
2077
|
+
confirmedLost: true,
|
|
2078
|
+
});
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
firstRenewalError ??= error;
|
|
2083
|
+
if (stopped) return;
|
|
2084
|
+
const retryAt = readOutboxNow(runtime.now);
|
|
2085
|
+
const remainingMs = renewalDeadline.getTime() - retryAt.getTime();
|
|
2086
|
+
if (remainingMs <= 0) {
|
|
2087
|
+
fail({
|
|
2088
|
+
error: activeDeadlineEndsFirst
|
|
2089
|
+
? new OutboxMaxActiveDurationError({
|
|
2090
|
+
id: message.id,
|
|
2091
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2092
|
+
})
|
|
2093
|
+
: new OutboxLeaseLostError({
|
|
2094
|
+
id: message.id,
|
|
2095
|
+
message: `Outbox claim renewal for message "${message.id}" did not recover before the lease expired.`,
|
|
2096
|
+
cause: error,
|
|
2097
|
+
}),
|
|
2098
|
+
message,
|
|
2099
|
+
operation: activeDeadlineEndsFirst
|
|
2100
|
+
? "maxActiveDuration"
|
|
2101
|
+
: "renewClaim",
|
|
2102
|
+
state: "lost",
|
|
2103
|
+
confirmedLost: false,
|
|
2104
|
+
});
|
|
2105
|
+
return;
|
|
2106
|
+
}
|
|
2107
|
+
delayMs = Math.max(
|
|
2108
|
+
1,
|
|
2109
|
+
Math.min(runtime.heartbeatMs, Math.floor(remainingMs / 3)),
|
|
2110
|
+
);
|
|
2111
|
+
}
|
|
2112
|
+
} catch (error) {
|
|
2113
|
+
if (stopped || controller.signal.aborted) return;
|
|
2114
|
+
fail({
|
|
2115
|
+
error: new OutboxLeaseLostError({
|
|
2116
|
+
id: message.id,
|
|
2117
|
+
message: `Outbox claim renewal failed unexpectedly for message "${message.id}".`,
|
|
2118
|
+
cause: error,
|
|
2119
|
+
}),
|
|
2120
|
+
message,
|
|
2121
|
+
operation: "renewClaim",
|
|
2122
|
+
state: "lost",
|
|
2123
|
+
confirmedLost: false,
|
|
2124
|
+
});
|
|
2125
|
+
}
|
|
2126
|
+
};
|
|
2127
|
+
|
|
2128
|
+
const running = run();
|
|
2129
|
+
|
|
2130
|
+
const stop = async (abortRenewal = true) => {
|
|
2131
|
+
stopped = true;
|
|
2132
|
+
if (abortRenewal || !renewing) controller.abort();
|
|
2133
|
+
await running;
|
|
2134
|
+
controller.abort();
|
|
2135
|
+
};
|
|
2136
|
+
|
|
2137
|
+
return {
|
|
2138
|
+
lost,
|
|
2139
|
+
stop: () => stop(true),
|
|
2140
|
+
async stopAndExtend() {
|
|
2141
|
+
await stop(false);
|
|
2142
|
+
if (terminalFailure) {
|
|
2143
|
+
return { lockedUntil, failure: terminalFailure };
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
const renewalAt = readOutboxNow(runtime.now);
|
|
2147
|
+
const activeDeadlineEndsFirst =
|
|
2148
|
+
activeUntil.getTime() <= lockedUntil.getTime();
|
|
2149
|
+
const finalRenewalDeadline = new Date(
|
|
2150
|
+
Math.min(lockedUntil.getTime(), activeUntil.getTime()),
|
|
2151
|
+
);
|
|
2152
|
+
if (renewalAt.getTime() >= finalRenewalDeadline.getTime()) {
|
|
2153
|
+
return {
|
|
2154
|
+
lockedUntil,
|
|
2155
|
+
failure: {
|
|
2156
|
+
error: activeDeadlineEndsFirst
|
|
2157
|
+
? new OutboxMaxActiveDurationError({
|
|
2158
|
+
id: message.id,
|
|
2159
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2160
|
+
})
|
|
2161
|
+
: new OutboxLeaseLostError({ id: message.id }),
|
|
2162
|
+
message,
|
|
2163
|
+
operation: activeDeadlineEndsFirst
|
|
2164
|
+
? "maxActiveDuration"
|
|
2165
|
+
: "renewClaim",
|
|
2166
|
+
state: "lost",
|
|
2167
|
+
confirmedLost: !activeDeadlineEndsFirst,
|
|
2168
|
+
},
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
const finalController = new AbortController();
|
|
2173
|
+
const renewal = await runBoundedClaimRenewal({
|
|
2174
|
+
outbox: options.outbox,
|
|
2175
|
+
message,
|
|
2176
|
+
runtime,
|
|
2177
|
+
renewalAt,
|
|
2178
|
+
deadline: finalRenewalDeadline,
|
|
2179
|
+
signal: finalController.signal,
|
|
2180
|
+
});
|
|
2181
|
+
finalController.abort();
|
|
2182
|
+
|
|
2183
|
+
if (renewal.kind === "succeeded") {
|
|
2184
|
+
return {
|
|
2185
|
+
lockedUntil: renewal.lockedUntil,
|
|
2186
|
+
renewalError: firstRenewalError,
|
|
2187
|
+
renewalRecovered: firstRenewalError === undefined ? undefined : true,
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
if (renewal.kind === "failed") {
|
|
2191
|
+
if (!(renewal.error instanceof OutboxClaimError)) {
|
|
2192
|
+
const failedAt = readOutboxNow(runtime.now);
|
|
2193
|
+
if (
|
|
2194
|
+
failedAt.getTime() < lockedUntil.getTime() &&
|
|
2195
|
+
failedAt.getTime() < activeUntil.getTime()
|
|
2196
|
+
) {
|
|
2197
|
+
return {
|
|
2198
|
+
lockedUntil,
|
|
2199
|
+
renewalError: renewal.error,
|
|
2200
|
+
renewalRecovered: false,
|
|
2201
|
+
};
|
|
2202
|
+
}
|
|
2203
|
+
if (
|
|
2204
|
+
activeDeadlineEndsFirst &&
|
|
2205
|
+
failedAt.getTime() >= activeUntil.getTime()
|
|
2206
|
+
) {
|
|
2207
|
+
return {
|
|
2208
|
+
lockedUntil,
|
|
2209
|
+
failure: {
|
|
2210
|
+
error: new OutboxMaxActiveDurationError({
|
|
2211
|
+
id: message.id,
|
|
2212
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2213
|
+
}),
|
|
2214
|
+
message,
|
|
2215
|
+
operation: "maxActiveDuration",
|
|
2216
|
+
state: "lost",
|
|
2217
|
+
confirmedLost: false,
|
|
2218
|
+
},
|
|
2219
|
+
};
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
return {
|
|
2223
|
+
lockedUntil,
|
|
2224
|
+
failure: {
|
|
2225
|
+
error: renewal.error,
|
|
2226
|
+
message,
|
|
2227
|
+
operation: "renewClaim",
|
|
2228
|
+
state: "lost",
|
|
2229
|
+
confirmedLost: renewal.error instanceof OutboxClaimError,
|
|
2230
|
+
},
|
|
2231
|
+
};
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
return {
|
|
2235
|
+
lockedUntil,
|
|
2236
|
+
failure: {
|
|
2237
|
+
error:
|
|
2238
|
+
renewal.kind === "waitFailed"
|
|
2239
|
+
? new OutboxLeaseLostError({
|
|
2240
|
+
id: message.id,
|
|
2241
|
+
message: activeDeadlineEndsFirst
|
|
2242
|
+
? `Could not enforce the maximum active duration for outbox message "${message.id}".`
|
|
2243
|
+
: `Could not enforce the final renewal deadline for outbox message "${message.id}".`,
|
|
2244
|
+
cause: renewal.error,
|
|
2245
|
+
})
|
|
2246
|
+
: activeDeadlineEndsFirst
|
|
2247
|
+
? new OutboxMaxActiveDurationError({
|
|
2248
|
+
id: message.id,
|
|
2249
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2250
|
+
})
|
|
2251
|
+
: new OutboxLeaseLostError({
|
|
2252
|
+
id: message.id,
|
|
2253
|
+
message: `Outbox claim renewal for message "${message.id}" did not complete before settlement.`,
|
|
2254
|
+
}),
|
|
2255
|
+
message,
|
|
2256
|
+
operation: activeDeadlineEndsFirst
|
|
2257
|
+
? "maxActiveDuration"
|
|
2258
|
+
: "renewClaim",
|
|
2259
|
+
state: "lost",
|
|
2260
|
+
confirmedLost: false,
|
|
2261
|
+
},
|
|
2262
|
+
};
|
|
2263
|
+
},
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
async function settleClaim(options: {
|
|
2268
|
+
operation: OutboxSettlementFailure["operation"];
|
|
2269
|
+
message: ClaimedOutboxMessage;
|
|
2270
|
+
lockedUntil: Date;
|
|
2271
|
+
activeUntil: Date;
|
|
2272
|
+
runtime: ResolvedDrainRuntime;
|
|
2273
|
+
settle(now: Date): Promise<void>;
|
|
2274
|
+
}): Promise<{ ok: true } | { ok: false; error: unknown; claimLost: boolean }> {
|
|
2275
|
+
let lastError: unknown;
|
|
2276
|
+
|
|
2277
|
+
for (let attempt = 1; attempt <= OUTBOX_SETTLEMENT_ATTEMPTS; attempt += 1) {
|
|
2278
|
+
const settlementAt = readOutboxNow(options.runtime.now);
|
|
2279
|
+
const settlementDeadline = new Date(
|
|
2280
|
+
Math.min(options.lockedUntil.getTime(), options.activeUntil.getTime()),
|
|
2281
|
+
);
|
|
2282
|
+
const settlement = await runBoundedSettlement({
|
|
2283
|
+
runtime: options.runtime,
|
|
2284
|
+
settlementAt,
|
|
2285
|
+
deadline: settlementDeadline,
|
|
2286
|
+
settle: options.settle,
|
|
2287
|
+
});
|
|
2288
|
+
if (settlement.kind === "succeeded") {
|
|
2289
|
+
return { ok: true };
|
|
2290
|
+
}
|
|
2291
|
+
if (settlement.kind === "deadline") {
|
|
2292
|
+
return {
|
|
2293
|
+
ok: false,
|
|
2294
|
+
error: new OutboxLeaseLostError({
|
|
2295
|
+
id: options.message.id,
|
|
2296
|
+
message: `Outbox ${options.operation} for message "${options.message.id}" did not complete before its settlement deadline.`,
|
|
2297
|
+
}),
|
|
2298
|
+
claimLost: true,
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
if (settlement.kind === "waitFailed") {
|
|
2302
|
+
return { ok: false, error: settlement.error, claimLost: false };
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
const error = settlement.error;
|
|
2306
|
+
lastError = error;
|
|
2307
|
+
if (error instanceof OutboxClaimError) {
|
|
2308
|
+
return { ok: false, error, claimLost: true };
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
const remainingMs =
|
|
2312
|
+
settlementDeadline.getTime() -
|
|
2313
|
+
readOutboxNow(options.runtime.now).getTime();
|
|
2314
|
+
if (attempt >= OUTBOX_SETTLEMENT_ATTEMPTS || remainingMs <= 0) break;
|
|
2315
|
+
|
|
2316
|
+
const waitController = new AbortController();
|
|
2317
|
+
try {
|
|
2318
|
+
await options.runtime.wait(
|
|
2319
|
+
Math.max(
|
|
2320
|
+
1,
|
|
2321
|
+
Math.min(
|
|
2322
|
+
OUTBOX_SETTLEMENT_RETRY_DELAY_MS * 2 ** (attempt - 1),
|
|
2323
|
+
remainingMs,
|
|
2324
|
+
),
|
|
2325
|
+
),
|
|
2326
|
+
waitController.signal,
|
|
2327
|
+
);
|
|
2328
|
+
} catch (error) {
|
|
2329
|
+
return { ok: false, error, claimLost: false };
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
return { ok: false, error: lastError, claimLost: false };
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
async function runBoundedSettlement(options: {
|
|
2337
|
+
runtime: ResolvedDrainRuntime;
|
|
2338
|
+
settlementAt: Date;
|
|
2339
|
+
deadline: Date;
|
|
2340
|
+
settle(now: Date): Promise<void>;
|
|
2341
|
+
}): Promise<BoundedSettlementResult> {
|
|
2342
|
+
const remainingMs =
|
|
2343
|
+
options.deadline.getTime() - options.settlementAt.getTime();
|
|
2344
|
+
if (remainingMs <= 0) return { kind: "deadline" };
|
|
2345
|
+
|
|
2346
|
+
const waitController = new AbortController();
|
|
2347
|
+
const settlement = Promise.resolve()
|
|
2348
|
+
.then(() => options.settle(options.settlementAt))
|
|
2349
|
+
.then(
|
|
2350
|
+
() => ({ kind: "succeeded" as const }),
|
|
2351
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
2352
|
+
);
|
|
2353
|
+
const deadline = Promise.resolve()
|
|
2354
|
+
.then(() => options.runtime.wait(remainingMs, waitController.signal))
|
|
2355
|
+
.then(
|
|
2356
|
+
() => ({ kind: "deadline" as const }),
|
|
2357
|
+
(error: unknown) => ({ kind: "waitFailed" as const, error }),
|
|
2358
|
+
);
|
|
2359
|
+
|
|
2360
|
+
const result = await Promise.race([settlement, deadline]);
|
|
2361
|
+
waitController.abort();
|
|
2362
|
+
return result;
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
async function recordDeadLetter(
|
|
2366
|
+
options: DrainOutboxOptions,
|
|
2367
|
+
instrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2368
|
+
jobInstrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2369
|
+
message: OutboxMessage,
|
|
2370
|
+
error: unknown,
|
|
2371
|
+
details: Record<string, unknown> = {},
|
|
2372
|
+
): Promise<void> {
|
|
2373
|
+
try {
|
|
2374
|
+
await options.onDeadLetter?.(error, message);
|
|
2375
|
+
} catch {
|
|
2376
|
+
// Dead-letter observers must not change the settled message state.
|
|
2377
|
+
}
|
|
2378
|
+
instrumentation.record({
|
|
2379
|
+
type: "outbox",
|
|
2380
|
+
...options.instrumentationContext,
|
|
2381
|
+
messageId: message.id,
|
|
2382
|
+
messageKind: message.kind,
|
|
2383
|
+
messageName: message.name,
|
|
2384
|
+
status: "deadLettered",
|
|
2385
|
+
details: outboxInstrumentationDetails(message, {
|
|
2386
|
+
...details,
|
|
2387
|
+
error: serializeOutboxError(error),
|
|
2388
|
+
}),
|
|
2389
|
+
});
|
|
2390
|
+
if (message.kind === "job") {
|
|
2391
|
+
jobInstrumentation.record({
|
|
2392
|
+
type: "job",
|
|
2393
|
+
...options.instrumentationContext,
|
|
2394
|
+
jobName: message.name,
|
|
2395
|
+
status: "deadLettered",
|
|
2396
|
+
details: outboxInstrumentationDetails(message, {
|
|
2397
|
+
...details,
|
|
2398
|
+
error: serializeOutboxError(error),
|
|
2399
|
+
}),
|
|
2400
|
+
});
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
|
|
2404
|
+
async function processClaimedMessage(
|
|
2405
|
+
options: DrainOutboxOptions,
|
|
2406
|
+
runtime: ResolvedDrainRuntime,
|
|
2407
|
+
instrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2408
|
+
jobInstrumentation: ReturnType<typeof createProviderInstrumentation>,
|
|
2409
|
+
tracing: TracingPort | undefined,
|
|
2410
|
+
message: ClaimedOutboxMessage,
|
|
2411
|
+
): Promise<MessageDrainOutcome> {
|
|
2412
|
+
const emptyOutcome = (): MessageDrainOutcome => ({
|
|
2413
|
+
delivered: 0,
|
|
2414
|
+
retried: 0,
|
|
2415
|
+
deadLettered: 0,
|
|
2416
|
+
settlementFailed: 0,
|
|
2417
|
+
leaseLost: 0,
|
|
2418
|
+
});
|
|
2419
|
+
const outcome = emptyOutcome();
|
|
2420
|
+
const startedAt = readOutboxNow(runtime.now);
|
|
2421
|
+
const activeUntil = new Date(startedAt.getTime() + runtime.maxActiveMs);
|
|
2422
|
+
const heartbeat = createClaimHeartbeat(
|
|
2423
|
+
options,
|
|
2424
|
+
runtime,
|
|
2425
|
+
message,
|
|
2426
|
+
activeUntil,
|
|
2427
|
+
);
|
|
2428
|
+
const lifetimeController = new AbortController();
|
|
2429
|
+
|
|
2430
|
+
const parentTrace = parseTraceCarrier(message.trace);
|
|
2431
|
+
const traceAttributes = {
|
|
2432
|
+
"beignet.outbox.message_kind": message.kind,
|
|
2433
|
+
"beignet.outbox.message_name": message.name,
|
|
2434
|
+
} as const;
|
|
2435
|
+
const delivery = Promise.resolve()
|
|
2436
|
+
.then(() =>
|
|
2437
|
+
runWithTracing(
|
|
2438
|
+
tracing,
|
|
2439
|
+
{
|
|
2440
|
+
name: `beignet.outbox deliver ${message.name}`,
|
|
2441
|
+
type: "outbox",
|
|
2442
|
+
kind: "consumer",
|
|
2443
|
+
parent: parentTrace,
|
|
2444
|
+
attributes: traceAttributes,
|
|
2445
|
+
metricAttributes: traceAttributes,
|
|
2446
|
+
},
|
|
2447
|
+
(span) =>
|
|
2448
|
+
deliverOutboxMessage(
|
|
2449
|
+
options,
|
|
2450
|
+
message,
|
|
2451
|
+
captureTraceCarrier(span?.context ?? parentTrace),
|
|
2452
|
+
),
|
|
2453
|
+
),
|
|
2454
|
+
)
|
|
2455
|
+
.then(
|
|
2456
|
+
() => ({ kind: "succeeded" as const }),
|
|
2457
|
+
(error: unknown) => ({ kind: "failed" as const, error }),
|
|
2458
|
+
);
|
|
2459
|
+
const maximumActive = Promise.resolve()
|
|
2460
|
+
.then(() => runtime.wait(runtime.maxActiveMs, lifetimeController.signal))
|
|
2461
|
+
.then(
|
|
2462
|
+
() => ({ kind: "maxActive" as const }),
|
|
2463
|
+
(error: unknown) => ({ kind: "waitFailed" as const, error }),
|
|
2464
|
+
);
|
|
2465
|
+
const leaseLost = heartbeat.lost.then((failure) => ({
|
|
2466
|
+
kind: "leaseLost" as const,
|
|
2467
|
+
failure,
|
|
2468
|
+
}));
|
|
2469
|
+
|
|
2470
|
+
const deliveryOutcome = await Promise.race([
|
|
2471
|
+
delivery,
|
|
2472
|
+
maximumActive,
|
|
2473
|
+
leaseLost,
|
|
2474
|
+
]);
|
|
2475
|
+
lifetimeController.abort();
|
|
2476
|
+
|
|
2477
|
+
if (deliveryOutcome.kind === "maxActive") {
|
|
2478
|
+
await heartbeat.stop();
|
|
2479
|
+
const failure: OutboxLeaseFailure = {
|
|
2480
|
+
error: new OutboxMaxActiveDurationError({
|
|
2481
|
+
id: message.id,
|
|
2482
|
+
maxActiveMs: runtime.maxActiveMs,
|
|
2483
|
+
}),
|
|
2484
|
+
message,
|
|
2485
|
+
operation: "maxActiveDuration",
|
|
2486
|
+
state: "lost",
|
|
2487
|
+
confirmedLost: false,
|
|
2488
|
+
};
|
|
2489
|
+
await notifyLeaseFailure(options, failure);
|
|
2490
|
+
instrumentation.custom({
|
|
2491
|
+
name: "outbox.lease.lost",
|
|
2492
|
+
label: "Outbox claim no longer confirmed",
|
|
2493
|
+
summary: `Stopped renewing ${message.kind} "${message.name}" after its active-delivery limit`,
|
|
2494
|
+
details: outboxInstrumentationDetails(message, {
|
|
2495
|
+
operation: failure.operation,
|
|
2496
|
+
state: failure.state,
|
|
2497
|
+
error: serializeOutboxError(failure.error),
|
|
2498
|
+
}),
|
|
2499
|
+
});
|
|
2500
|
+
outcome.leaseLost = 1;
|
|
2501
|
+
return outcome;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
if (deliveryOutcome.kind === "waitFailed") {
|
|
2505
|
+
await heartbeat.stop();
|
|
2506
|
+
const failure: OutboxLeaseFailure = {
|
|
2507
|
+
error: new OutboxLeaseLostError({
|
|
2508
|
+
id: message.id,
|
|
2509
|
+
message: `Could not enforce the active-delivery limit for outbox message "${message.id}".`,
|
|
2510
|
+
cause: deliveryOutcome.error,
|
|
2511
|
+
}),
|
|
2512
|
+
message,
|
|
2513
|
+
operation: "maxActiveDuration",
|
|
2514
|
+
state: "lost",
|
|
2515
|
+
confirmedLost: false,
|
|
2516
|
+
};
|
|
2517
|
+
await notifyLeaseFailure(options, failure);
|
|
2518
|
+
instrumentation.custom({
|
|
2519
|
+
name: "outbox.lease.lost",
|
|
2520
|
+
label: "Outbox claim no longer confirmed",
|
|
2521
|
+
summary: `Could not enforce the active-delivery limit for ${message.kind} "${message.name}"`,
|
|
2522
|
+
details: outboxInstrumentationDetails(message, {
|
|
2523
|
+
operation: failure.operation,
|
|
2524
|
+
state: failure.state,
|
|
2525
|
+
error: serializeOutboxError(failure.error),
|
|
2526
|
+
}),
|
|
2527
|
+
});
|
|
2528
|
+
outcome.leaseLost = 1;
|
|
2529
|
+
return outcome;
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
if (deliveryOutcome.kind === "leaseLost") {
|
|
2533
|
+
await heartbeat.stop();
|
|
2534
|
+
await notifyLeaseFailure(options, deliveryOutcome.failure);
|
|
2535
|
+
instrumentation.custom({
|
|
2536
|
+
name: "outbox.lease.lost",
|
|
2537
|
+
label: "Outbox claim lost",
|
|
2538
|
+
summary: `Could not keep the claim for ${message.kind} "${message.name}" active`,
|
|
2539
|
+
details: outboxInstrumentationDetails(message, {
|
|
2540
|
+
operation: deliveryOutcome.failure.operation,
|
|
2541
|
+
state: deliveryOutcome.failure.state,
|
|
2542
|
+
confirmedLost: deliveryOutcome.failure.confirmedLost,
|
|
2543
|
+
error: serializeOutboxError(deliveryOutcome.failure.error),
|
|
2544
|
+
}),
|
|
2545
|
+
});
|
|
2546
|
+
outcome.leaseLost = 1;
|
|
2547
|
+
return outcome;
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
const lease = await heartbeat.stopAndExtend();
|
|
2551
|
+
if (lease.failure) {
|
|
2552
|
+
await notifyLeaseFailure(options, lease.failure);
|
|
2553
|
+
instrumentation.custom({
|
|
2554
|
+
name: "outbox.lease.lost",
|
|
2555
|
+
label: "Outbox claim lost",
|
|
2556
|
+
summary: `Could not confirm the claim for ${message.kind} "${message.name}" before settlement`,
|
|
2557
|
+
details: outboxInstrumentationDetails(message, {
|
|
2558
|
+
operation: lease.failure.operation,
|
|
2559
|
+
state: lease.failure.state,
|
|
2560
|
+
confirmedLost: lease.failure.confirmedLost,
|
|
2561
|
+
error: serializeOutboxError(lease.failure.error),
|
|
2562
|
+
}),
|
|
2563
|
+
});
|
|
2564
|
+
outcome.leaseLost = 1;
|
|
2565
|
+
return outcome;
|
|
2566
|
+
}
|
|
2567
|
+
|
|
2568
|
+
const reportRenewalOutcome = async () => {
|
|
2569
|
+
if (lease.renewalError === undefined) return;
|
|
2570
|
+
const failure: OutboxLeaseFailure = {
|
|
2571
|
+
error: lease.renewalError,
|
|
2572
|
+
message,
|
|
2573
|
+
operation: "renewClaim",
|
|
2574
|
+
state: lease.renewalRecovered ? "recovered" : "degraded",
|
|
2575
|
+
confirmedLost: false,
|
|
2576
|
+
};
|
|
2577
|
+
await notifyLeaseFailure(options, failure);
|
|
2578
|
+
instrumentation.custom({
|
|
2579
|
+
name: lease.renewalRecovered
|
|
2580
|
+
? "outbox.lease.renewal.recovered"
|
|
2581
|
+
: "outbox.lease.renewal.degraded",
|
|
2582
|
+
label: lease.renewalRecovered
|
|
2583
|
+
? "Outbox claim renewal recovered"
|
|
2584
|
+
: "Outbox claim renewal degraded",
|
|
2585
|
+
summary: lease.renewalRecovered
|
|
2586
|
+
? `Recovered claim renewal for ${message.kind} "${message.name}"`
|
|
2587
|
+
: `Continued ${message.kind} "${message.name}" settlement under its last confirmed lease`,
|
|
2588
|
+
details: outboxInstrumentationDetails(message, {
|
|
2589
|
+
state: failure.state,
|
|
2590
|
+
error: serializeOutboxError(lease.renewalError),
|
|
2591
|
+
}),
|
|
2592
|
+
});
|
|
2593
|
+
};
|
|
2594
|
+
|
|
2595
|
+
if (deliveryOutcome.kind === "succeeded") {
|
|
2596
|
+
const settlement = await settleClaim({
|
|
2597
|
+
operation: "markDelivered",
|
|
2598
|
+
message,
|
|
2599
|
+
lockedUntil: lease.lockedUntil,
|
|
2600
|
+
activeUntil,
|
|
2601
|
+
runtime,
|
|
2602
|
+
settle: (now) =>
|
|
2603
|
+
options.outbox.markDelivered({
|
|
2604
|
+
id: message.id,
|
|
2605
|
+
claimToken: message.claimToken,
|
|
2606
|
+
now,
|
|
2607
|
+
}),
|
|
2608
|
+
});
|
|
2609
|
+
await reportRenewalOutcome();
|
|
2610
|
+
if (!settlement.ok) {
|
|
2611
|
+
const failure: OutboxSettlementFailure = {
|
|
2612
|
+
error: settlement.error,
|
|
2613
|
+
message,
|
|
2614
|
+
operation: "markDelivered",
|
|
2615
|
+
deliverySucceeded: true,
|
|
2616
|
+
};
|
|
2617
|
+
await notifySettlementFailure(options, failure);
|
|
2618
|
+
instrumentation.custom({
|
|
2619
|
+
name: "outbox.settlement.failed",
|
|
2620
|
+
label: "Outbox settlement failed",
|
|
2621
|
+
summary: `Delivered ${message.kind} "${message.name}", but could not confirm its durable acknowledgement`,
|
|
2622
|
+
details: outboxInstrumentationDetails(message, {
|
|
2623
|
+
operation: failure.operation,
|
|
2624
|
+
deliverySucceeded: true,
|
|
2625
|
+
settlementError: serializeOutboxError(settlement.error),
|
|
2626
|
+
}),
|
|
2627
|
+
});
|
|
2628
|
+
outcome.settlementFailed = 1;
|
|
2629
|
+
if (settlement.claimLost) outcome.leaseLost = 1;
|
|
2630
|
+
return outcome;
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
instrumentation.record({
|
|
2634
|
+
type: "outbox",
|
|
2635
|
+
...options.instrumentationContext,
|
|
2636
|
+
messageId: message.id,
|
|
2637
|
+
messageKind: message.kind,
|
|
2638
|
+
messageName: message.name,
|
|
2639
|
+
status: "delivered",
|
|
2640
|
+
details: outboxInstrumentationDetails(message),
|
|
2641
|
+
});
|
|
2642
|
+
outcome.delivered = 1;
|
|
2643
|
+
return outcome;
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
const deliveryError = deliveryOutcome.error;
|
|
2647
|
+
const failedAt = readOutboxNow(runtime.now);
|
|
2648
|
+
const shouldRetry = shouldRetryOutboxMessage(options, message, deliveryError);
|
|
2649
|
+
const deadLetter = !shouldRetry;
|
|
2650
|
+
const retryDelayMs = deadLetter
|
|
2651
|
+
? 0
|
|
2652
|
+
: resolveRetryDelayMs(options, message, deliveryError, failedAt);
|
|
2653
|
+
const retryAt = deadLetter
|
|
2654
|
+
? undefined
|
|
2655
|
+
: new Date(failedAt.getTime() + retryDelayMs);
|
|
2656
|
+
const settlement = await settleClaim({
|
|
2657
|
+
operation: "markFailed",
|
|
2658
|
+
message,
|
|
2659
|
+
lockedUntil: lease.lockedUntil,
|
|
2660
|
+
activeUntil,
|
|
2661
|
+
runtime,
|
|
2662
|
+
settle: (now) =>
|
|
2663
|
+
options.outbox.markFailed({
|
|
2664
|
+
id: message.id,
|
|
2665
|
+
claimToken: message.claimToken,
|
|
2666
|
+
error: deliveryError,
|
|
2667
|
+
deadLetter,
|
|
2668
|
+
now,
|
|
2669
|
+
retryAt,
|
|
2670
|
+
}),
|
|
2671
|
+
});
|
|
2672
|
+
|
|
2673
|
+
await reportRenewalOutcome();
|
|
2674
|
+
|
|
2675
|
+
try {
|
|
2676
|
+
await options.onError?.(deliveryError, message);
|
|
2677
|
+
} catch {
|
|
2678
|
+
// Delivery observers cannot change retry, dead-letter, or recovery state.
|
|
2679
|
+
}
|
|
2680
|
+
|
|
2681
|
+
if (!settlement.ok) {
|
|
2682
|
+
const failure: OutboxSettlementFailure = {
|
|
2683
|
+
error: settlement.error,
|
|
2684
|
+
message,
|
|
2685
|
+
operation: "markFailed",
|
|
2686
|
+
deliverySucceeded: false,
|
|
2687
|
+
deliveryError,
|
|
2688
|
+
};
|
|
2689
|
+
await notifySettlementFailure(options, failure);
|
|
2690
|
+
instrumentation.custom({
|
|
2691
|
+
name: "outbox.settlement.failed",
|
|
2692
|
+
label: "Outbox settlement failed",
|
|
2693
|
+
summary: `Could not settle failed ${message.kind} "${message.name}"`,
|
|
2694
|
+
details: outboxInstrumentationDetails(message, {
|
|
2695
|
+
operation: failure.operation,
|
|
2696
|
+
deliverySucceeded: false,
|
|
2697
|
+
deliveryError: serializeOutboxError(deliveryError),
|
|
2698
|
+
settlementError: serializeOutboxError(settlement.error),
|
|
2699
|
+
}),
|
|
2700
|
+
});
|
|
2701
|
+
outcome.settlementFailed = 1;
|
|
2702
|
+
if (settlement.claimLost) outcome.leaseLost = 1;
|
|
2703
|
+
return outcome;
|
|
2704
|
+
}
|
|
2705
|
+
|
|
2706
|
+
if (deadLetter) {
|
|
2707
|
+
await recordDeadLetter(
|
|
2708
|
+
options,
|
|
2709
|
+
instrumentation,
|
|
2710
|
+
jobInstrumentation,
|
|
2711
|
+
message,
|
|
2712
|
+
deliveryError,
|
|
2713
|
+
);
|
|
2714
|
+
outcome.deadLettered = 1;
|
|
2715
|
+
return outcome;
|
|
2716
|
+
}
|
|
2717
|
+
|
|
2718
|
+
instrumentation.record({
|
|
2719
|
+
type: "outbox",
|
|
2720
|
+
...options.instrumentationContext,
|
|
2721
|
+
messageId: message.id,
|
|
2722
|
+
messageKind: message.kind,
|
|
2723
|
+
messageName: message.name,
|
|
2724
|
+
status: "retryScheduled",
|
|
2725
|
+
details: outboxInstrumentationDetails(message, {
|
|
2726
|
+
retryDelayMs,
|
|
2727
|
+
retryAt: retryAt?.toISOString(),
|
|
2728
|
+
error: serializeOutboxError(deliveryError),
|
|
2729
|
+
}),
|
|
2730
|
+
});
|
|
2731
|
+
if (message.kind === "job") {
|
|
2732
|
+
jobInstrumentation.record({
|
|
2733
|
+
type: "job",
|
|
2734
|
+
...options.instrumentationContext,
|
|
2735
|
+
jobName: message.name,
|
|
2736
|
+
status: "retryScheduled",
|
|
2737
|
+
details: outboxInstrumentationDetails(message, {
|
|
2738
|
+
retryDelayMs,
|
|
2739
|
+
retryAt: retryAt?.toISOString(),
|
|
2740
|
+
error: serializeOutboxError(deliveryError),
|
|
2741
|
+
}),
|
|
2742
|
+
});
|
|
2743
|
+
}
|
|
2744
|
+
outcome.retried = 1;
|
|
2745
|
+
return outcome;
|
|
2746
|
+
}
|
|
2747
|
+
|
|
1477
2748
|
/**
|
|
1478
|
-
* Claim and deliver one
|
|
2749
|
+
* Claim and deliver one bounded set of outbox messages.
|
|
1479
2750
|
*
|
|
1480
|
-
*
|
|
1481
|
-
*
|
|
1482
|
-
*
|
|
1483
|
-
*
|
|
1484
|
-
* then dead-lettered.
|
|
2751
|
+
* The drain claims only enough messages to fill active delivery slots and
|
|
2752
|
+
* renews each active claim until delivery settles or reaches its configured
|
|
2753
|
+
* maximum duration. It remains an at-least-once transport: a process can still
|
|
2754
|
+
* terminate after the external effect succeeds but before acknowledgement.
|
|
1485
2755
|
*/
|
|
1486
2756
|
export async function drainOutbox(
|
|
1487
2757
|
options: DrainOutboxOptions,
|
|
1488
2758
|
): Promise<DrainOutboxResult> {
|
|
1489
|
-
const
|
|
1490
|
-
|
|
2759
|
+
const runtime = resolveDrainRuntime(options);
|
|
2760
|
+
assertOutboxDrainPort(options.outbox);
|
|
1491
2761
|
assertOutboxDeliveryCapabilities(options);
|
|
1492
2762
|
const instrumentation = createProviderInstrumentation(
|
|
1493
2763
|
options.instrumentation,
|
|
@@ -1504,165 +2774,110 @@ export async function drainOutbox(
|
|
|
1504
2774
|
},
|
|
1505
2775
|
);
|
|
1506
2776
|
const tracing = resolveTracingPort(options.instrumentation);
|
|
1507
|
-
|
|
1508
|
-
const now = options.now ?? new Date();
|
|
1509
|
-
const messages = await options.outbox.claimBatch({
|
|
1510
|
-
limit: batchSize,
|
|
1511
|
-
now,
|
|
1512
|
-
leaseMs: options.leaseMs,
|
|
1513
|
-
});
|
|
1514
2777
|
const result: DrainOutboxResult = {
|
|
1515
|
-
claimed:
|
|
2778
|
+
claimed: 0,
|
|
1516
2779
|
delivered: 0,
|
|
1517
2780
|
retried: 0,
|
|
1518
2781
|
deadLettered: 0,
|
|
2782
|
+
abandonedDeadLettered: 0,
|
|
2783
|
+
settlementFailed: 0,
|
|
2784
|
+
leaseLost: 0,
|
|
1519
2785
|
};
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
kind: "consumer",
|
|
1534
|
-
parent: parentTrace,
|
|
1535
|
-
attributes: traceAttributes,
|
|
1536
|
-
metricAttributes: traceAttributes,
|
|
1537
|
-
},
|
|
1538
|
-
(span) =>
|
|
1539
|
-
deliverOutboxMessage(
|
|
1540
|
-
options,
|
|
1541
|
-
message,
|
|
1542
|
-
captureTraceCarrier(span?.context ?? parentTrace),
|
|
1543
|
-
),
|
|
1544
|
-
);
|
|
1545
|
-
await options.outbox.markDelivered({
|
|
1546
|
-
id: message.id,
|
|
1547
|
-
claimToken: message.claimToken,
|
|
1548
|
-
now,
|
|
1549
|
-
});
|
|
1550
|
-
instrumentation.record({
|
|
1551
|
-
type: "outbox",
|
|
1552
|
-
...options.instrumentationContext,
|
|
1553
|
-
messageId: message.id,
|
|
1554
|
-
messageKind: message.kind,
|
|
1555
|
-
messageName: message.name,
|
|
1556
|
-
status: "delivered",
|
|
1557
|
-
details: outboxInstrumentationDetails(message),
|
|
2786
|
+
let remaining = runtime.batchSize;
|
|
2787
|
+
let sourceExhausted = false;
|
|
2788
|
+
|
|
2789
|
+
while (remaining > 0 && !sourceExhausted) {
|
|
2790
|
+
const active: ClaimedOutboxMessage[] = [];
|
|
2791
|
+
const abandoned: OutboxMessage[] = [];
|
|
2792
|
+
|
|
2793
|
+
while (active.length < runtime.concurrency && remaining > 0) {
|
|
2794
|
+
const limit = Math.min(runtime.concurrency - active.length, remaining);
|
|
2795
|
+
const selected = await options.outbox.claimBatch({
|
|
2796
|
+
limit,
|
|
2797
|
+
now: readOutboxNow(runtime.now),
|
|
2798
|
+
leaseMs: runtime.leaseMs,
|
|
1558
2799
|
});
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
// Preserve the delivery failure path so the message is retried or
|
|
1565
|
-
// dead-lettered even if the observer fails.
|
|
2800
|
+
const selectedCount =
|
|
2801
|
+
selected.claimed.length + selected.deadLettered.length;
|
|
2802
|
+
if (selectedCount === 0) {
|
|
2803
|
+
sourceExhausted = true;
|
|
2804
|
+
break;
|
|
1566
2805
|
}
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
: resolveRetryDelayMs(options, message, error, now);
|
|
1572
|
-
try {
|
|
1573
|
-
await options.outbox.markFailed({
|
|
1574
|
-
id: message.id,
|
|
1575
|
-
claimToken: message.claimToken,
|
|
1576
|
-
error,
|
|
1577
|
-
deadLetter,
|
|
1578
|
-
now,
|
|
1579
|
-
retryAt: deadLetter
|
|
1580
|
-
? undefined
|
|
1581
|
-
: new Date(now.getTime() + retryDelayMs),
|
|
1582
|
-
});
|
|
1583
|
-
} catch (settlementError) {
|
|
1584
|
-
try {
|
|
1585
|
-
await options.onSettlementError?.(settlementError, message, error);
|
|
1586
|
-
} catch {
|
|
1587
|
-
// Preserve the settlement failure when its observer also fails.
|
|
1588
|
-
}
|
|
1589
|
-
instrumentation.custom({
|
|
1590
|
-
name: "outbox.settlement.failed",
|
|
1591
|
-
label: "Outbox settlement failed",
|
|
1592
|
-
summary: `Could not settle failed ${message.kind} "${message.name}"`,
|
|
1593
|
-
details: outboxInstrumentationDetails(message, {
|
|
1594
|
-
deliveryError: serializeOutboxError(error),
|
|
1595
|
-
settlementError: serializeOutboxError(settlementError),
|
|
1596
|
-
}),
|
|
1597
|
-
});
|
|
1598
|
-
continue;
|
|
2806
|
+
if (selectedCount > limit) {
|
|
2807
|
+
throw new Error(
|
|
2808
|
+
`Outbox claimBatch returned ${selectedCount} messages for a limit of ${limit}.`,
|
|
2809
|
+
);
|
|
1599
2810
|
}
|
|
1600
2811
|
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
type: "job",
|
|
1648
|
-
...options.instrumentationContext,
|
|
1649
|
-
jobName: message.name,
|
|
1650
|
-
status: "retryScheduled",
|
|
1651
|
-
details: outboxInstrumentationDetails(message, {
|
|
1652
|
-
retryDelayMs,
|
|
1653
|
-
retryAt,
|
|
1654
|
-
error: serializeOutboxError(error),
|
|
1655
|
-
}),
|
|
1656
|
-
});
|
|
1657
|
-
}
|
|
1658
|
-
result.retried += 1;
|
|
1659
|
-
}
|
|
2812
|
+
remaining -= selectedCount;
|
|
2813
|
+
result.claimed += selected.claimed.length;
|
|
2814
|
+
active.push(...selected.claimed);
|
|
2815
|
+
abandoned.push(...selected.deadLettered);
|
|
2816
|
+
}
|
|
2817
|
+
|
|
2818
|
+
// Construct active delivery promises first so their heartbeats protect
|
|
2819
|
+
// freshly claimed rows while abandoned-message observers run.
|
|
2820
|
+
const outcomePromises = active.map((message) =>
|
|
2821
|
+
processClaimedMessage(
|
|
2822
|
+
options,
|
|
2823
|
+
runtime,
|
|
2824
|
+
instrumentation,
|
|
2825
|
+
jobInstrumentation,
|
|
2826
|
+
tracing,
|
|
2827
|
+
message,
|
|
2828
|
+
),
|
|
2829
|
+
);
|
|
2830
|
+
const abandonedPromises = abandoned.map(async (message) => {
|
|
2831
|
+
const error = new OutboxAbandonedClaimError({
|
|
2832
|
+
id: message.id,
|
|
2833
|
+
attempts: message.attempts,
|
|
2834
|
+
maxAttempts: message.maxAttempts,
|
|
2835
|
+
});
|
|
2836
|
+
await recordDeadLetter(
|
|
2837
|
+
options,
|
|
2838
|
+
instrumentation,
|
|
2839
|
+
jobInstrumentation,
|
|
2840
|
+
message,
|
|
2841
|
+
error,
|
|
2842
|
+
{ abandoned: true },
|
|
2843
|
+
);
|
|
2844
|
+
});
|
|
2845
|
+
const [outcomes] = await Promise.all([
|
|
2846
|
+
Promise.all(outcomePromises),
|
|
2847
|
+
Promise.all(abandonedPromises),
|
|
2848
|
+
]);
|
|
2849
|
+
result.deadLettered += abandoned.length;
|
|
2850
|
+
result.abandonedDeadLettered += abandoned.length;
|
|
2851
|
+
|
|
2852
|
+
for (const outcome of outcomes) {
|
|
2853
|
+
result.delivered += outcome.delivered;
|
|
2854
|
+
result.retried += outcome.retried;
|
|
2855
|
+
result.deadLettered += outcome.deadLettered;
|
|
2856
|
+
result.settlementFailed += outcome.settlementFailed;
|
|
2857
|
+
result.leaseLost += outcome.leaseLost;
|
|
1660
2858
|
}
|
|
1661
2859
|
}
|
|
1662
2860
|
|
|
1663
2861
|
return result;
|
|
1664
2862
|
}
|
|
1665
2863
|
|
|
2864
|
+
function assertOutboxDrainPort(outbox: OutboxPort): void {
|
|
2865
|
+
const candidate = outbox as unknown as Record<string, unknown>;
|
|
2866
|
+
const missing = [
|
|
2867
|
+
"claimBatch",
|
|
2868
|
+
"renewClaim",
|
|
2869
|
+
"markDelivered",
|
|
2870
|
+
"markFailed",
|
|
2871
|
+
].filter((method) => typeof candidate[method] !== "function");
|
|
2872
|
+
if (missing.length > 0) {
|
|
2873
|
+
throw new Error(
|
|
2874
|
+
`Cannot drain this outbox: the outbox port is missing ${missing
|
|
2875
|
+
.map((method) => `${method}()`)
|
|
2876
|
+
.join(", ")}.`,
|
|
2877
|
+
);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
|
|
1666
2881
|
function assertOutboxDeliveryCapabilities(options: DrainOutboxOptions): void {
|
|
1667
2882
|
const missing: string[] = [];
|
|
1668
2883
|
|