@message-queue-toolkit/core 23.2.0 → 24.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +360 -0
- package/dist/events/DomainEventEmitter.d.ts +2 -2
- package/dist/events/DomainEventEmitter.js +4 -5
- package/dist/events/DomainEventEmitter.js.map +1 -1
- package/dist/events/EventRegistry.js +1 -1
- package/dist/events/EventRegistry.js.map +1 -1
- package/dist/events/baseEventSchemas.d.ts +1 -1
- package/dist/events/baseEventSchemas.js +1 -1
- package/dist/events/baseEventSchemas.js.map +1 -1
- package/dist/events/eventTypes.d.ts +1 -1
- package/dist/index.d.ts +24 -23
- package/dist/index.js +20 -19
- package/dist/index.js.map +1 -1
- package/dist/message-deduplication/messageDeduplicationTypes.d.ts +1 -1
- package/dist/messages/baseMessageSchemas.d.ts +1 -1
- package/dist/messages/baseMessageSchemas.js +1 -1
- package/dist/messages/baseMessageSchemas.js.map +1 -1
- package/dist/payload-store/offloadedPayloadMessageSchemas.d.ts +24 -3
- package/dist/payload-store/offloadedPayloadMessageSchemas.js +39 -5
- package/dist/payload-store/offloadedPayloadMessageSchemas.js.map +1 -1
- package/dist/payload-store/payloadStoreTypes.d.ts +63 -3
- package/dist/payload-store/payloadStoreTypes.js +27 -0
- package/dist/payload-store/payloadStoreTypes.js.map +1 -1
- package/dist/queues/AbstractPublisherManager.d.ts +1 -1
- package/dist/queues/AbstractPublisherManager.js +4 -4
- package/dist/queues/AbstractPublisherManager.js.map +1 -1
- package/dist/queues/AbstractQueueService.d.ts +33 -3
- package/dist/queues/AbstractQueueService.js +137 -28
- package/dist/queues/AbstractQueueService.js.map +1 -1
- package/dist/queues/HandlerContainer.d.ts +49 -1
- package/dist/queues/HandlerContainer.js +65 -12
- package/dist/queues/HandlerContainer.js.map +1 -1
- package/dist/queues/HandlerSpy.d.ts +1 -0
- package/dist/queues/HandlerSpy.js +6 -5
- package/dist/queues/HandlerSpy.js.map +1 -1
- package/dist/queues/MessageSchemaContainer.js.map +1 -1
- package/dist/types/queueOptionsTypes.d.ts +1 -1
- package/dist/utils/matchUtils.js +5 -5
- package/dist/utils/matchUtils.js.map +1 -1
- package/dist/utils/waitUtils.js +1 -1
- package/dist/utils/waitUtils.js.map +1 -1
- package/package.json +15 -7
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
import { z } from 'zod/v4';
|
|
2
|
+
/**
|
|
3
|
+
* Multi-store payload reference schema.
|
|
4
|
+
* Contains information about where and how the payload was stored.
|
|
5
|
+
*/
|
|
6
|
+
export declare const PAYLOAD_REF_SCHEMA: z.ZodObject<{
|
|
7
|
+
id: z.ZodString;
|
|
8
|
+
store: z.ZodString;
|
|
9
|
+
size: z.ZodNumber;
|
|
10
|
+
}, z.core.$strip>;
|
|
11
|
+
export type PayloadRef = z.output<typeof PAYLOAD_REF_SCHEMA>;
|
|
2
12
|
/**
|
|
3
13
|
* When the payload is too large to be sent in a single message, it is offloaded to a storage service and a pointer to the offloaded payload is sent instead.
|
|
4
|
-
* This schema represents the payload that is sent in place of the original payload.
|
|
14
|
+
* This schema represents the reference to the payload that is sent in place of the original payload.
|
|
15
|
+
*
|
|
16
|
+
* The schema supports two formats:
|
|
17
|
+
* 1. New format, payloadRef object with id, store, and size
|
|
18
|
+
* 2. Old format, offloadedPayloadPointer and offloadedPayloadSize (for backward compatibility)
|
|
19
|
+
*
|
|
20
|
+
* At least one format must be present in the message.
|
|
5
21
|
*/
|
|
6
22
|
export declare const OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA: z.ZodObject<{
|
|
7
|
-
|
|
8
|
-
|
|
23
|
+
payloadRef: z.ZodOptional<z.ZodObject<{
|
|
24
|
+
id: z.ZodString;
|
|
25
|
+
store: z.ZodString;
|
|
26
|
+
size: z.ZodNumber;
|
|
27
|
+
}, z.core.$strip>>;
|
|
28
|
+
offloadedPayloadPointer: z.ZodOptional<z.ZodString>;
|
|
29
|
+
offloadedPayloadSize: z.ZodOptional<z.ZodNumber>;
|
|
9
30
|
}, z.core.$loose>;
|
|
10
31
|
export type OffloadedPayloadPointerPayload = z.output<typeof OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA>;
|
|
11
32
|
export declare function isOffloadedPayloadPointerPayload(value: unknown): value is OffloadedPayloadPointerPayload;
|
|
@@ -1,16 +1,50 @@
|
|
|
1
1
|
import { z } from 'zod/v4';
|
|
2
|
+
/**
|
|
3
|
+
* Multi-store payload reference schema.
|
|
4
|
+
* Contains information about where and how the payload was stored.
|
|
5
|
+
*/
|
|
6
|
+
export const PAYLOAD_REF_SCHEMA = z.object({
|
|
7
|
+
/** Unique identifier for the stored payload */
|
|
8
|
+
id: z.string().min(1),
|
|
9
|
+
/** Name/identifier of the store where the payload is stored */
|
|
10
|
+
store: z.string().min(1),
|
|
11
|
+
/** Size of the payload in bytes */
|
|
12
|
+
size: z.number().int().positive(),
|
|
13
|
+
});
|
|
2
14
|
/**
|
|
3
15
|
* When the payload is too large to be sent in a single message, it is offloaded to a storage service and a pointer to the offloaded payload is sent instead.
|
|
4
|
-
* This schema represents the payload that is sent in place of the original payload.
|
|
16
|
+
* This schema represents the reference to the payload that is sent in place of the original payload.
|
|
17
|
+
*
|
|
18
|
+
* The schema supports two formats:
|
|
19
|
+
* 1. New format, payloadRef object with id, store, and size
|
|
20
|
+
* 2. Old format, offloadedPayloadPointer and offloadedPayloadSize (for backward compatibility)
|
|
21
|
+
*
|
|
22
|
+
* At least one format must be present in the message.
|
|
5
23
|
*/
|
|
6
24
|
export const OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA = z
|
|
7
25
|
.object({
|
|
8
|
-
|
|
9
|
-
|
|
26
|
+
// New, extended payload reference (supports multi-store functionality).
|
|
27
|
+
payloadRef: PAYLOAD_REF_SCHEMA.optional(),
|
|
28
|
+
// Legacy payload reference, preserved for backward compatibility.
|
|
29
|
+
offloadedPayloadPointer: z.string().min(1).optional(),
|
|
30
|
+
offloadedPayloadSize: z.number().int().positive().optional(),
|
|
10
31
|
})
|
|
11
32
|
// Pass-through allows to pass message ID, type, timestamp and message-deduplication-related fields that are using dynamic keys.
|
|
12
|
-
.passthrough()
|
|
33
|
+
.passthrough()
|
|
34
|
+
.refine((data) => {
|
|
35
|
+
// At least one format must be present
|
|
36
|
+
// For the legacy format, both offloadedPayloadPointer and offloadedPayloadSize are required
|
|
37
|
+
return (data.payloadRef !== undefined ||
|
|
38
|
+
(data.offloadedPayloadPointer !== undefined && data.offloadedPayloadSize !== undefined));
|
|
39
|
+
}, {
|
|
40
|
+
message: 'Either payloadRef or both offloadedPayloadPointer and offloadedPayloadSize must be present',
|
|
41
|
+
});
|
|
13
42
|
export function isOffloadedPayloadPointerPayload(value) {
|
|
14
|
-
|
|
43
|
+
if (typeof value !== 'object' || value === null) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
const payload = value;
|
|
47
|
+
return (payload.payloadRef !== undefined ||
|
|
48
|
+
(payload.offloadedPayloadPointer !== undefined && payload.offloadedPayloadSize !== undefined));
|
|
15
49
|
}
|
|
16
50
|
//# sourceMappingURL=offloadedPayloadMessageSchemas.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offloadedPayloadMessageSchemas.js","sourceRoot":"","sources":["../../lib/payload-store/offloadedPayloadMessageSchemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG,CAAC;KACtD,MAAM,CAAC;IACN,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"offloadedPayloadMessageSchemas.js","sourceRoot":"","sources":["../../lib/payload-store/offloadedPayloadMessageSchemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,+CAA+C;IAC/C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,+DAA+D;IAC/D,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,mCAAmC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAA;AAIF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG,CAAC;KACtD,MAAM,CAAC;IACN,wEAAwE;IACxE,UAAU,EAAE,kBAAkB,CAAC,QAAQ,EAAE;IACzC,kEAAkE;IAClE,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7D,CAAC;IACF,gIAAgI;KAC/H,WAAW,EAAE;KACb,MAAM,CACL,CAAC,IAAI,EAAE,EAAE;IACP,sCAAsC;IACtC,4FAA4F;IAC5F,OAAO,CACL,IAAI,CAAC,UAAU,KAAK,SAAS;QAC7B,CAAC,IAAI,CAAC,uBAAuB,KAAK,SAAS,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS,CAAC,CACxF,CAAA;AACH,CAAC,EACD;IACE,OAAO,EACL,4FAA4F;CAC/F,CACF,CAAA;AAMH,MAAM,UAAU,gCAAgC,CAC9C,KAAc;IAEd,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,OAAO,GAAG,KAAuC,CAAA;IACvD,OAAO,CACL,OAAO,CAAC,UAAU,KAAK,SAAS;QAChC,CAAC,OAAO,CAAC,uBAAuB,KAAK,SAAS,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAC9F,CAAA;AACH,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Readable } from 'node:stream';
|
|
2
|
-
export interface
|
|
2
|
+
export interface PayloadStore {
|
|
3
3
|
/** Store the payload and return a key that can be used to retrieve it later. */
|
|
4
4
|
storePayload(payload: SerializedPayload): Promise<string>;
|
|
5
5
|
/** Retrieve the previously stored payload. */
|
|
@@ -16,11 +16,71 @@ export declare function isDestroyable(value: unknown): value is Destroyable<unkn
|
|
|
16
16
|
export interface PayloadSerializer {
|
|
17
17
|
serialize(payload: unknown): Promise<SerializedPayload | Destroyable<SerializedPayload>>;
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Single-store configuration (simple mode).
|
|
21
|
+
* Use this when you have only one payload store.
|
|
22
|
+
*/
|
|
23
|
+
export type SinglePayloadStoreConfig = {
|
|
20
24
|
/** Threshold in bytes after which the payload should be stored in the store. */
|
|
21
25
|
messageSizeThreshold: number;
|
|
22
26
|
/** The store to use for storing the payload. */
|
|
23
|
-
store:
|
|
27
|
+
store: PayloadStore;
|
|
28
|
+
/**
|
|
29
|
+
* Identifier for this store (used in offloaded payload messages).
|
|
30
|
+
* This name is embedded in the message payload to identify which store holds the data.
|
|
31
|
+
*/
|
|
32
|
+
storeName: string;
|
|
24
33
|
/** The serializer to use for serializing the payload. */
|
|
25
34
|
serializer?: PayloadSerializer;
|
|
26
35
|
};
|
|
36
|
+
/**
|
|
37
|
+
* Multi-store configuration (advanced mode).
|
|
38
|
+
* Use this when you need to support multiple payload stores (e.g., for migration).
|
|
39
|
+
*/
|
|
40
|
+
export type MultiPayloadStoreConfig<StoreNames extends string = string> = {
|
|
41
|
+
/** Threshold in bytes after which the payload should be stored in the store. */
|
|
42
|
+
messageSizeThreshold: number;
|
|
43
|
+
/** Map of store identifiers to store instances. */
|
|
44
|
+
stores: Record<StoreNames, PayloadStore>;
|
|
45
|
+
/**
|
|
46
|
+
* Store identifier to use for outgoing messages.
|
|
47
|
+
* Must be a key from the stores map (enforced at compile time).
|
|
48
|
+
*/
|
|
49
|
+
outgoingStore: NoInfer<StoreNames>;
|
|
50
|
+
/**
|
|
51
|
+
* Optional: Default store identifier to use when retrieving messages that only have
|
|
52
|
+
* the legacy offloadedPayloadPointer field (without payloadRef).
|
|
53
|
+
* If not specified, will throw an error when encountering legacy format.
|
|
54
|
+
* Must be a key from the stores map if provided (enforced at compile time).
|
|
55
|
+
*/
|
|
56
|
+
defaultIncomingStore?: NoInfer<StoreNames>;
|
|
57
|
+
/** The serializer to use for serializing the payload. */
|
|
58
|
+
serializer?: PayloadSerializer;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Payload store configuration - supports both single-store and multi-store modes.
|
|
62
|
+
*/
|
|
63
|
+
export type PayloadStoreConfig = SinglePayloadStoreConfig | MultiPayloadStoreConfig;
|
|
64
|
+
/**
|
|
65
|
+
* Type guard to check if config is multi-store configuration.
|
|
66
|
+
*/
|
|
67
|
+
export declare function isMultiPayloadStoreConfig(config: PayloadStoreConfig): config is MultiPayloadStoreConfig<string>;
|
|
68
|
+
/**
|
|
69
|
+
* Helper function to create a multi-store config with compile-time validation.
|
|
70
|
+
* TypeScript will infer store names from the `stores` object and ensure
|
|
71
|
+
* `outgoingStore` and `defaultIncomingStore` are valid keys.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```typescript
|
|
75
|
+
* const config = createMultiStoreConfig({
|
|
76
|
+
* messageSizeThreshold: 256 * 1024,
|
|
77
|
+
* stores: {
|
|
78
|
+
* 'store-a': storeA,
|
|
79
|
+
* 'store-b': storeB,
|
|
80
|
+
* },
|
|
81
|
+
* outgoingStore: 'store-a', // Valid
|
|
82
|
+
* // outgoingStore: 'store-c', // Compile error
|
|
83
|
+
* })
|
|
84
|
+
* ```
|
|
85
|
+
*/
|
|
86
|
+
export declare function createMultiStoreConfig<StoreNames extends string>(config: MultiPayloadStoreConfig<StoreNames>): MultiPayloadStoreConfig<StoreNames>;
|
|
@@ -1,4 +1,31 @@
|
|
|
1
1
|
export function isDestroyable(value) {
|
|
2
2
|
return typeof value === 'object' && value !== null && 'destroy' in value;
|
|
3
3
|
}
|
|
4
|
+
/**
|
|
5
|
+
* Type guard to check if config is multi-store configuration.
|
|
6
|
+
*/
|
|
7
|
+
export function isMultiPayloadStoreConfig(config) {
|
|
8
|
+
return 'stores' in config && 'outgoingStore' in config;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Helper function to create a multi-store config with compile-time validation.
|
|
12
|
+
* TypeScript will infer store names from the `stores` object and ensure
|
|
13
|
+
* `outgoingStore` and `defaultIncomingStore` are valid keys.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```typescript
|
|
17
|
+
* const config = createMultiStoreConfig({
|
|
18
|
+
* messageSizeThreshold: 256 * 1024,
|
|
19
|
+
* stores: {
|
|
20
|
+
* 'store-a': storeA,
|
|
21
|
+
* 'store-b': storeB,
|
|
22
|
+
* },
|
|
23
|
+
* outgoingStore: 'store-a', // Valid
|
|
24
|
+
* // outgoingStore: 'store-c', // Compile error
|
|
25
|
+
* })
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export function createMultiStoreConfig(config) {
|
|
29
|
+
return config;
|
|
30
|
+
}
|
|
4
31
|
//# sourceMappingURL=payloadStoreTypes.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"payloadStoreTypes.js","sourceRoot":"","sources":["../../lib/payload-store/payloadStoreTypes.ts"],"names":[],"mappings":"AAmBA,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,IAAI,KAAK,CAAA;AAC1E,CAAC"}
|
|
1
|
+
{"version":3,"file":"payloadStoreTypes.js","sourceRoot":"","sources":["../../lib/payload-store/payloadStoreTypes.ts"],"names":[],"mappings":"AAmBA,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,IAAI,KAAK,CAAA;AAC1E,CAAC;AA6DD;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAA0B;IAE1B,OAAO,QAAQ,IAAI,MAAM,IAAI,eAAe,IAAI,MAAM,CAAA;AACxD,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAA2C;IAE3C,OAAO,MAAM,CAAA;AACf,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { z } from 'zod/v4';
|
|
2
|
-
import type { EventRegistry } from '../events/EventRegistry.ts';
|
|
3
2
|
import type { PublisherBaseEventType } from '../events/baseEventSchemas.ts';
|
|
3
|
+
import type { EventRegistry } from '../events/EventRegistry.ts';
|
|
4
4
|
import type { CommonEventDefinition } from '../events/eventTypes.ts';
|
|
5
5
|
import type { MetadataFiller } from '../messages/MetadataFiller.ts';
|
|
6
6
|
import type { AsyncPublisher, SyncPublisher } from '../types/MessageQueueTypes.ts';
|
|
@@ -103,18 +103,18 @@ export class AbstractPublisherManager {
|
|
|
103
103
|
}
|
|
104
104
|
resolveMessage(messageDefinition, message, precedingEventMetadata) {
|
|
105
105
|
const producedMetadata = this.metadataFiller.produceMetadata(message,
|
|
106
|
-
// @ts-
|
|
106
|
+
// @ts-expect-error
|
|
107
107
|
messageDefinition, precedingEventMetadata);
|
|
108
|
-
// @ts-
|
|
108
|
+
// @ts-expect-error
|
|
109
109
|
const resolvedMetadata = message[this.metadataField]
|
|
110
110
|
? {
|
|
111
111
|
...producedMetadata,
|
|
112
|
-
// @ts-
|
|
112
|
+
// @ts-expect-error
|
|
113
113
|
...message[this.metadataField],
|
|
114
114
|
}
|
|
115
115
|
: // @ts-ignore
|
|
116
116
|
producedMetadata;
|
|
117
|
-
// @ts-
|
|
117
|
+
// @ts-expect-error
|
|
118
118
|
return {
|
|
119
119
|
id: this.metadataFiller.produceId(),
|
|
120
120
|
timestamp: this.metadataFiller.produceTimestamp(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractPublisherManager.js","sourceRoot":"","sources":["../../lib/queues/AbstractPublisherManager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"AbstractPublisherManager.js","sourceRoot":"","sources":["../../lib/queues/AbstractPublisherManager.ts"],"names":[],"mappings":"AA4BA,MAAM,OAAgB,wBAAwB;IAgB3B,gBAAgB,CAOhC;IAEkB,mBAAmB,CAAa;IAEhC,cAAc,CAGhC;IACkB,aAAa,CAAQ;IAExC,qFAAqF;IAClE,gBAAgB,GAAgD,EAGlF,CAAA;IACkB,OAAO,CAAS;IACzB,oBAAoB,GAAwC,EAGrE,CAAA;IACgB,qBAAqB,CAAkB;IAExD,YAAsB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,aAAa,EACb,aAAa,EACb,OAAO,GAmBR;QACC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAA;QAElD,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,eAAe,CAAC,CAAA;QAClD,IAAI,CAAC,kBAAkB,EAAE,CAAA;IAC3B,CAAC;IAIS,+BAA+B,CAAC,YAAoB;QAC5D,OAAO,EAAE,CAAA;IACX,CAAC;IAEO,cAAc,CAAC,MAAiC;QACtD,KAAK,MAAM,cAAc,IAAI,MAAM,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAA;YAE3D,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;YACzC,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QACzD,CAAC;IACH,CAAC;IAEO,kBAAkB;QACxB,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAChD,IAAI,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3C,SAAQ;YACV,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtE,OAAO,KAAK,CAAC,cAAc,CAAA;YAC7B,CAAC,CAAC,CAAA;YACF,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAA;YAC9D,MAAM,eAAe,GAAG,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAA;YAEzE,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAC3E,IAAI,CAAC,qBAAqB,EAC1B;gBACE,GAAG,IAAI,CAAC,mBAAmB;gBAC3B,cAAc;gBACd,cAAc;gBACd,GAAG,eAAe;aACnB,CACF,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,gBAAiC;QAC9D,IAAI,gBAAgB,EAAE,CAAC;YACrB,KAAK,MAAM,WAAW,IAAI,gBAAgB,EAAE,CAAC;gBAC3C,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;gBAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAA;gBACzD,CAAC;gBACD,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAA;YAChC,CAAC;YACD,OAAM;QACR,CAAC;QAED,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACpD,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAA;QACrD,CAAC;IACH,CAAC;IAEM,eAAe,CAAC,WAAyB,EAAE,SAAwB;QACxE,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,SAAS,CAAA;IACpD,CAAC;IAEM,qBAAqB,CAAC,eAAoC;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAA;QAC5D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;QACpE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QACzC,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IAC1D,CAAC;IAEM,KAAK,CAAC,OAAO,CAClB,WAAyB,EACzB,OAA8D,EAC9D,sBAA8C,EAC9C,cAAmC;QAEnC,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QACxD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BAA2B,WAAW,8EAA8E,CACrH,CAAA;QACH,CAAC;QACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,wBAAwB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;QAC7E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,iCAAiC,WAAW,eAAe,OAAO,CAAC,IAAI,8BAA8B,CACtG,CAAA;QACH,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,OAAO,EAAE,sBAAsB,CAAC,CAAA;QAE/F,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAO,SAA6C,CAAC,OAAO,CAAC,eAAe,EAAE,cAAc,CAAC,CAAA;QAC/F,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,cAAc,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAES,wBAAwB,CAChC,WAAyB,EACzB,OAA8D;QAE9D,2BAA2B;QAC3B,OAAO,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,IAAI,CAC5C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,IAAI,CAClE,CAAA;IACH,CAAC;IAES,cAAc,CACtB,iBAAkD,EAClD,OAA8D,EAC9D,sBAA8C;QAE9C,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,CAC1D,OAAO;QACP,mBAAmB;QACnB,iBAAiB,EACjB,sBAAsB,CACvB,CAAA;QAED,mBAAmB;QACnB,MAAM,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC;YAClD,CAAC,CAAC;gBACE,GAAG,gBAAgB;gBACnB,mBAAmB;gBACnB,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC;aAC/B;YACH,CAAC,CAAC,aAAa;gBACb,gBAAgB,CAAA;QAEpB,mBAAmB;QACnB,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE;YACnC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE;YACjD,GAAG,OAAO;YACV,QAAQ,EAAE,gBAAgB;SAC3B,CAAA;IACH,CAAC;IAEM,iBAAiB;QACtB,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE;YACnC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE;SAClD,CAAA;IACH,CAAC;IAED;;OAEG;IACI,UAAU,CAAC,WAAyB;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAA;QAExD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,2BAA2B,WAAW,8EAA8E,CACrH,CAAA;QACH,CAAC;QAED,OAAO,SAAS,CAAC,UAAU,CAAA;IAC7B,CAAC;CACF"}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { type CommonLogger, type Either, type ErrorReporter, type ErrorResolver } from '@lokalise/node-core';
|
|
2
|
+
import type { MakeRequired } from '@lokalise/universal-ts-utils/node';
|
|
2
3
|
import type { ZodSchema, ZodType } from 'zod/v4';
|
|
3
4
|
import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors.ts';
|
|
4
|
-
import { type AcquireLockTimeoutError } from '../message-deduplication/AcquireLockTimeoutError.
|
|
5
|
+
import { type AcquireLockTimeoutError } from '../message-deduplication/AcquireLockTimeoutError.ts';
|
|
5
6
|
import { type DeduplicationRequester, type MessageDeduplicationConfig, type ReleasableLock } from '../message-deduplication/messageDeduplicationTypes.ts';
|
|
6
7
|
import { type OffloadedPayloadPointerPayload } from '../payload-store/offloadedPayloadMessageSchemas.ts';
|
|
7
|
-
import type {
|
|
8
|
+
import type { MultiPayloadStoreConfig, SinglePayloadStoreConfig } from '../payload-store/payloadStoreTypes.ts';
|
|
8
9
|
import type { MessageProcessingResult } from '../types/MessageQueueTypes.ts';
|
|
9
10
|
import type { DeletionConfig, MessageMetricsManager, QueueDependencies, QueueOptions } from '../types/queueOptionsTypes.ts';
|
|
10
11
|
import type { BarrierCallback, BarrierResult, MessageHandlerConfig, PreHandlingOutputs, Prehandler, PrehandlerResult } from './HandlerContainer.ts';
|
|
@@ -45,7 +46,7 @@ export declare abstract class AbstractQueueService<MessagePayloadSchemas extends
|
|
|
45
46
|
protected readonly creationConfig?: QueueConfiguration;
|
|
46
47
|
protected readonly locatorConfig?: QueueLocatorType;
|
|
47
48
|
protected readonly deletionConfig?: DeletionConfig;
|
|
48
|
-
protected readonly payloadStoreConfig?:
|
|
49
|
+
protected readonly payloadStoreConfig?: MakeRequired<SinglePayloadStoreConfig, 'serializer'> | MakeRequired<MultiPayloadStoreConfig, 'serializer'>;
|
|
49
50
|
protected readonly messageDeduplicationConfig?: MessageDeduplicationConfig;
|
|
50
51
|
protected readonly messageMetricsManager?: MessageMetricsManager<MessagePayloadSchemas>;
|
|
51
52
|
protected readonly _handlerSpy?: HandlerSpy<MessagePayloadSchemas>;
|
|
@@ -92,15 +93,44 @@ export declare abstract class AbstractQueueService<MessagePayloadSchemas extends
|
|
|
92
93
|
protected abstract preHandlerBarrier<BarrierOutput>(message: MessagePayloadSchemas, messageType: string, preHandlerOutput: PrehandlerOutput): Promise<BarrierResult<BarrierOutput>>;
|
|
93
94
|
protected abstract processMessage(message: MessagePayloadSchemas, messageType: string, preHandlingOutputs: PreHandlingOutputs<PrehandlerOutput, any>): Promise<Either<'retryLater', 'success'>>;
|
|
94
95
|
abstract close(): Promise<unknown>;
|
|
96
|
+
/**
|
|
97
|
+
* Resolves the store and store name for outgoing (publishing) messages.
|
|
98
|
+
* For multi-store: uses outgoingStore from config.
|
|
99
|
+
* For single-store: uses the configured store and storeName.
|
|
100
|
+
* @throws Error if payloadStoreConfig is not configured or the named store is not found.
|
|
101
|
+
*/
|
|
102
|
+
private resolveOutgoingStore;
|
|
103
|
+
/**
|
|
104
|
+
* Resolves store from payloadRef (new format).
|
|
105
|
+
*/
|
|
106
|
+
private resolveStoreFromPayloadRef;
|
|
107
|
+
/**
|
|
108
|
+
* Resolves store from legacy pointer (old format).
|
|
109
|
+
*/
|
|
110
|
+
private resolveStoreFromLegacyPointer;
|
|
111
|
+
/**
|
|
112
|
+
* Resolves the store for incoming (consuming) messages based on payload reference.
|
|
113
|
+
* For multi-store with payloadRef: uses the store specified in payloadRef.
|
|
114
|
+
* For multi-store with legacy format: uses defaultIncomingStore.
|
|
115
|
+
* For single-store: always uses the configured store.
|
|
116
|
+
*/
|
|
117
|
+
private resolveIncomingStore;
|
|
95
118
|
/**
|
|
96
119
|
* Offload message payload to an external store if it exceeds the threshold.
|
|
97
120
|
* Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded.
|
|
98
121
|
* Requires message size as only the implementation knows how to calculate it.
|
|
122
|
+
*
|
|
123
|
+
* For multi-store configuration, uses the configured outgoingStore.
|
|
124
|
+
* For single-store configuration, uses the single store.
|
|
125
|
+
*
|
|
126
|
+
* The returned payload includes both the new payloadRef format and legacy fields for backward compatibility.
|
|
99
127
|
*/
|
|
100
128
|
protected offloadMessagePayloadIfNeeded(message: MessagePayloadSchemas, messageSizeFn: () => number): Promise<MessagePayloadSchemas | OffloadedPayloadPointerPayload>;
|
|
101
129
|
/**
|
|
102
130
|
* Retrieve previously offloaded message payload using provided pointer payload.
|
|
103
131
|
* Returns the original payload or an error if the payload was not found or could not be parsed.
|
|
132
|
+
*
|
|
133
|
+
* Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer).
|
|
104
134
|
*/
|
|
105
135
|
protected retrieveOffloadedMessagePayload(maybeOffloadedPayloadPointerPayload: unknown): Promise<Either<Error, unknown>>;
|
|
106
136
|
/**
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import { types } from 'node:util';
|
|
2
|
-
import { stringValueSerializer, } from '@lokalise/node-core';
|
|
3
|
-
import { resolveGlobalErrorLogObject } from '@lokalise/node-core';
|
|
2
|
+
import { resolveGlobalErrorLogObject, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
3
|
import { MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA, } from '@message-queue-toolkit/schemas';
|
|
5
|
-
import { isAcquireLockTimeoutError, } from
|
|
4
|
+
import { isAcquireLockTimeoutError, } from "../message-deduplication/AcquireLockTimeoutError.js";
|
|
6
5
|
import { DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS, DeduplicationRequesterEnum, noopReleasableLock, } from "../message-deduplication/messageDeduplicationTypes.js";
|
|
7
6
|
import { jsonStreamStringifySerializer } from "../payload-store/JsonStreamStringifySerializer.js";
|
|
8
7
|
import { OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA, } from "../payload-store/offloadedPayloadMessageSchemas.js";
|
|
9
|
-
import { isDestroyable } from "../payload-store/payloadStoreTypes.js";
|
|
8
|
+
import { isDestroyable, isMultiPayloadStoreConfig } from "../payload-store/payloadStoreTypes.js";
|
|
10
9
|
import { isRetryDateExceeded } from "../utils/dateUtils.js";
|
|
11
10
|
import { streamWithKnownSizeToString } from "../utils/streamUtils.js";
|
|
12
11
|
import { toDatePreprocessor } from "../utils/toDateProcessor.js";
|
|
@@ -134,7 +133,7 @@ export class AbstractQueueService {
|
|
|
134
133
|
}
|
|
135
134
|
}
|
|
136
135
|
resolveProcessedMessageMetadata(message, processingResult, messageProcessingStartTimestamp, messageProcessingEndTimestamp, queueName, messageId) {
|
|
137
|
-
// @ts-
|
|
136
|
+
// @ts-expect-error
|
|
138
137
|
const resolvedMessageId = message?.[this.messageIdField] ?? messageId;
|
|
139
138
|
const messageTimestamp = message ? this.tryToExtractTimestamp(message)?.getTime() : undefined;
|
|
140
139
|
const messageType = message && this.messageTypeField in message
|
|
@@ -174,13 +173,12 @@ export class AbstractQueueService {
|
|
|
174
173
|
}
|
|
175
174
|
async preHandlerBarrierInternal(barrier, message, executionContext, preHandlerOutput) {
|
|
176
175
|
if (!barrier) {
|
|
177
|
-
// @ts-
|
|
176
|
+
// @ts-expect-error
|
|
178
177
|
return {
|
|
179
178
|
isPassing: true,
|
|
180
179
|
output: undefined,
|
|
181
180
|
};
|
|
182
181
|
}
|
|
183
|
-
// @ts-ignore
|
|
184
182
|
return await barrier(message, executionContext, preHandlerOutput);
|
|
185
183
|
}
|
|
186
184
|
shouldBeRetried(message, maxRetryDuration) {
|
|
@@ -201,7 +199,7 @@ export class AbstractQueueService {
|
|
|
201
199
|
* will be used to prevent infinite retries on the same message
|
|
202
200
|
*/
|
|
203
201
|
if (!this.tryToExtractTimestamp(message)) {
|
|
204
|
-
// @ts-
|
|
202
|
+
// @ts-expect-error
|
|
205
203
|
messageCopy[this.messageTimestampField] = new Date().toISOString();
|
|
206
204
|
this.logger.warn(`${this.messageTimestampField} not defined, adding it automatically`);
|
|
207
205
|
}
|
|
@@ -209,15 +207,14 @@ export class AbstractQueueService {
|
|
|
209
207
|
* add/increment the number of retries performed to exponential message delay
|
|
210
208
|
*/
|
|
211
209
|
const numberOfRetries = this.tryToExtractNumberOfRetries(message);
|
|
212
|
-
// @ts-
|
|
210
|
+
// @ts-expect-error
|
|
213
211
|
messageCopy[this.messageRetryLaterCountField] =
|
|
214
212
|
numberOfRetries !== undefined ? numberOfRetries + 1 : 0;
|
|
215
213
|
return messageCopy;
|
|
216
214
|
}
|
|
217
215
|
tryToExtractTimestamp(message) {
|
|
218
|
-
// @ts-ignore
|
|
219
216
|
if (this.messageTimestampField in message) {
|
|
220
|
-
// @ts-
|
|
217
|
+
// @ts-expect-error
|
|
221
218
|
const res = toDatePreprocessor(message[this.messageTimestampField]);
|
|
222
219
|
if (!(res instanceof Date)) {
|
|
223
220
|
throw new Error(`${this.messageTimestampField} invalid type`);
|
|
@@ -229,7 +226,7 @@ export class AbstractQueueService {
|
|
|
229
226
|
tryToExtractNumberOfRetries(message) {
|
|
230
227
|
if (this.messageRetryLaterCountField in message &&
|
|
231
228
|
typeof message[this.messageRetryLaterCountField] === 'number') {
|
|
232
|
-
// @ts-
|
|
229
|
+
// @ts-expect-error
|
|
233
230
|
return message[this.messageRetryLaterCountField];
|
|
234
231
|
}
|
|
235
232
|
return undefined;
|
|
@@ -243,51 +240,149 @@ export class AbstractQueueService {
|
|
|
243
240
|
resolve(preHandlerOutput);
|
|
244
241
|
}
|
|
245
242
|
else {
|
|
246
|
-
// biome-ignore lint/style/noNonNullAssertion:
|
|
247
|
-
preHandlers[index](message, executionContext,
|
|
248
|
-
// @ts-ignore
|
|
249
|
-
preHandlerOutput, this.resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index + 1, preHandlerOutput, resolve, reject));
|
|
243
|
+
// biome-ignore lint/style/noNonNullAssertion: It's ok
|
|
244
|
+
preHandlers[index](message, executionContext, preHandlerOutput, this.resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index + 1, preHandlerOutput, resolve, reject));
|
|
250
245
|
}
|
|
251
246
|
};
|
|
252
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* Resolves the store and store name for outgoing (publishing) messages.
|
|
250
|
+
* For multi-store: uses outgoingStore from config.
|
|
251
|
+
* For single-store: uses the configured store and storeName.
|
|
252
|
+
* @throws Error if payloadStoreConfig is not configured or the named store is not found.
|
|
253
|
+
*/
|
|
254
|
+
resolveOutgoingStore() {
|
|
255
|
+
if (!this.payloadStoreConfig) {
|
|
256
|
+
throw new Error('Payload store is not configured');
|
|
257
|
+
}
|
|
258
|
+
if (isMultiPayloadStoreConfig(this.payloadStoreConfig)) {
|
|
259
|
+
const storeName = this.payloadStoreConfig.outgoingStore;
|
|
260
|
+
const store = this.payloadStoreConfig.stores[storeName];
|
|
261
|
+
if (!store) {
|
|
262
|
+
throw new Error(`Outgoing store "${storeName}" not found in stores configuration. Available stores: ${Object.keys(this.payloadStoreConfig.stores).join(', ')}`);
|
|
263
|
+
}
|
|
264
|
+
return { store, storeName };
|
|
265
|
+
}
|
|
266
|
+
// Single-store configuration
|
|
267
|
+
return { store: this.payloadStoreConfig.store, storeName: this.payloadStoreConfig.storeName };
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Resolves store from payloadRef (new format).
|
|
271
|
+
*/
|
|
272
|
+
resolveStoreFromPayloadRef(config, payloadRef) {
|
|
273
|
+
if (isMultiPayloadStoreConfig(config)) {
|
|
274
|
+
const store = config.stores[payloadRef.store];
|
|
275
|
+
if (!store) {
|
|
276
|
+
return {
|
|
277
|
+
error: new Error(`Store "${payloadRef.store}" specified in payloadRef not found in stores configuration. Available stores: ${Object.keys(config.stores).join(', ')}`),
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
return { result: { store, payloadId: payloadRef.id } };
|
|
281
|
+
}
|
|
282
|
+
// Single-store config - validate that payloadRef.store matches configured store name
|
|
283
|
+
if (payloadRef.store !== config.storeName) {
|
|
284
|
+
return {
|
|
285
|
+
error: new Error(`Store "${payloadRef.store}" specified in payloadRef does not match configured store name "${config.storeName}". ` +
|
|
286
|
+
'This may indicate a misconfiguration or that the message was published by a different system. ' +
|
|
287
|
+
'If you need to consume messages from multiple stores, consider using MultiPayloadStoreConfig.'),
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
return { result: { store: config.store, payloadId: payloadRef.id } };
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Resolves store from legacy pointer (old format).
|
|
294
|
+
*/
|
|
295
|
+
resolveStoreFromLegacyPointer(config, legacyPointer) {
|
|
296
|
+
if (isMultiPayloadStoreConfig(config)) {
|
|
297
|
+
if (!config.defaultIncomingStore) {
|
|
298
|
+
return {
|
|
299
|
+
error: new Error('Message contains legacy offloadedPayloadPointer format, but no defaultIncomingStore is configured in multi-store setup. Please configure defaultIncomingStore or migrate messages to use payloadRef format.'),
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
const store = config.stores[config.defaultIncomingStore];
|
|
303
|
+
if (!store) {
|
|
304
|
+
return {
|
|
305
|
+
error: new Error(`Default incoming store "${config.defaultIncomingStore}" not found in stores configuration. Available stores: ${Object.keys(config.stores).join(', ')}`),
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
return { result: { store, payloadId: legacyPointer } };
|
|
309
|
+
}
|
|
310
|
+
return { result: { store: config.store, payloadId: legacyPointer } };
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Resolves the store for incoming (consuming) messages based on payload reference.
|
|
314
|
+
* For multi-store with payloadRef: uses the store specified in payloadRef.
|
|
315
|
+
* For multi-store with legacy format: uses defaultIncomingStore.
|
|
316
|
+
* For single-store: always uses the configured store.
|
|
317
|
+
*/
|
|
318
|
+
resolveIncomingStore(payloadRef, legacyPointer) {
|
|
319
|
+
if (!this.payloadStoreConfig) {
|
|
320
|
+
return { error: new Error('Payload store is not configured') };
|
|
321
|
+
}
|
|
322
|
+
if (payloadRef) {
|
|
323
|
+
return this.resolveStoreFromPayloadRef(this.payloadStoreConfig, payloadRef);
|
|
324
|
+
}
|
|
325
|
+
if (legacyPointer) {
|
|
326
|
+
return this.resolveStoreFromLegacyPointer(this.payloadStoreConfig, legacyPointer);
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
error: new Error('Invalid offloaded payload: neither payloadRef nor offloadedPayloadPointer is present'),
|
|
330
|
+
};
|
|
331
|
+
}
|
|
253
332
|
/**
|
|
254
333
|
* Offload message payload to an external store if it exceeds the threshold.
|
|
255
334
|
* Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded.
|
|
256
335
|
* Requires message size as only the implementation knows how to calculate it.
|
|
336
|
+
*
|
|
337
|
+
* For multi-store configuration, uses the configured outgoingStore.
|
|
338
|
+
* For single-store configuration, uses the single store.
|
|
339
|
+
*
|
|
340
|
+
* The returned payload includes both the new payloadRef format and legacy fields for backward compatibility.
|
|
257
341
|
*/
|
|
258
342
|
async offloadMessagePayloadIfNeeded(message, messageSizeFn) {
|
|
259
343
|
if (!this.payloadStoreConfig ||
|
|
260
344
|
messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) {
|
|
261
345
|
return message;
|
|
262
346
|
}
|
|
263
|
-
|
|
347
|
+
const { store, storeName } = this.resolveOutgoingStore();
|
|
264
348
|
const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message);
|
|
349
|
+
let payloadId;
|
|
265
350
|
try {
|
|
266
|
-
|
|
351
|
+
payloadId = await store.storePayload(serializedPayload);
|
|
267
352
|
}
|
|
268
353
|
finally {
|
|
269
354
|
if (isDestroyable(serializedPayload)) {
|
|
270
355
|
await serializedPayload.destroy();
|
|
271
356
|
}
|
|
272
357
|
}
|
|
358
|
+
// Return message with both new and legacy formats for backward compatibility
|
|
273
359
|
return {
|
|
274
|
-
|
|
360
|
+
// Extended payload reference format
|
|
361
|
+
payloadRef: {
|
|
362
|
+
id: payloadId,
|
|
363
|
+
store: storeName,
|
|
364
|
+
size: serializedPayload.size,
|
|
365
|
+
},
|
|
366
|
+
// Legacy format for backward compatibility
|
|
367
|
+
offloadedPayloadPointer: payloadId,
|
|
275
368
|
offloadedPayloadSize: serializedPayload.size,
|
|
276
|
-
// @ts-
|
|
369
|
+
// @ts-expect-error
|
|
277
370
|
[this.messageIdField]: message[this.messageIdField],
|
|
278
|
-
// @ts-
|
|
371
|
+
// @ts-expect-error
|
|
279
372
|
[this.messageTypeField]: message[this.messageTypeField],
|
|
280
|
-
// @ts-
|
|
373
|
+
// @ts-expect-error
|
|
281
374
|
[this.messageTimestampField]: message[this.messageTimestampField],
|
|
282
|
-
// @ts-
|
|
375
|
+
// @ts-expect-error
|
|
283
376
|
[this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField],
|
|
284
|
-
// @ts-
|
|
377
|
+
// @ts-expect-error
|
|
285
378
|
[this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField],
|
|
286
379
|
};
|
|
287
380
|
}
|
|
288
381
|
/**
|
|
289
382
|
* Retrieve previously offloaded message payload using provided pointer payload.
|
|
290
383
|
* Returns the original payload or an error if the payload was not found or could not be parsed.
|
|
384
|
+
*
|
|
385
|
+
* Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer).
|
|
291
386
|
*/
|
|
292
387
|
async retrieveOffloadedMessagePayload(maybeOffloadedPayloadPointerPayload) {
|
|
293
388
|
if (!this.payloadStoreConfig) {
|
|
@@ -303,13 +398,27 @@ export class AbstractQueueService {
|
|
|
303
398
|
}),
|
|
304
399
|
};
|
|
305
400
|
}
|
|
306
|
-
const
|
|
401
|
+
const parsedPayload = pointerPayloadParseResult.data;
|
|
402
|
+
// Resolve which store to use
|
|
403
|
+
const storeResult = this.resolveIncomingStore(parsedPayload.payloadRef, parsedPayload.offloadedPayloadPointer);
|
|
404
|
+
if (storeResult.error) {
|
|
405
|
+
return storeResult;
|
|
406
|
+
}
|
|
407
|
+
const { store, payloadId } = storeResult.result;
|
|
408
|
+
const payloadSize = parsedPayload.payloadRef?.size ?? parsedPayload.offloadedPayloadSize;
|
|
409
|
+
if (payloadSize === undefined) {
|
|
410
|
+
return {
|
|
411
|
+
error: new Error('Invalid offloaded payload: payload size is missing from both payloadRef and offloadedPayloadSize'),
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
// Retrieve the payload from the resolved store
|
|
415
|
+
const serializedOffloadedPayloadReadable = await store.retrievePayload(payloadId);
|
|
307
416
|
if (serializedOffloadedPayloadReadable === null) {
|
|
308
417
|
return {
|
|
309
|
-
error: new Error(`Payload with key ${
|
|
418
|
+
error: new Error(`Payload with key ${payloadId} was not found in the store`),
|
|
310
419
|
};
|
|
311
420
|
}
|
|
312
|
-
const serializedOffloadedPayloadString = await streamWithKnownSizeToString(serializedOffloadedPayloadReadable,
|
|
421
|
+
const serializedOffloadedPayloadString = await streamWithKnownSizeToString(serializedOffloadedPayloadReadable, payloadSize);
|
|
313
422
|
try {
|
|
314
423
|
return { result: JSON.parse(serializedOffloadedPayloadString) };
|
|
315
424
|
}
|
|
@@ -384,7 +493,7 @@ export class AbstractQueueService {
|
|
|
384
493
|
return !!this.messageDeduplicationConfig && !!this.getMessageDeduplicationId(message);
|
|
385
494
|
}
|
|
386
495
|
getMessageDeduplicationId(message) {
|
|
387
|
-
// @ts-
|
|
496
|
+
// @ts-expect-error
|
|
388
497
|
return message[this.messageDeduplicationIdField];
|
|
389
498
|
}
|
|
390
499
|
getParsedMessageDeduplicationOptions(message) {
|