@anabranch/queue 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +159 -0
- package/esm/anabranch/index.d.ts +44 -0
- package/esm/anabranch/index.d.ts.map +1 -0
- package/esm/anabranch/index.js +41 -0
- package/esm/anabranch/streams/channel.d.ts +15 -0
- package/esm/anabranch/streams/channel.d.ts.map +1 -0
- package/esm/anabranch/streams/channel.js +122 -0
- package/esm/anabranch/streams/source.d.ts +75 -0
- package/esm/anabranch/streams/source.d.ts.map +1 -0
- package/esm/anabranch/streams/source.js +77 -0
- package/esm/anabranch/streams/stream.d.ts +431 -0
- package/esm/anabranch/streams/stream.d.ts.map +1 -0
- package/esm/anabranch/streams/stream.js +627 -0
- package/esm/anabranch/streams/task.d.ts +117 -0
- package/esm/anabranch/streams/task.d.ts.map +1 -0
- package/esm/anabranch/streams/task.js +419 -0
- package/esm/anabranch/streams/util.d.ts +33 -0
- package/esm/anabranch/streams/util.d.ts.map +1 -0
- package/esm/anabranch/streams/util.js +18 -0
- package/esm/package.json +3 -0
- package/esm/queue/adapter.d.ts +98 -0
- package/esm/queue/adapter.d.ts.map +1 -0
- package/esm/queue/adapter.js +1 -0
- package/esm/queue/errors.d.ts +45 -0
- package/esm/queue/errors.d.ts.map +1 -0
- package/esm/queue/errors.js +113 -0
- package/esm/queue/in-memory.d.ts +61 -0
- package/esm/queue/in-memory.d.ts.map +1 -0
- package/esm/queue/in-memory.js +291 -0
- package/esm/queue/index.d.ts +79 -0
- package/esm/queue/index.d.ts.map +1 -0
- package/esm/queue/index.js +74 -0
- package/esm/queue/queue.d.ts +166 -0
- package/esm/queue/queue.d.ts.map +1 -0
- package/esm/queue/queue.js +284 -0
- package/package.json +24 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue adapter interface for queue-agnostic operations.
|
|
3
|
+
*
|
|
4
|
+
* Implement this interface to create drivers for specific message brokers.
|
|
5
|
+
* The Queue class wraps adapters with Task/Stream semantics.
|
|
6
|
+
*
|
|
7
|
+
* For connection lifecycle management, use QueueConnector which produces adapters.
|
|
8
|
+
* The adapter's close() method releases the connection (e.g., back to a pool)
|
|
9
|
+
* rather than terminating it — termination is the connector's responsibility.
|
|
10
|
+
*/
|
|
11
|
+
export interface QueueMessage<T = unknown> {
|
|
12
|
+
/** Unique message identifier */
|
|
13
|
+
id: string;
|
|
14
|
+
/** Message payload */
|
|
15
|
+
data: T;
|
|
16
|
+
/** Number of times this message has been delivered */
|
|
17
|
+
attempt: number;
|
|
18
|
+
/** Timestamp when the message was first enqueued */
|
|
19
|
+
timestamp: number;
|
|
20
|
+
/** Optional metadata from the broker */
|
|
21
|
+
metadata?: Record<string, unknown>;
|
|
22
|
+
}
|
|
23
|
+
/** Options for sending a message with delay or scheduling. */
|
|
24
|
+
export interface SendOptions {
|
|
25
|
+
/** Delay in milliseconds before the message becomes available */
|
|
26
|
+
delayMs?: number;
|
|
27
|
+
/** Explicit scheduled delivery time */
|
|
28
|
+
scheduledAt?: Date;
|
|
29
|
+
/** Override the default dead letter queue for this message */
|
|
30
|
+
deadLetterQueue?: string;
|
|
31
|
+
/** Message priority (higher = more important, if supported) */
|
|
32
|
+
priority?: number;
|
|
33
|
+
}
|
|
34
|
+
/** Options for negative acknowledgment. */
|
|
35
|
+
export interface NackOptions {
|
|
36
|
+
/** Requeue the message instead of dead-letter routing */
|
|
37
|
+
requeue?: boolean;
|
|
38
|
+
/** Delay before the message is requeued */
|
|
39
|
+
delay?: number;
|
|
40
|
+
/** Explicit dead letter queue target */
|
|
41
|
+
deadLetter?: boolean;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Queue adapter interface for low-level queue operations.
|
|
45
|
+
*/
|
|
46
|
+
export interface QueueAdapter {
|
|
47
|
+
send<T>(queue: string, data: T, options?: SendOptions): Promise<string>;
|
|
48
|
+
receive<T>(queue: string, count?: number): Promise<QueueMessage<T>[]>;
|
|
49
|
+
ack(queue: string, ...ids: string[]): Promise<void>;
|
|
50
|
+
nack(queue: string, id: string, options?: NackOptions): Promise<void>;
|
|
51
|
+
close(): Promise<void>;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Extended adapter interface for broker-native streaming.
|
|
55
|
+
* Implement this if your broker has push-based message consumption
|
|
56
|
+
* (e.g., RabbitMQ channels, Kafka consumer groups, SQS long polling).
|
|
57
|
+
*/
|
|
58
|
+
export interface StreamAdapter extends QueueAdapter {
|
|
59
|
+
subscribe<T>(queue: string, options?: {
|
|
60
|
+
signal?: AbortSignal;
|
|
61
|
+
}): AsyncIterable<QueueMessage<T>>;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Connector that produces connected QueueAdapter instances.
|
|
65
|
+
*
|
|
66
|
+
* Implement this to provide connection acquisition logic for your message broker.
|
|
67
|
+
* Handles pool checkout, connection creation, and termination on error.
|
|
68
|
+
*/
|
|
69
|
+
export interface QueueConnector {
|
|
70
|
+
/**
|
|
71
|
+
* Acquire a connected adapter.
|
|
72
|
+
* @param signal Optional AbortSignal for cancellation
|
|
73
|
+
* @throws QueueConnectionFailed if the connection cannot be established
|
|
74
|
+
*/
|
|
75
|
+
connect(signal?: AbortSignal): Promise<QueueAdapter>;
|
|
76
|
+
/**
|
|
77
|
+
* Close all connections and clean up resources.
|
|
78
|
+
* After calling end(), the connector cannot be used to create new adapters.
|
|
79
|
+
*/
|
|
80
|
+
end(): Promise<void>;
|
|
81
|
+
}
|
|
82
|
+
/** Queue configuration options. */
|
|
83
|
+
export interface QueueOptions {
|
|
84
|
+
/** Maximum delivery attempts before routing to dead letter queue */
|
|
85
|
+
maxAttempts?: number;
|
|
86
|
+
/** Message visibility timeout (milliseconds) - time between delivery and ACK/NACK */
|
|
87
|
+
visibilityTimeout?: number;
|
|
88
|
+
/** Default dead letter queue for failed messages */
|
|
89
|
+
deadLetterQueue?: string;
|
|
90
|
+
/** Dead letter queue specific options */
|
|
91
|
+
deadLetterOptions?: {
|
|
92
|
+
/** DLQ's own max delivery attempts */
|
|
93
|
+
maxAttempts?: number;
|
|
94
|
+
/** Delay for DLQ messages */
|
|
95
|
+
delay?: number;
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/queue/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,OAAO;IACvC,gCAAgC;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,sBAAsB;IACtB,IAAI,EAAE,CAAC,CAAC;IACR,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IAC1B,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,8DAA8D;IAC9D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,2CAA2C;AAC3C,MAAM,WAAW,WAAW;IAC1B,yDAAyD;IACzD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,CAAC,EACJ,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,CAAC,EACP,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,OAAO,CAAC,CAAC,EACP,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAE9B,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,SAAS,CAAC,CAAC,EACT,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACjC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;CACnC;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAErD;;;OAGG;IACH,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtB;AAED,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qFAAqF;IACrF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oDAAoD;IACpD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,iBAAiB,CAAC,EAAE;QAClB,sCAAsC;QACtC,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,6BAA6B;QAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error types for queue operations.
|
|
3
|
+
*/
|
|
4
|
+
/** Base error for all queue-related failures. */
|
|
5
|
+
export declare class QueueError extends Error {
|
|
6
|
+
readonly queue?: string | undefined;
|
|
7
|
+
readonly messageId?: string | undefined;
|
|
8
|
+
constructor(message: string, queue?: string | undefined, messageId?: string | undefined);
|
|
9
|
+
}
|
|
10
|
+
/** Connection establishment failed. */
|
|
11
|
+
export declare class QueueConnectionFailed extends QueueError {
|
|
12
|
+
readonly originalError?: unknown | undefined;
|
|
13
|
+
constructor(message: string, originalError?: unknown | undefined);
|
|
14
|
+
}
|
|
15
|
+
/** Message send operation failed. */
|
|
16
|
+
export declare class QueueSendFailed extends QueueError {
|
|
17
|
+
readonly originalError?: unknown | undefined;
|
|
18
|
+
constructor(message: string, queue: string, originalError?: unknown | undefined);
|
|
19
|
+
}
|
|
20
|
+
/** Message receive operation failed. */
|
|
21
|
+
export declare class QueueReceiveFailed extends QueueError {
|
|
22
|
+
readonly originalError?: unknown | undefined;
|
|
23
|
+
constructor(message: string, queue: string, originalError?: unknown | undefined);
|
|
24
|
+
}
|
|
25
|
+
/** Acknowledgment operation failed. */
|
|
26
|
+
export declare class QueueAckFailed extends QueueError {
|
|
27
|
+
readonly originalError?: unknown | undefined;
|
|
28
|
+
constructor(message: string, queue: string, messageId: string, originalError?: unknown | undefined);
|
|
29
|
+
}
|
|
30
|
+
/** Consumer handler failed unexpectedly. */
|
|
31
|
+
export declare class QueueConsumeFailed extends QueueError {
|
|
32
|
+
readonly originalError?: unknown | undefined;
|
|
33
|
+
constructor(message: string, queue: string, messageId: string, originalError?: unknown | undefined);
|
|
34
|
+
}
|
|
35
|
+
/** Message exceeded maximum delivery attempts. */
|
|
36
|
+
export declare class QueueMaxAttemptsExceeded extends QueueError {
|
|
37
|
+
readonly attempts: number;
|
|
38
|
+
constructor(queue: string, messageId: string, attempts: number);
|
|
39
|
+
}
|
|
40
|
+
/** Connection close operation failed. */
|
|
41
|
+
export declare class QueueCloseFailed extends QueueError {
|
|
42
|
+
readonly originalError?: unknown | undefined;
|
|
43
|
+
constructor(message: string, originalError?: unknown | undefined);
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/queue/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,iDAAiD;AACjD,qBAAa,UAAW,SAAQ,KAAK;aAGjB,KAAK,CAAC,EAAE,MAAM;aACd,SAAS,CAAC,EAAE,MAAM;gBAFlC,OAAO,EAAE,MAAM,EACC,KAAK,CAAC,EAAE,MAAM,YAAA,EACd,SAAS,CAAC,EAAE,MAAM,YAAA;CAKrC;AAED,uCAAuC;AACvC,qBAAa,qBAAsB,SAAQ,UAAU;aACN,aAAa,CAAC,EAAE,OAAO;gBAAxD,OAAO,EAAE,MAAM,EAAkB,aAAa,CAAC,EAAE,OAAO,YAAA;CAIrE;AAED,qCAAqC;AACrC,qBAAa,eAAgB,SAAQ,UAAU;aAI3B,aAAa,CAAC,EAAE,OAAO;gBAFvC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACG,aAAa,CAAC,EAAE,OAAO,YAAA;CAK1C;AAED,wCAAwC;AACxC,qBAAa,kBAAmB,SAAQ,UAAU;aAI9B,aAAa,CAAC,EAAE,OAAO;gBAFvC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACG,aAAa,CAAC,EAAE,OAAO,YAAA;CAK1C;AAED,uCAAuC;AACvC,qBAAa,cAAe,SAAQ,UAAU;aAK1B,aAAa,CAAC,EAAE,OAAO;gBAHvC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACD,aAAa,CAAC,EAAE,OAAO,YAAA;CAK1C;AAED,4CAA4C;AAC5C,qBAAa,kBAAmB,SAAQ,UAAU;aAK9B,aAAa,CAAC,EAAE,OAAO;gBAHvC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACD,aAAa,CAAC,EAAE,OAAO,YAAA;CAK1C;AAED,kDAAkD;AAClD,qBAAa,wBAAyB,SAAQ,UAAU;aAIpC,QAAQ,EAAE,MAAM;gBAFhC,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACD,QAAQ,EAAE,MAAM;CASnC;AAED,yCAAyC;AACzC,qBAAa,gBAAiB,SAAQ,UAAU;aAG5B,aAAa,CAAC,EAAE,OAAO;gBADvC,OAAO,EAAE,MAAM,EACC,aAAa,CAAC,EAAE,OAAO,YAAA;CAK1C"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error types for queue operations.
|
|
3
|
+
*/
|
|
4
|
+
/** Base error for all queue-related failures. */
|
|
5
|
+
export class QueueError extends Error {
|
|
6
|
+
constructor(message, queue, messageId) {
|
|
7
|
+
super(message);
|
|
8
|
+
Object.defineProperty(this, "queue", {
|
|
9
|
+
enumerable: true,
|
|
10
|
+
configurable: true,
|
|
11
|
+
writable: true,
|
|
12
|
+
value: queue
|
|
13
|
+
});
|
|
14
|
+
Object.defineProperty(this, "messageId", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: messageId
|
|
19
|
+
});
|
|
20
|
+
this.name = "QueueError";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/** Connection establishment failed. */
|
|
24
|
+
export class QueueConnectionFailed extends QueueError {
|
|
25
|
+
constructor(message, originalError) {
|
|
26
|
+
super(`Connection failed: ${message}`, undefined, undefined);
|
|
27
|
+
Object.defineProperty(this, "originalError", {
|
|
28
|
+
enumerable: true,
|
|
29
|
+
configurable: true,
|
|
30
|
+
writable: true,
|
|
31
|
+
value: originalError
|
|
32
|
+
});
|
|
33
|
+
this.name = "QueueConnectionFailed";
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** Message send operation failed. */
|
|
37
|
+
export class QueueSendFailed extends QueueError {
|
|
38
|
+
constructor(message, queue, originalError) {
|
|
39
|
+
super(message, queue);
|
|
40
|
+
Object.defineProperty(this, "originalError", {
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
writable: true,
|
|
44
|
+
value: originalError
|
|
45
|
+
});
|
|
46
|
+
this.name = "QueueSendFailed";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Message receive operation failed. */
|
|
50
|
+
export class QueueReceiveFailed extends QueueError {
|
|
51
|
+
constructor(message, queue, originalError) {
|
|
52
|
+
super(message, queue);
|
|
53
|
+
Object.defineProperty(this, "originalError", {
|
|
54
|
+
enumerable: true,
|
|
55
|
+
configurable: true,
|
|
56
|
+
writable: true,
|
|
57
|
+
value: originalError
|
|
58
|
+
});
|
|
59
|
+
this.name = "QueueReceiveFailed";
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Acknowledgment operation failed. */
|
|
63
|
+
export class QueueAckFailed extends QueueError {
|
|
64
|
+
constructor(message, queue, messageId, originalError) {
|
|
65
|
+
super(message, queue, messageId);
|
|
66
|
+
Object.defineProperty(this, "originalError", {
|
|
67
|
+
enumerable: true,
|
|
68
|
+
configurable: true,
|
|
69
|
+
writable: true,
|
|
70
|
+
value: originalError
|
|
71
|
+
});
|
|
72
|
+
this.name = "QueueAckFailed";
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Consumer handler failed unexpectedly. */
|
|
76
|
+
export class QueueConsumeFailed extends QueueError {
|
|
77
|
+
constructor(message, queue, messageId, originalError) {
|
|
78
|
+
super(message, queue, messageId);
|
|
79
|
+
Object.defineProperty(this, "originalError", {
|
|
80
|
+
enumerable: true,
|
|
81
|
+
configurable: true,
|
|
82
|
+
writable: true,
|
|
83
|
+
value: originalError
|
|
84
|
+
});
|
|
85
|
+
this.name = "QueueConsumeFailed";
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/** Message exceeded maximum delivery attempts. */
|
|
89
|
+
export class QueueMaxAttemptsExceeded extends QueueError {
|
|
90
|
+
constructor(queue, messageId, attempts) {
|
|
91
|
+
super(`Message ${messageId} exceeded ${attempts} delivery attempts in queue ${queue}`, queue, messageId);
|
|
92
|
+
Object.defineProperty(this, "attempts", {
|
|
93
|
+
enumerable: true,
|
|
94
|
+
configurable: true,
|
|
95
|
+
writable: true,
|
|
96
|
+
value: attempts
|
|
97
|
+
});
|
|
98
|
+
this.name = "QueueMaxAttemptsExceeded";
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** Connection close operation failed. */
|
|
102
|
+
export class QueueCloseFailed extends QueueError {
|
|
103
|
+
constructor(message, originalError) {
|
|
104
|
+
super(message);
|
|
105
|
+
Object.defineProperty(this, "originalError", {
|
|
106
|
+
enumerable: true,
|
|
107
|
+
configurable: true,
|
|
108
|
+
writable: true,
|
|
109
|
+
value: originalError
|
|
110
|
+
});
|
|
111
|
+
this.name = "QueueCloseFailed";
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { QueueAdapter, QueueConnector, QueueOptions } from "./adapter.js";
|
|
2
|
+
/**
|
|
3
|
+
* Creates an in-memory queue connector using a simple message store.
|
|
4
|
+
*
|
|
5
|
+
* Messages are stored in memory only and will be lost on process restart.
|
|
6
|
+
* Uses bucketed priority queues for delayed message support.
|
|
7
|
+
*
|
|
8
|
+
* @example Basic usage
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { Queue, createInMemory } from "@anabranch/queue";
|
|
11
|
+
*
|
|
12
|
+
* const connector = createInMemory();
|
|
13
|
+
*
|
|
14
|
+
* // Send a message
|
|
15
|
+
* await Queue.withConnection(connector, (queue) =>
|
|
16
|
+
* queue.send("notifications", { type: "welcome", userId: 123 })
|
|
17
|
+
* ).run();
|
|
18
|
+
*
|
|
19
|
+
* // Receive messages
|
|
20
|
+
* const { successes } = await Queue.withConnection(connector, (queue) =>
|
|
21
|
+
* queue.stream("notifications", { count: 10 })
|
|
22
|
+
* .map(async (msg) => await processNotification(msg.data))
|
|
23
|
+
* .partition()
|
|
24
|
+
* ).run();
|
|
25
|
+
* ```
|
|
26
|
+
*
|
|
27
|
+
* @example With delayed messages
|
|
28
|
+
* ```ts
|
|
29
|
+
* await Queue.withConnection(connector, (queue) =>
|
|
30
|
+
* queue.send("notifications", reminder, { delayMs: 30_000 })
|
|
31
|
+
* ).run();
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* @example With dead letter queue
|
|
35
|
+
* ```ts
|
|
36
|
+
* const connector = createInMemory({
|
|
37
|
+
* queues: {
|
|
38
|
+
* orders: {
|
|
39
|
+
* maxAttempts: 3,
|
|
40
|
+
* deadLetterQueue: "orders-failed",
|
|
41
|
+
* },
|
|
42
|
+
* },
|
|
43
|
+
* });
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export declare function createInMemory(options?: InMemoryOptions): InMemoryConnector;
|
|
47
|
+
/** In-memory queue connector options. */
|
|
48
|
+
export interface InMemoryOptions {
|
|
49
|
+
/** Maximum buffer size per queue (enforces backpressure) */
|
|
50
|
+
maxBufferSize?: number;
|
|
51
|
+
/** Callback when a message is dropped due to buffer overflow */
|
|
52
|
+
onDrop?: (value: unknown) => void;
|
|
53
|
+
/** Per-queue configuration */
|
|
54
|
+
queues?: Record<string, QueueOptions>;
|
|
55
|
+
}
|
|
56
|
+
/** In-memory queue connector. */
|
|
57
|
+
export interface InMemoryConnector extends QueueConnector {
|
|
58
|
+
connect(): Promise<QueueAdapter>;
|
|
59
|
+
end(): Promise<void>;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=in-memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"in-memory.d.ts","sourceRoot":"","sources":["../../src/queue/in-memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,YAAY,EACZ,cAAc,EAEd,YAAY,EAEb,MAAM,cAAc,CAAC;AA2BtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,iBAAiB,CAwS3E;AAkBD,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IAClC,8BAA8B;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,iCAAiC;AACjC,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,OAAO,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;IACjC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtB"}
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import { QueueReceiveFailed, QueueSendFailed } from "./errors.js";
|
|
2
|
+
function generateId() {
|
|
3
|
+
return crypto.randomUUID();
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Creates an in-memory queue connector using a simple message store.
|
|
7
|
+
*
|
|
8
|
+
* Messages are stored in memory only and will be lost on process restart.
|
|
9
|
+
* Uses bucketed priority queues for delayed message support.
|
|
10
|
+
*
|
|
11
|
+
* @example Basic usage
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { Queue, createInMemory } from "@anabranch/queue";
|
|
14
|
+
*
|
|
15
|
+
* const connector = createInMemory();
|
|
16
|
+
*
|
|
17
|
+
* // Send a message
|
|
18
|
+
* await Queue.withConnection(connector, (queue) =>
|
|
19
|
+
* queue.send("notifications", { type: "welcome", userId: 123 })
|
|
20
|
+
* ).run();
|
|
21
|
+
*
|
|
22
|
+
* // Receive messages
|
|
23
|
+
* const { successes } = await Queue.withConnection(connector, (queue) =>
|
|
24
|
+
* queue.stream("notifications", { count: 10 })
|
|
25
|
+
* .map(async (msg) => await processNotification(msg.data))
|
|
26
|
+
* .partition()
|
|
27
|
+
* ).run();
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* @example With delayed messages
|
|
31
|
+
* ```ts
|
|
32
|
+
* await Queue.withConnection(connector, (queue) =>
|
|
33
|
+
* queue.send("notifications", reminder, { delayMs: 30_000 })
|
|
34
|
+
* ).run();
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* @example With dead letter queue
|
|
38
|
+
* ```ts
|
|
39
|
+
* const connector = createInMemory({
|
|
40
|
+
* queues: {
|
|
41
|
+
* orders: {
|
|
42
|
+
* maxAttempts: 3,
|
|
43
|
+
* deadLetterQueue: "orders-failed",
|
|
44
|
+
* },
|
|
45
|
+
* },
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export function createInMemory(options) {
|
|
50
|
+
const queues = new Map();
|
|
51
|
+
const timerIds = new Set();
|
|
52
|
+
let ended = false;
|
|
53
|
+
const defaultOptions = {
|
|
54
|
+
maxAttempts: 3,
|
|
55
|
+
visibilityTimeout: 30000,
|
|
56
|
+
deadLetterQueue: "",
|
|
57
|
+
deadLetterOptions: {
|
|
58
|
+
maxAttempts: 1,
|
|
59
|
+
delay: 0,
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
const queueConfigs = options?.queues ?? {};
|
|
63
|
+
const maxBufferSize = options?.maxBufferSize ?? Infinity;
|
|
64
|
+
const onDrop = options?.onDrop;
|
|
65
|
+
return {
|
|
66
|
+
connect() {
|
|
67
|
+
let adapterRef = undefined;
|
|
68
|
+
const routeToDlq = (queue, message) => {
|
|
69
|
+
const dlqName = queue.options.deadLetterQueue;
|
|
70
|
+
if (dlqName && adapterRef) {
|
|
71
|
+
adapterRef.send(dlqName, {
|
|
72
|
+
originalId: message.id,
|
|
73
|
+
originalQueue: "source",
|
|
74
|
+
data: message.data,
|
|
75
|
+
attempt: message.attempt,
|
|
76
|
+
timestamp: message.timestamp,
|
|
77
|
+
}).catch((err) => {
|
|
78
|
+
onDrop?.({ message, error: err });
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const findMessage = (queue, id) => {
|
|
83
|
+
return (queue.messages.find((m) => m.id === id) ??
|
|
84
|
+
queue.inflight.get(id)?.msg ??
|
|
85
|
+
Array.from(queue.delayed.values())
|
|
86
|
+
.flat()
|
|
87
|
+
.find((m) => m.id === id));
|
|
88
|
+
};
|
|
89
|
+
const handleNack = (queue, id, nackOptions) => {
|
|
90
|
+
const existing = findMessage(queue, id);
|
|
91
|
+
if (nackOptions?.deadLetter) {
|
|
92
|
+
routeToDlq(queue, existing ??
|
|
93
|
+
{
|
|
94
|
+
id,
|
|
95
|
+
data: null,
|
|
96
|
+
attempt: 1,
|
|
97
|
+
timestamp: Date.now(),
|
|
98
|
+
});
|
|
99
|
+
return Promise.resolve();
|
|
100
|
+
}
|
|
101
|
+
if (nackOptions?.requeue) {
|
|
102
|
+
queue.inflight.delete(id);
|
|
103
|
+
const attempt = existing ? existing.attempt + 1 : 2;
|
|
104
|
+
if (attempt > queue.options.maxAttempts) {
|
|
105
|
+
routeToDlq(queue, {
|
|
106
|
+
id,
|
|
107
|
+
data: existing?.data ?? null,
|
|
108
|
+
attempt,
|
|
109
|
+
timestamp: Date.now(),
|
|
110
|
+
});
|
|
111
|
+
return Promise.resolve();
|
|
112
|
+
}
|
|
113
|
+
const message = {
|
|
114
|
+
id,
|
|
115
|
+
data: (existing?.data ?? null),
|
|
116
|
+
attempt,
|
|
117
|
+
timestamp: Date.now(),
|
|
118
|
+
};
|
|
119
|
+
const delay = nackOptions.delay ?? 0;
|
|
120
|
+
if (delay > 0) {
|
|
121
|
+
const bucket = Math.ceil((Date.now() + delay) / 1000);
|
|
122
|
+
if (!queue.delayed.has(bucket)) {
|
|
123
|
+
queue.delayed.set(bucket, []);
|
|
124
|
+
}
|
|
125
|
+
queue.delayed.get(bucket).push(message);
|
|
126
|
+
scheduleDelayedProcessingInner(queue);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
queue.messages.push(message);
|
|
130
|
+
}
|
|
131
|
+
return Promise.resolve();
|
|
132
|
+
}
|
|
133
|
+
return Promise.resolve();
|
|
134
|
+
};
|
|
135
|
+
const adapter = {
|
|
136
|
+
send(queueName, data, sendOptions) {
|
|
137
|
+
if (ended) {
|
|
138
|
+
return Promise.reject(new QueueSendFailed("Connector ended", queueName));
|
|
139
|
+
}
|
|
140
|
+
let queue = queues.get(queueName);
|
|
141
|
+
if (!queue) {
|
|
142
|
+
const config = queueConfigs[queueName] ?? {};
|
|
143
|
+
queue = {
|
|
144
|
+
messages: [],
|
|
145
|
+
delayed: new Map(),
|
|
146
|
+
inflight: new Map(),
|
|
147
|
+
options: {
|
|
148
|
+
...defaultOptions,
|
|
149
|
+
...config,
|
|
150
|
+
deadLetterQueue: config.deadLetterQueue ?? "",
|
|
151
|
+
deadLetterOptions: {
|
|
152
|
+
...defaultOptions.deadLetterOptions,
|
|
153
|
+
...config.deadLetterOptions,
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
queues.set(queueName, queue);
|
|
158
|
+
}
|
|
159
|
+
const id = generateId();
|
|
160
|
+
const delayMs = sendOptions?.delayMs ??
|
|
161
|
+
(sendOptions?.scheduledAt
|
|
162
|
+
? Math.max(0, sendOptions.scheduledAt.getTime() - Date.now())
|
|
163
|
+
: 0);
|
|
164
|
+
const message = {
|
|
165
|
+
id,
|
|
166
|
+
data,
|
|
167
|
+
attempt: 1,
|
|
168
|
+
timestamp: Date.now(),
|
|
169
|
+
};
|
|
170
|
+
if (delayMs > 0) {
|
|
171
|
+
const bucket = Math.ceil((Date.now() + delayMs) / 1000);
|
|
172
|
+
if (!queue.delayed.has(bucket)) {
|
|
173
|
+
queue.delayed.set(bucket, []);
|
|
174
|
+
}
|
|
175
|
+
queue.delayed.get(bucket).push(message);
|
|
176
|
+
scheduleDelayedProcessingInner(queue);
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
if (queue.messages.length >= maxBufferSize) {
|
|
180
|
+
if (maxBufferSize !== Infinity) {
|
|
181
|
+
onDrop?.(message);
|
|
182
|
+
}
|
|
183
|
+
return Promise.reject(new Error("Buffer full"));
|
|
184
|
+
}
|
|
185
|
+
queue.messages.push(message);
|
|
186
|
+
}
|
|
187
|
+
return Promise.resolve(id);
|
|
188
|
+
},
|
|
189
|
+
receive(queueName, count) {
|
|
190
|
+
if (ended) {
|
|
191
|
+
return Promise.reject(new QueueReceiveFailed("Connector ended", queueName));
|
|
192
|
+
}
|
|
193
|
+
const queue = queues.get(queueName);
|
|
194
|
+
if (!queue) {
|
|
195
|
+
return Promise.resolve([]);
|
|
196
|
+
}
|
|
197
|
+
promoteDelayedMessagesInner(queue);
|
|
198
|
+
expireVisibleMessages(queue);
|
|
199
|
+
const messages = [];
|
|
200
|
+
const toReceive = count ?? 10;
|
|
201
|
+
const now = Date.now();
|
|
202
|
+
while (messages.length < toReceive && queue.messages.length > 0) {
|
|
203
|
+
const msg = queue.messages.shift();
|
|
204
|
+
if (msg) {
|
|
205
|
+
messages.push(msg);
|
|
206
|
+
queue.inflight.set(msg.id, { msg, deliveredAt: now });
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return Promise.resolve(messages);
|
|
210
|
+
},
|
|
211
|
+
ack(queueName, ...ids) {
|
|
212
|
+
if (ended) {
|
|
213
|
+
return Promise.reject(new QueueSendFailed("Connector ended", queueName));
|
|
214
|
+
}
|
|
215
|
+
const queue = queues.get(queueName);
|
|
216
|
+
if (!queue)
|
|
217
|
+
return Promise.resolve();
|
|
218
|
+
for (const id of ids) {
|
|
219
|
+
queue.inflight.delete(id);
|
|
220
|
+
}
|
|
221
|
+
queue.messages = queue.messages.filter((m) => !ids.includes(m.id));
|
|
222
|
+
return Promise.resolve();
|
|
223
|
+
},
|
|
224
|
+
nack(queueName, id, nackOptions) {
|
|
225
|
+
if (ended) {
|
|
226
|
+
return Promise.reject(new QueueSendFailed("Connector ended", queueName));
|
|
227
|
+
}
|
|
228
|
+
const queue = queues.get(queueName);
|
|
229
|
+
if (!queue)
|
|
230
|
+
return Promise.resolve();
|
|
231
|
+
return handleNack(queue, id, nackOptions);
|
|
232
|
+
},
|
|
233
|
+
close() {
|
|
234
|
+
return Promise.resolve();
|
|
235
|
+
},
|
|
236
|
+
};
|
|
237
|
+
adapterRef = adapter;
|
|
238
|
+
const promoteDelayedMessagesInner = (queue) => {
|
|
239
|
+
const now = Math.ceil(Date.now() / 1000);
|
|
240
|
+
for (const [bucket, messages] of queue.delayed) {
|
|
241
|
+
if (bucket <= now) {
|
|
242
|
+
queue.messages.push(...messages);
|
|
243
|
+
queue.delayed.delete(bucket);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
const scheduleDelayedProcessingInner = (queue) => {
|
|
248
|
+
if (queue.delayed.size === 0)
|
|
249
|
+
return;
|
|
250
|
+
const nextBucket = Math.min(...queue.delayed.keys());
|
|
251
|
+
const delay = Math.max(0, nextBucket * 1000 - Date.now());
|
|
252
|
+
const timerId = setTimeout(() => {
|
|
253
|
+
timerIds.delete(timerId);
|
|
254
|
+
promoteDelayedMessagesInner(queue);
|
|
255
|
+
if (queue.delayed.size > 0) {
|
|
256
|
+
scheduleDelayedProcessingInner(queue);
|
|
257
|
+
}
|
|
258
|
+
}, Math.max(delay, 10));
|
|
259
|
+
timerIds.add(timerId);
|
|
260
|
+
};
|
|
261
|
+
return Promise.resolve(adapter);
|
|
262
|
+
},
|
|
263
|
+
end() {
|
|
264
|
+
ended = true;
|
|
265
|
+
for (const timerId of timerIds) {
|
|
266
|
+
clearTimeout(timerId);
|
|
267
|
+
}
|
|
268
|
+
timerIds.clear();
|
|
269
|
+
for (const queue of queues.values()) {
|
|
270
|
+
queue.messages = [];
|
|
271
|
+
queue.delayed.clear();
|
|
272
|
+
queue.inflight.clear();
|
|
273
|
+
}
|
|
274
|
+
queues.clear();
|
|
275
|
+
return Promise.resolve();
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
const checkVisibilityTimeout = (queue) => {
|
|
280
|
+
const now = Date.now();
|
|
281
|
+
const timeout = queue.options.visibilityTimeout;
|
|
282
|
+
for (const [id, entry] of queue.inflight) {
|
|
283
|
+
if (entry.deliveredAt + timeout <= now) {
|
|
284
|
+
queue.inflight.delete(id);
|
|
285
|
+
queue.messages.push(entry.msg);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
function expireVisibleMessages(queue) {
|
|
290
|
+
checkVisibilityTimeout(queue);
|
|
291
|
+
}
|