@znight/messaging 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +622 -2
- package/dist/async-CFh_bvex.mjs +216 -0
- package/dist/async-CFh_bvex.mjs.map +1 -0
- package/dist/codec-Dku-NXXU.mjs +83 -0
- package/dist/codec-Dku-NXXU.mjs.map +1 -0
- package/dist/compiler.d.mts +75 -0
- package/dist/compiler.mjs +1023 -0
- package/dist/compiler.mjs.map +1 -0
- package/dist/driver-ZxLqy_Ap.d.mts +216 -0
- package/dist/index.d.mts +459 -0
- package/dist/index.mjs +870 -0
- package/dist/index.mjs.map +1 -0
- package/dist/kafka.d.mts +51 -0
- package/dist/kafka.mjs +517 -0
- package/dist/kafka.mjs.map +1 -0
- package/dist/memory.d.mts +124 -0
- package/dist/memory.mjs +429 -0
- package/dist/memory.mjs.map +1 -0
- package/dist/rabbitmq.d.mts +65 -0
- package/dist/rabbitmq.mjs +686 -0
- package/dist/rabbitmq.mjs.map +1 -0
- package/dist/shared-BJBpLS8V.mjs +52 -0
- package/dist/shared-BJBpLS8V.mjs.map +1 -0
- package/dist/sqs.d.mts +38 -0
- package/dist/sqs.mjs +421 -0
- package/dist/sqs.mjs.map +1 -0
- package/package.json +123 -4
- package/src/broker.ts +156 -0
- package/src/codec.ts +98 -0
- package/src/compiler/analyze.ts +800 -0
- package/src/compiler/constants.ts +73 -0
- package/src/compiler/data.ts +75 -0
- package/src/compiler/generate.ts +131 -0
- package/src/compiler/index.ts +52 -0
- package/src/compiler/patterns.ts +54 -0
- package/src/compiler/retry.ts +136 -0
- package/src/compiler/validate.ts +57 -0
- package/src/compiler/values.ts +76 -0
- package/src/consumers.ts +260 -0
- package/src/decorators.ts +131 -0
- package/src/driver.ts +142 -0
- package/src/drivers/kafka.ts +765 -0
- package/src/drivers/memory.ts +609 -0
- package/src/drivers/rabbitmq.ts +944 -0
- package/src/drivers/shared.ts +81 -0
- package/src/drivers/sqs.ts +583 -0
- package/src/errors.ts +95 -0
- package/src/headers.ts +70 -0
- package/src/hooks.ts +2 -0
- package/src/index.ts +89 -0
- package/src/internal/async.ts +115 -0
- package/src/message.ts +96 -0
- package/src/metadata.ts +65 -0
- package/src/plugin.ts +236 -0
- package/src/retry.ts +129 -0
- package/src/runtime.ts +424 -0
- package/src/standard-schema.ts +68 -0
- package/src/tokens.ts +14 -0
- package/src/validation.ts +61 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/** Helpers shared by the drivers (internal). */
|
|
2
|
+
import type { DeadLetterRequest } from '../driver.ts'
|
|
3
|
+
|
|
4
|
+
import { headerRecord, MessagingHeaders } from '../headers.ts'
|
|
5
|
+
import { errorMessage, truncate } from '../internal/async.ts'
|
|
6
|
+
|
|
7
|
+
/** Longest error message copied into the `x-znight-error` header. */
|
|
8
|
+
export const MAX_ERROR_HEADER_LENGTH = 1024
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Headers of a dead-lettered copy: the original headers without the attempt counter (so that a
|
|
12
|
+
* redrive starts over) plus the reason, the error, the original queue and the attempts made.
|
|
13
|
+
*/
|
|
14
|
+
export function deadLetterHeaders(
|
|
15
|
+
headers: Readonly<Record<string, string>>,
|
|
16
|
+
request: DeadLetterRequest,
|
|
17
|
+
originalQueue: string,
|
|
18
|
+
): Record<string, string> {
|
|
19
|
+
const result = headerRecord()
|
|
20
|
+
for (const [name, value] of Object.entries(headers)) if (name !== MessagingHeaders.Attempt) result[name] = value
|
|
21
|
+
result[MessagingHeaders.DeadLetterReason] = request.reason
|
|
22
|
+
result[MessagingHeaders.Error] = truncate(errorMessage(request.error), MAX_ERROR_HEADER_LENGTH)
|
|
23
|
+
result[MessagingHeaders.OriginalQueue] = originalQueue
|
|
24
|
+
result[MessagingHeaders.Attempts] = String(request.attempt)
|
|
25
|
+
return result
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Converts a header value of any type to a string (`undefined` for null/undefined). */
|
|
29
|
+
export function headerValueToString(value: unknown): string | undefined {
|
|
30
|
+
if (value === undefined || value === null) return undefined
|
|
31
|
+
if (typeof value === 'string') return value
|
|
32
|
+
if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') return String(value)
|
|
33
|
+
if (value instanceof Uint8Array) return new TextDecoder().decode(value)
|
|
34
|
+
if (value instanceof Date) return value.toISOString()
|
|
35
|
+
try {
|
|
36
|
+
return JSON.stringify(value, (_key, inner: unknown) => (typeof inner === 'bigint' ? inner.toString() : inner))
|
|
37
|
+
} catch {
|
|
38
|
+
return String(value)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Builds a string header record from arbitrary entries (null-prototype, `undefined` values skipped). */
|
|
43
|
+
export function toHeaderRecord(entries: Iterable<readonly [string, unknown]>): Record<string, string> {
|
|
44
|
+
const result = headerRecord()
|
|
45
|
+
for (const [name, value] of entries) {
|
|
46
|
+
const text = headerValueToString(value)
|
|
47
|
+
if (text !== undefined) result[name] = text
|
|
48
|
+
}
|
|
49
|
+
return result
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Reads a header (exact name). */
|
|
53
|
+
export function readHeader(headers: Readonly<Record<string, string>>, name: string): string | undefined {
|
|
54
|
+
return Object.hasOwn(headers, name) ? headers[name] : undefined
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Positive finite number option with a default. */
|
|
58
|
+
export function durationOption(value: unknown, fallback: number, name: string): number {
|
|
59
|
+
if (value === undefined) return fallback
|
|
60
|
+
if (typeof value !== 'number' || Number.isNaN(value) || value < 0) {
|
|
61
|
+
throw new TypeError(`[znight] ${name} must be a number of milliseconds >= 0, got ${String(value)}`)
|
|
62
|
+
}
|
|
63
|
+
return value
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** A promise with its resolvers. */
|
|
67
|
+
export interface Deferred<T> {
|
|
68
|
+
readonly promise: Promise<T>
|
|
69
|
+
resolve(value: T): void
|
|
70
|
+
reject(error: unknown): void
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function deferred<T>(): Deferred<T> {
|
|
74
|
+
let resolve!: (value: T) => void
|
|
75
|
+
let reject!: (error: unknown) => void
|
|
76
|
+
const promise = new Promise<T>((res, rej) => {
|
|
77
|
+
resolve = res
|
|
78
|
+
reject = rej
|
|
79
|
+
})
|
|
80
|
+
return { promise, resolve, reject }
|
|
81
|
+
}
|
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@znight/messaging/sqs`: Amazon SQS driver built on `@aws-sdk/client-sqs` v3 (optional peer
|
|
3
|
+
* dependency). Standard and FIFO queues.
|
|
4
|
+
*
|
|
5
|
+
* - Long polling (`WaitTimeSeconds`, default 20 s), never more messages in flight than the consumer
|
|
6
|
+
* concurrency (batches of up to 10).
|
|
7
|
+
* - Attempts come from `ApproximateReceiveCount`.
|
|
8
|
+
* - Retries change the visibility timeout of the message (`ChangeMessageVisibility`): the message
|
|
9
|
+
* reappears after the delay (up to 12 h), no republication.
|
|
10
|
+
* - The visibility of in-flight messages is extended while their handler runs (lease extension).
|
|
11
|
+
* - Dead letters are sent to the dead-letter queue (`<queue>-dlq`, or the target of the queue
|
|
12
|
+
* `RedrivePolicy`), then deleted from the queue. At subscribe time the driver verifies that the
|
|
13
|
+
* dead-letter queue exists and that the redrive `maxReceiveCount` allows every `@Retry` attempt.
|
|
14
|
+
* - Headers are message attributes (SQS allows 10 per message); binary payloads are sent base64
|
|
15
|
+
* encoded (`x-znight-encoding: base64`).
|
|
16
|
+
*/
|
|
17
|
+
import type { Message, MessageAttributeValue, SendMessageCommandInput, SQSClientConfig } from '@aws-sdk/client-sqs'
|
|
18
|
+
import type {
|
|
19
|
+
DeadLetterRequest,
|
|
20
|
+
DeliveryHandler,
|
|
21
|
+
DriverCapabilities,
|
|
22
|
+
DriverContext,
|
|
23
|
+
DriverDelivery,
|
|
24
|
+
MessagingDriver,
|
|
25
|
+
RetryRequest,
|
|
26
|
+
SubscribeOptions,
|
|
27
|
+
Subscription,
|
|
28
|
+
} from '../driver.ts'
|
|
29
|
+
import type { OutboundMessage } from '../message.ts'
|
|
30
|
+
|
|
31
|
+
import {
|
|
32
|
+
ChangeMessageVisibilityCommand,
|
|
33
|
+
DeleteMessageCommand,
|
|
34
|
+
GetQueueAttributesCommand,
|
|
35
|
+
GetQueueUrlCommand,
|
|
36
|
+
ReceiveMessageCommand,
|
|
37
|
+
SendMessageCommand,
|
|
38
|
+
SQSClient,
|
|
39
|
+
} from '@aws-sdk/client-sqs'
|
|
40
|
+
|
|
41
|
+
import { bodyToBytes } from '../codec.ts'
|
|
42
|
+
import { brokerUnavailableError, consumerError } from '../errors.ts'
|
|
43
|
+
import { headerRecord, MessagingHeaders } from '../headers.ts'
|
|
44
|
+
import { errorMessage, sleep, truncate } from '../internal/async.ts'
|
|
45
|
+
import { durationOption, MAX_ERROR_HEADER_LENGTH } from './shared.ts'
|
|
46
|
+
|
|
47
|
+
/** Options of {@link sqs}. */
|
|
48
|
+
export interface SqsDriverOptions {
|
|
49
|
+
/** An existing client (you keep ownership: it is not destroyed on close). */
|
|
50
|
+
readonly client?: SQSClient
|
|
51
|
+
/** Configuration of the client created by the driver (region, endpoint, credentials...). */
|
|
52
|
+
readonly clientConfig?: SQSClientConfig
|
|
53
|
+
/** Queue URLs by queue name, to skip `GetQueueUrl`. Queue arguments may also be full URLs. */
|
|
54
|
+
readonly queueUrls?: Readonly<Record<string, string>>
|
|
55
|
+
/** Long polling duration, in seconds (0–20). Default: 20. */
|
|
56
|
+
readonly waitTimeSeconds?: number
|
|
57
|
+
/** Visibility timeout of received messages, in seconds. Default: the queue attribute. */
|
|
58
|
+
readonly visibilityTimeout?: number
|
|
59
|
+
/** Extend the visibility of messages while they are handled. Default: `true`. */
|
|
60
|
+
readonly extendVisibility?: boolean
|
|
61
|
+
/** Longest retry delay waited in-process. Default: 1000 ms (SQS delays have a 1 s granularity). */
|
|
62
|
+
readonly maxInProcessDelayMs?: number
|
|
63
|
+
/** Default dead-letter queue name. Default: `<queue>-dlq` (`<name>-dlq.fifo` for FIFO queues). */
|
|
64
|
+
readonly deadLetterName?: (queue: string) => string
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Longest visibility timeout (12 hours). */
|
|
68
|
+
const MAX_VISIBILITY_SECONDS = 43_200
|
|
69
|
+
/** Longest `DelaySeconds` (15 minutes). */
|
|
70
|
+
const MAX_DELAY_SECONDS = 900
|
|
71
|
+
const MAX_ATTRIBUTES = 10
|
|
72
|
+
const ATTRIBUTE_NAME = /^(?!aws\.|amazon\.)(?!\.)(?!.*\.\.)[\w.-]{1,256}(?<!\.)$/i
|
|
73
|
+
/** Message attributes carrying transport fields (not exposed as headers). */
|
|
74
|
+
const TRANSPORT_ATTRIBUTES = new Set<string>([
|
|
75
|
+
MessagingHeaders.Id,
|
|
76
|
+
MessagingHeaders.Type,
|
|
77
|
+
MessagingHeaders.ContentType,
|
|
78
|
+
MessagingHeaders.Encoding,
|
|
79
|
+
])
|
|
80
|
+
|
|
81
|
+
/** Creates a {@link SqsDriver}. */
|
|
82
|
+
export function sqs(options: SqsDriverOptions = {}): SqsDriver {
|
|
83
|
+
return new SqsDriver(options)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
interface QueueInfo {
|
|
87
|
+
readonly name: string
|
|
88
|
+
readonly url: string
|
|
89
|
+
readonly fifo: boolean
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
interface SqsSubscription {
|
|
93
|
+
readonly queue: QueueInfo
|
|
94
|
+
readonly handler: DeliveryHandler
|
|
95
|
+
readonly options: SubscribeOptions
|
|
96
|
+
readonly deadLetter: QueueInfo | undefined
|
|
97
|
+
readonly visibilityTimeout: number
|
|
98
|
+
readonly abort: AbortController
|
|
99
|
+
/** Last delivery of each FIFO message group (serialised per group). */
|
|
100
|
+
readonly groups: Map<string, Promise<void>>
|
|
101
|
+
inflight: number
|
|
102
|
+
active: boolean
|
|
103
|
+
wake: (() => void) | undefined
|
|
104
|
+
loop: Promise<void> | undefined
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Amazon SQS {@link MessagingDriver}. See the module documentation. */
|
|
108
|
+
export class SqsDriver implements MessagingDriver {
|
|
109
|
+
readonly name = 'sqs'
|
|
110
|
+
readonly capabilities: DriverCapabilities
|
|
111
|
+
readonly #options: SqsDriverOptions
|
|
112
|
+
readonly #subscriptions = new Set<SqsSubscription>()
|
|
113
|
+
readonly #queues = new Map<string, Promise<QueueInfo>>()
|
|
114
|
+
#client: SQSClient | undefined
|
|
115
|
+
#ownsClient = false
|
|
116
|
+
#logger: DriverContext['logger'] = { debug() {}, info() {}, warn() {}, error() {} }
|
|
117
|
+
#closed = false
|
|
118
|
+
|
|
119
|
+
constructor(options: SqsDriverOptions = {}) {
|
|
120
|
+
const wait = options.waitTimeSeconds
|
|
121
|
+
if (wait !== undefined && !(Number.isInteger(wait) && wait >= 0 && wait <= 20)) {
|
|
122
|
+
throw new TypeError(`[znight] sqs(): waitTimeSeconds must be an integer between 0 and 20, got ${String(wait)}`)
|
|
123
|
+
}
|
|
124
|
+
const visibility = options.visibilityTimeout
|
|
125
|
+
if (
|
|
126
|
+
visibility !== undefined &&
|
|
127
|
+
!(Number.isInteger(visibility) && visibility >= 0 && visibility <= MAX_VISIBILITY_SECONDS)
|
|
128
|
+
) {
|
|
129
|
+
throw new TypeError('[znight] sqs(): visibilityTimeout must be an integer number of seconds (0–43200)')
|
|
130
|
+
}
|
|
131
|
+
this.#options = options
|
|
132
|
+
this.capabilities = Object.freeze({
|
|
133
|
+
nativeDelay: true,
|
|
134
|
+
maxNativeDelayMs: MAX_VISIBILITY_SECONDS * 1000,
|
|
135
|
+
maxInProcessDelayMs: durationOption(options.maxInProcessDelayMs, 1000, 'sqs maxInProcessDelayMs'),
|
|
136
|
+
deliveryCount: 'native',
|
|
137
|
+
maxPublishDelayMs: MAX_DELAY_SECONDS * 1000,
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The SQS client (available once connected). */
|
|
142
|
+
get client(): SQSClient | undefined {
|
|
143
|
+
return this.#client
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async connect(context: DriverContext): Promise<void> {
|
|
147
|
+
await Promise.resolve()
|
|
148
|
+
this.#logger = context.logger
|
|
149
|
+
if (this.#closed) throw brokerUnavailableError('SQS: the driver is closed, create a new one')
|
|
150
|
+
if (this.#client !== undefined) return
|
|
151
|
+
if (this.#options.client === undefined) {
|
|
152
|
+
this.#client = new SQSClient(this.#options.clientConfig ?? {})
|
|
153
|
+
this.#ownsClient = true
|
|
154
|
+
} else {
|
|
155
|
+
this.#client = this.#options.client
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async close(): Promise<void> {
|
|
160
|
+
if (this.#closed) return
|
|
161
|
+
this.#closed = true
|
|
162
|
+
const subscriptions = [...this.#subscriptions]
|
|
163
|
+
this.#subscriptions.clear()
|
|
164
|
+
for (const subscription of subscriptions) this.#stopPolling(subscription)
|
|
165
|
+
await Promise.allSettled(subscriptions.map(subscription => subscription.loop))
|
|
166
|
+
if (this.#ownsClient) this.#client?.destroy()
|
|
167
|
+
this.#client = undefined
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
deadLetterName(queue: string): string {
|
|
171
|
+
if (this.#options.deadLetterName !== undefined) return this.#options.deadLetterName(queue)
|
|
172
|
+
const name = queueName(queue)
|
|
173
|
+
return name.endsWith('.fifo') ? `${name.slice(0, -5)}-dlq.fifo` : `${name}-dlq`
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
#sqs(action: string): SQSClient {
|
|
177
|
+
const client = this.#client
|
|
178
|
+
if (client === undefined) {
|
|
179
|
+
throw brokerUnavailableError(
|
|
180
|
+
this.#closed ? `SQS: cannot ${action}, the driver is closed` : `SQS: cannot ${action} before connect()`,
|
|
181
|
+
)
|
|
182
|
+
}
|
|
183
|
+
return client
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Resolves (and caches) the URL of a queue name or URL. */
|
|
187
|
+
#queue(queue: string): Promise<QueueInfo> {
|
|
188
|
+
let info = this.#queues.get(queue)
|
|
189
|
+
if (info === undefined) {
|
|
190
|
+
info = this.#resolveQueue(queue)
|
|
191
|
+
this.#queues.set(queue, info)
|
|
192
|
+
info.catch(() => this.#queues.delete(queue))
|
|
193
|
+
}
|
|
194
|
+
return info
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
async #resolveQueue(queue: string): Promise<QueueInfo> {
|
|
198
|
+
const name = queueName(queue)
|
|
199
|
+
const configured = this.#options.queueUrls?.[queue]
|
|
200
|
+
if (configured !== undefined) return { name, url: configured, fifo: name.endsWith('.fifo') }
|
|
201
|
+
if (/^https?:\/\//.test(queue)) return { name, url: queue, fifo: name.endsWith('.fifo') }
|
|
202
|
+
try {
|
|
203
|
+
const result = await this.#sqs(`resolve the queue "${queue}"`).send(new GetQueueUrlCommand({ QueueName: queue }))
|
|
204
|
+
if (result.QueueUrl === undefined) throw new Error('no QueueUrl in the response')
|
|
205
|
+
return { name, url: result.QueueUrl, fifo: name.endsWith('.fifo') }
|
|
206
|
+
} catch (error) {
|
|
207
|
+
throw consumerError(`SQS: queue "${queue}" not found (${errorMessage(error)})`, [
|
|
208
|
+
'Create the queue, or map its URL with sqs({ queueUrls: { name: url } })',
|
|
209
|
+
])
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// -----------------------------------------------------------------------------------------------
|
|
214
|
+
// Publishing
|
|
215
|
+
// -----------------------------------------------------------------------------------------------
|
|
216
|
+
|
|
217
|
+
async publish(queue: string, message: OutboundMessage): Promise<void> {
|
|
218
|
+
const info = await this.#queue(queue)
|
|
219
|
+
const binary = typeof message.body !== 'string'
|
|
220
|
+
const attributes: Record<string, string> = headerRecord()
|
|
221
|
+
attributes[MessagingHeaders.Id] = message.id
|
|
222
|
+
if (message.type !== undefined) attributes[MessagingHeaders.Type] = message.type
|
|
223
|
+
attributes[MessagingHeaders.ContentType] = message.contentType
|
|
224
|
+
if (binary) attributes[MessagingHeaders.Encoding] = 'base64'
|
|
225
|
+
for (const [name, value] of Object.entries(message.headers)) {
|
|
226
|
+
if (TRANSPORT_ATTRIBUTES.has(name)) continue
|
|
227
|
+
attributes[name] = value
|
|
228
|
+
}
|
|
229
|
+
const input: SendMessageCommandInput = {
|
|
230
|
+
QueueUrl: info.url,
|
|
231
|
+
MessageBody: encodeBody(message.body),
|
|
232
|
+
MessageAttributes: toAttributes(attributes, queue),
|
|
233
|
+
}
|
|
234
|
+
if (message.delayMs > 0) {
|
|
235
|
+
if (info.fifo) throw new RangeError(`[znight] SQS FIFO queue "${queue}" does not support per-message delays`)
|
|
236
|
+
input.DelaySeconds = Math.min(MAX_DELAY_SECONDS, Math.ceil(message.delayMs / 1000))
|
|
237
|
+
}
|
|
238
|
+
if (info.fifo) {
|
|
239
|
+
input.MessageGroupId = message.key ?? 'default'
|
|
240
|
+
input.MessageDeduplicationId = message.id
|
|
241
|
+
}
|
|
242
|
+
await this.#sqs(`publish to "${queue}"`).send(new SendMessageCommand(input))
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// -----------------------------------------------------------------------------------------------
|
|
246
|
+
// Consuming
|
|
247
|
+
// -----------------------------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
async subscribe(queue: string, handler: DeliveryHandler, options: SubscribeOptions): Promise<Subscription> {
|
|
250
|
+
const info = await this.#queue(queue)
|
|
251
|
+
const client = this.#sqs(`subscribe to "${queue}"`)
|
|
252
|
+
const { Attributes: attributes = {} } = await client.send(
|
|
253
|
+
new GetQueueAttributesCommand({ QueueUrl: info.url, AttributeNames: ['VisibilityTimeout', 'RedrivePolicy'] }),
|
|
254
|
+
)
|
|
255
|
+
const visibilityTimeout = this.#options.visibilityTimeout ?? (Number(attributes.VisibilityTimeout) || 30)
|
|
256
|
+
const redrive = parseRedrivePolicy(attributes.RedrivePolicy)
|
|
257
|
+
if (redrive !== undefined && redrive.maxReceiveCount < options.maxAttempts) {
|
|
258
|
+
throw consumerError(
|
|
259
|
+
`SQS: queue "${queue}" moves messages to its dead-letter queue after ${redrive.maxReceiveCount} receive(s), but ${options.consumer} allows ${options.maxAttempts} attempt(s)`,
|
|
260
|
+
[
|
|
261
|
+
`Raise the RedrivePolicy maxReceiveCount of "${queue}" to at least ${options.maxAttempts}`,
|
|
262
|
+
'Or lower the @Retry attempts',
|
|
263
|
+
],
|
|
264
|
+
)
|
|
265
|
+
}
|
|
266
|
+
let deadLetter: QueueInfo | undefined
|
|
267
|
+
let dlqName = options.deadLetterQueue
|
|
268
|
+
if (dlqName !== undefined && redrive !== undefined) {
|
|
269
|
+
const target = queueName(redrive.deadLetterTargetArn)
|
|
270
|
+
if (dlqName === this.deadLetterName(queue)) dlqName = target
|
|
271
|
+
else if (dlqName !== target) {
|
|
272
|
+
this.#logger.warn(
|
|
273
|
+
`SQS: ${options.consumer} dead-letters to "${dlqName}" but the RedrivePolicy of "${queue}" targets "${target}"`,
|
|
274
|
+
)
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (dlqName !== undefined) deadLetter = await this.#queue(dlqName)
|
|
278
|
+
|
|
279
|
+
const subscription: SqsSubscription = {
|
|
280
|
+
queue: info,
|
|
281
|
+
handler,
|
|
282
|
+
options,
|
|
283
|
+
deadLetter,
|
|
284
|
+
visibilityTimeout,
|
|
285
|
+
abort: new AbortController(),
|
|
286
|
+
groups: new Map(),
|
|
287
|
+
inflight: 0,
|
|
288
|
+
active: true,
|
|
289
|
+
wake: undefined,
|
|
290
|
+
loop: undefined,
|
|
291
|
+
}
|
|
292
|
+
this.#subscriptions.add(subscription)
|
|
293
|
+
subscription.loop = this.#poll(subscription)
|
|
294
|
+
return {
|
|
295
|
+
cancel: async (): Promise<void> => {
|
|
296
|
+
this.#stopPolling(subscription)
|
|
297
|
+
await subscription.loop
|
|
298
|
+
},
|
|
299
|
+
close: async (): Promise<void> => {
|
|
300
|
+
this.#stopPolling(subscription)
|
|
301
|
+
this.#subscriptions.delete(subscription)
|
|
302
|
+
await subscription.loop
|
|
303
|
+
},
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#stopPolling(subscription: SqsSubscription): void {
|
|
308
|
+
subscription.active = false
|
|
309
|
+
subscription.abort.abort()
|
|
310
|
+
subscription.wake?.()
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
async #poll(subscription: SqsSubscription): Promise<void> {
|
|
314
|
+
const { queue, options } = subscription
|
|
315
|
+
const signal = subscription.abort.signal
|
|
316
|
+
let errorDelay = 500
|
|
317
|
+
while (subscription.active) {
|
|
318
|
+
const free = options.concurrency - subscription.inflight
|
|
319
|
+
if (free <= 0) {
|
|
320
|
+
await new Promise<void>(resolve => {
|
|
321
|
+
subscription.wake = resolve
|
|
322
|
+
})
|
|
323
|
+
subscription.wake = undefined
|
|
324
|
+
continue
|
|
325
|
+
}
|
|
326
|
+
try {
|
|
327
|
+
const result = await this.#sqs(`receive from "${queue.name}"`).send(
|
|
328
|
+
new ReceiveMessageCommand({
|
|
329
|
+
QueueUrl: queue.url,
|
|
330
|
+
MaxNumberOfMessages: Math.min(10, free),
|
|
331
|
+
WaitTimeSeconds: this.#options.waitTimeSeconds ?? 20,
|
|
332
|
+
VisibilityTimeout: subscription.visibilityTimeout,
|
|
333
|
+
MessageSystemAttributeNames: ['ApproximateReceiveCount', 'SentTimestamp', 'MessageGroupId'],
|
|
334
|
+
MessageAttributeNames: ['All'],
|
|
335
|
+
}),
|
|
336
|
+
{ abortSignal: signal },
|
|
337
|
+
)
|
|
338
|
+
errorDelay = 500
|
|
339
|
+
for (const message of result.Messages ?? []) {
|
|
340
|
+
if (subscription.active) this.#deliver(subscription, message)
|
|
341
|
+
// stopped while receiving: the message becomes visible again after its visibility timeout
|
|
342
|
+
}
|
|
343
|
+
} catch (error) {
|
|
344
|
+
if (!subscription.active) break
|
|
345
|
+
this.#logger.warn(`SQS: receiving from "${queue.name}" failed, retrying in ${errorDelay} ms`, { err: error })
|
|
346
|
+
try {
|
|
347
|
+
await sleep(errorDelay, signal)
|
|
348
|
+
} catch {
|
|
349
|
+
break
|
|
350
|
+
}
|
|
351
|
+
errorDelay = Math.min(errorDelay * 2, 30_000)
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
#deliver(subscription: SqsSubscription, message: Message): void {
|
|
357
|
+
const { queue } = subscription
|
|
358
|
+
const client = this.#sqs('handle a message')
|
|
359
|
+
const receiptHandle = message.ReceiptHandle as string
|
|
360
|
+
const attributes = message.MessageAttributes ?? {}
|
|
361
|
+
const transport = headerRecord()
|
|
362
|
+
const headers = headerRecord()
|
|
363
|
+
for (const [name, attribute] of Object.entries(attributes)) {
|
|
364
|
+
const value = attributeToString(attribute)
|
|
365
|
+
if (value === undefined) continue
|
|
366
|
+
if (TRANSPORT_ATTRIBUTES.has(name)) transport[name] = value
|
|
367
|
+
else headers[name] = value
|
|
368
|
+
}
|
|
369
|
+
const encoded = message.Body ?? ''
|
|
370
|
+
const body = transport[MessagingHeaders.Encoding] === 'base64' ? Buffer.from(encoded, 'base64') : encoded
|
|
371
|
+
const system = message.Attributes ?? {}
|
|
372
|
+
const receiveCount = Number(system.ApproximateReceiveCount)
|
|
373
|
+
const sentAt = Number(system.SentTimestamp)
|
|
374
|
+
const id = transport[MessagingHeaders.Id] ?? message.MessageId ?? receiptHandle
|
|
375
|
+
const groupId = system.MessageGroupId
|
|
376
|
+
|
|
377
|
+
subscription.inflight++
|
|
378
|
+
let lease: ReturnType<typeof setInterval> | undefined
|
|
379
|
+
/** Lease extension in flight (one at a time). */
|
|
380
|
+
let extending: Promise<void> | undefined
|
|
381
|
+
const extendEvery = Math.floor((subscription.visibilityTimeout * 1000) / 2)
|
|
382
|
+
if ((this.#options.extendVisibility ?? true) && extendEvery >= 1000) {
|
|
383
|
+
lease = setInterval(() => {
|
|
384
|
+
if (extending !== undefined) return
|
|
385
|
+
extending = client
|
|
386
|
+
.send(
|
|
387
|
+
new ChangeMessageVisibilityCommand({
|
|
388
|
+
QueueUrl: queue.url,
|
|
389
|
+
ReceiptHandle: receiptHandle,
|
|
390
|
+
VisibilityTimeout: subscription.visibilityTimeout,
|
|
391
|
+
}),
|
|
392
|
+
)
|
|
393
|
+
.then(
|
|
394
|
+
() => undefined,
|
|
395
|
+
(error: unknown) => {
|
|
396
|
+
this.#logger.warn(`SQS: failed to extend the visibility of message ${id}`, { err: error })
|
|
397
|
+
},
|
|
398
|
+
)
|
|
399
|
+
.finally(() => {
|
|
400
|
+
extending = undefined
|
|
401
|
+
})
|
|
402
|
+
}, extendEvery)
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Stops extending the lease before a settlement, once the extension in flight landed: otherwise it
|
|
406
|
+
* could reach SQS after the settlement and override the visibility it set (retry delay, release).
|
|
407
|
+
* When the settlement fails, the runtime releases the message (or its visibility timeout expires).
|
|
408
|
+
*/
|
|
409
|
+
const stopLease = async (): Promise<void> => {
|
|
410
|
+
if (lease !== undefined) clearInterval(lease)
|
|
411
|
+
lease = undefined
|
|
412
|
+
await extending
|
|
413
|
+
}
|
|
414
|
+
let settled = false
|
|
415
|
+
const check = (action: string): void => {
|
|
416
|
+
if (settled) throw new Error(`[sqs] message ${id} was already settled (${action})`)
|
|
417
|
+
}
|
|
418
|
+
const done = (): void => {
|
|
419
|
+
settled = true
|
|
420
|
+
}
|
|
421
|
+
const changeVisibility = async (seconds: number): Promise<void> => {
|
|
422
|
+
await client.send(
|
|
423
|
+
new ChangeMessageVisibilityCommand({
|
|
424
|
+
QueueUrl: queue.url,
|
|
425
|
+
ReceiptHandle: receiptHandle,
|
|
426
|
+
VisibilityTimeout: seconds,
|
|
427
|
+
}),
|
|
428
|
+
)
|
|
429
|
+
}
|
|
430
|
+
const remove = async (): Promise<void> => {
|
|
431
|
+
await client.send(new DeleteMessageCommand({ QueueUrl: queue.url, ReceiptHandle: receiptHandle }))
|
|
432
|
+
}
|
|
433
|
+
const delivery: DriverDelivery = {
|
|
434
|
+
id,
|
|
435
|
+
body,
|
|
436
|
+
contentType: transport[MessagingHeaders.ContentType],
|
|
437
|
+
headers,
|
|
438
|
+
type: transport[MessagingHeaders.Type],
|
|
439
|
+
key: groupId,
|
|
440
|
+
attempt: Number.isSafeInteger(receiveCount) && receiveCount > 0 ? receiveCount : 1,
|
|
441
|
+
redelivered: receiveCount > 1,
|
|
442
|
+
timestamp: Number.isFinite(sentAt) ? sentAt : undefined,
|
|
443
|
+
raw: message,
|
|
444
|
+
ack: async (): Promise<void> => {
|
|
445
|
+
check('ack')
|
|
446
|
+
await stopLease()
|
|
447
|
+
await remove()
|
|
448
|
+
done()
|
|
449
|
+
},
|
|
450
|
+
release: async (): Promise<void> => {
|
|
451
|
+
check('release')
|
|
452
|
+
await stopLease()
|
|
453
|
+
await changeVisibility(0)
|
|
454
|
+
done()
|
|
455
|
+
},
|
|
456
|
+
retry: async (request: RetryRequest): Promise<void> => {
|
|
457
|
+
check('retry')
|
|
458
|
+
await stopLease()
|
|
459
|
+
await changeVisibility(Math.min(MAX_VISIBILITY_SECONDS, Math.ceil(request.delayMs / 1000)))
|
|
460
|
+
done()
|
|
461
|
+
},
|
|
462
|
+
deadLetter: async (request: DeadLetterRequest): Promise<void> => {
|
|
463
|
+
check('dead-letter')
|
|
464
|
+
await stopLease()
|
|
465
|
+
const target =
|
|
466
|
+
request.queue === undefined ? undefined : (subscription.deadLetter ?? (await this.#queue(request.queue)))
|
|
467
|
+
if (target !== undefined) {
|
|
468
|
+
await client.send(
|
|
469
|
+
new SendMessageCommand(
|
|
470
|
+
this.#deadLetterInput(target, queue, encoded, transport, headers, request, groupId, id),
|
|
471
|
+
),
|
|
472
|
+
)
|
|
473
|
+
}
|
|
474
|
+
await remove()
|
|
475
|
+
done()
|
|
476
|
+
},
|
|
477
|
+
}
|
|
478
|
+
// FIFO queues: messages of a group are handled one after the other, in order
|
|
479
|
+
const serialKey = queue.fifo ? groupId : undefined
|
|
480
|
+
const previous = serialKey === undefined ? undefined : subscription.groups.get(serialKey)
|
|
481
|
+
const run = (): Promise<void> => subscription.handler(delivery)
|
|
482
|
+
const handled = (previous === undefined ? run() : previous.then(run, run))
|
|
483
|
+
.catch((error: unknown) => {
|
|
484
|
+
this.#logger.error(`SQS: unexpected failure handling message ${id}`, { err: error })
|
|
485
|
+
})
|
|
486
|
+
.finally(() => {
|
|
487
|
+
if (lease !== undefined) clearInterval(lease)
|
|
488
|
+
subscription.inflight--
|
|
489
|
+
if (serialKey !== undefined && subscription.groups.get(serialKey) === handled)
|
|
490
|
+
subscription.groups.delete(serialKey)
|
|
491
|
+
subscription.wake?.()
|
|
492
|
+
})
|
|
493
|
+
if (serialKey !== undefined) subscription.groups.set(serialKey, handled)
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
#deadLetterInput(
|
|
497
|
+
target: QueueInfo,
|
|
498
|
+
source: QueueInfo,
|
|
499
|
+
body: string,
|
|
500
|
+
transport: Readonly<Record<string, string>>,
|
|
501
|
+
headers: Readonly<Record<string, string>>,
|
|
502
|
+
request: DeadLetterRequest,
|
|
503
|
+
groupId: string | undefined,
|
|
504
|
+
id: string,
|
|
505
|
+
): SendMessageCommandInput {
|
|
506
|
+
const attributes = headerRecord()
|
|
507
|
+
Object.assign(attributes, transport)
|
|
508
|
+
attributes[MessagingHeaders.DeadLetterReason] = request.reason
|
|
509
|
+
attributes[MessagingHeaders.Error] =
|
|
510
|
+
truncate(errorMessage(request.error), MAX_ERROR_HEADER_LENGTH) || 'unknown error'
|
|
511
|
+
attributes[MessagingHeaders.OriginalQueue] = source.name
|
|
512
|
+
attributes[MessagingHeaders.Attempts] = String(request.attempt)
|
|
513
|
+
const dropped: string[] = []
|
|
514
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
515
|
+
if (Object.keys(attributes).length >= MAX_ATTRIBUTES) dropped.push(name)
|
|
516
|
+
else attributes[name] = value
|
|
517
|
+
}
|
|
518
|
+
if (dropped.length > 0) {
|
|
519
|
+
this.#logger.warn(
|
|
520
|
+
`SQS: dead-letter copy of message ${id} exceeds ${MAX_ATTRIBUTES} attributes, dropped: ${dropped.join(', ')}`,
|
|
521
|
+
)
|
|
522
|
+
}
|
|
523
|
+
const input: SendMessageCommandInput = {
|
|
524
|
+
QueueUrl: target.url,
|
|
525
|
+
MessageBody: body,
|
|
526
|
+
MessageAttributes: toAttributes(attributes, target.name),
|
|
527
|
+
}
|
|
528
|
+
if (target.fifo) {
|
|
529
|
+
input.MessageGroupId = groupId ?? 'default'
|
|
530
|
+
input.MessageDeduplicationId = `${id}-dead-letter`.slice(0, 128)
|
|
531
|
+
}
|
|
532
|
+
return input
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/** Queue name of a name, URL or ARN (last segment). */
|
|
537
|
+
function queueName(queue: string): string {
|
|
538
|
+
const segments = queue.split(/[/:]/)
|
|
539
|
+
return segments[segments.length - 1] ?? queue
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function encodeBody(body: Uint8Array | string): string {
|
|
543
|
+
return typeof body === 'string' ? body : Buffer.from(bodyToBytes(body)).toString('base64')
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function toAttributes(
|
|
547
|
+
attributes: Readonly<Record<string, string>>,
|
|
548
|
+
queue: string,
|
|
549
|
+
): Record<string, MessageAttributeValue> {
|
|
550
|
+
const entries = Object.entries(attributes)
|
|
551
|
+
if (entries.length > MAX_ATTRIBUTES) {
|
|
552
|
+
throw new RangeError(
|
|
553
|
+
`[znight] SQS messages carry at most ${MAX_ATTRIBUTES} attributes (Znight uses up to 4): too many headers for "${queue}"`,
|
|
554
|
+
)
|
|
555
|
+
}
|
|
556
|
+
const result: Record<string, MessageAttributeValue> = {}
|
|
557
|
+
for (const [name, value] of entries) {
|
|
558
|
+
if (!ATTRIBUTE_NAME.test(name)) throw new TypeError(`[znight] "${name}" is not a valid SQS message attribute name`)
|
|
559
|
+
if (value === '') throw new TypeError(`[znight] SQS message attributes cannot be empty (header "${name}")`)
|
|
560
|
+
result[name] = { DataType: 'String', StringValue: value }
|
|
561
|
+
}
|
|
562
|
+
return result
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function attributeToString(attribute: MessageAttributeValue): string | undefined {
|
|
566
|
+
if (attribute.StringValue !== undefined) return attribute.StringValue
|
|
567
|
+
if (attribute.BinaryValue !== undefined) return Buffer.from(attribute.BinaryValue).toString('base64')
|
|
568
|
+
return undefined
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
function parseRedrivePolicy(
|
|
572
|
+
value: string | undefined,
|
|
573
|
+
): { deadLetterTargetArn: string; maxReceiveCount: number } | undefined {
|
|
574
|
+
if (value === undefined || value === '') return undefined
|
|
575
|
+
try {
|
|
576
|
+
const parsed = JSON.parse(value) as { deadLetterTargetArn?: unknown; maxReceiveCount?: unknown }
|
|
577
|
+
const maxReceiveCount = Number(parsed.maxReceiveCount)
|
|
578
|
+
if (typeof parsed.deadLetterTargetArn !== 'string' || !Number.isFinite(maxReceiveCount)) return undefined
|
|
579
|
+
return { deadLetterTargetArn: parsed.deadLetterTargetArn, maxReceiveCount }
|
|
580
|
+
} catch {
|
|
581
|
+
return undefined
|
|
582
|
+
}
|
|
583
|
+
}
|