@othree.io/stethoscope 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitlab-ci.yml +3 -3
- package/README.md +316 -1
- package/lib/cjs/aws.d.ts +50 -0
- package/lib/cjs/aws.d.ts.map +1 -0
- package/lib/cjs/aws.js +160 -0
- package/lib/cjs/aws.js.map +1 -0
- package/lib/cjs/index.d.ts +20 -0
- package/lib/cjs/index.d.ts.map +1 -0
- package/lib/cjs/index.js +58 -0
- package/lib/cjs/index.js.map +1 -0
- package/lib/cjs/results-processor.d.ts +20 -0
- package/lib/cjs/results-processor.d.ts.map +1 -0
- package/lib/cjs/results-processor.js +51 -0
- package/lib/cjs/results-processor.js.map +1 -0
- package/lib/esm/aws.js +151 -0
- package/lib/esm/aws.js.map +1 -0
- package/lib/esm/index.js +40 -0
- package/lib/esm/index.js.map +1 -0
- package/lib/esm/results-processor.js +47 -0
- package/lib/esm/results-processor.js.map +1 -0
- package/package.json +23 -16
- package/lib/aws.d.ts +0 -107
- package/lib/aws.d.ts.map +0 -1
- package/lib/aws.js +0 -195
- package/lib/aws.js.map +0 -1
- package/lib/index.d.ts +0 -21
- package/lib/index.d.ts.map +0 -1
- package/lib/index.js +0 -54
- package/lib/index.js.map +0 -1
- package/lib/results-processor.d.ts +0 -43
- package/lib/results-processor.d.ts.map +0 -1
- package/lib/results-processor.js +0 -71
- package/lib/results-processor.js.map +0 -1
package/.gitlab-ci.yml
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
image: node:
|
|
1
|
+
image: node:20
|
|
2
2
|
|
|
3
3
|
cache:
|
|
4
4
|
key: ${CI_PIPELINE_ID}
|
|
@@ -17,13 +17,13 @@ test:
|
|
|
17
17
|
- npm test
|
|
18
18
|
|
|
19
19
|
deploy:
|
|
20
|
-
image: node:
|
|
20
|
+
image: node:22
|
|
21
21
|
stage: deploy
|
|
22
22
|
variables:
|
|
23
23
|
NPM_CONFIG_USERCONFIG: '$CI_PROJECT_DIR/.npmrc'
|
|
24
24
|
script:
|
|
25
25
|
- cat $NPM > .npmrc
|
|
26
|
-
- npx semantic-release
|
|
26
|
+
- npx semantic-release@25
|
|
27
27
|
only:
|
|
28
28
|
refs:
|
|
29
29
|
- main
|
package/README.md
CHANGED
|
@@ -1 +1,316 @@
|
|
|
1
|
-
# stethoscope
|
|
1
|
+
# @othree.io/stethoscope
|
|
2
|
+
|
|
3
|
+
Structured SQS message processing for AWS Lambda handlers with error classification, exponential backoff, automatic DLQ routing, FIFO ordering support, and partial batch failure reporting.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @othree.io/stethoscope
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
### Peer Dependencies
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"@othree.io/awsome": "^4.0.0",
|
|
16
|
+
"@othree.io/journal": "^2.0.0",
|
|
17
|
+
"@othree.io/optional": "^2.3.3",
|
|
18
|
+
"@aws-sdk/client-sqs": "^3.x"
|
|
19
|
+
}
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Quick Start
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
import { processSqs } from '@othree.io/stethoscope'
|
|
26
|
+
import { SQSClient } from '@aws-sdk/client-sqs'
|
|
27
|
+
import { TryAsync } from '@othree.io/optional'
|
|
28
|
+
import { SQSRecord } from 'aws-lambda'
|
|
29
|
+
|
|
30
|
+
type OrderInput = Readonly<{
|
|
31
|
+
orderId: string
|
|
32
|
+
amount: number
|
|
33
|
+
}>
|
|
34
|
+
|
|
35
|
+
type OrderResult = Readonly<{
|
|
36
|
+
orderId: string
|
|
37
|
+
status: string
|
|
38
|
+
}>
|
|
39
|
+
|
|
40
|
+
const sqsClient = new SQSClient({})
|
|
41
|
+
|
|
42
|
+
export const handler = processSqs<OrderInput, OrderResult>({
|
|
43
|
+
sqsClient,
|
|
44
|
+
configuration: {
|
|
45
|
+
dlqUrl: process.env.DLQ_URL!,
|
|
46
|
+
exponentialTimeouts: [5, 10, 30],
|
|
47
|
+
},
|
|
48
|
+
didItFail: (error) => error.name !== 'RetryableError',
|
|
49
|
+
runProcess: async (input) => {
|
|
50
|
+
return TryAsync(async () => {
|
|
51
|
+
const result = await processOrder(input)
|
|
52
|
+
return result
|
|
53
|
+
})
|
|
54
|
+
},
|
|
55
|
+
getProcessInput: async (record) => {
|
|
56
|
+
return TryAsync(async () => JSON.parse(record.body))
|
|
57
|
+
},
|
|
58
|
+
})
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## API
|
|
62
|
+
|
|
63
|
+
### `processSqs<Input, Output>(deps)`
|
|
64
|
+
|
|
65
|
+
The main entry point. Wires up the full SQS processing pipeline and returns a Lambda handler function `(event: SQSEvent) => Promise<SQSBatchResponse>`.
|
|
66
|
+
|
|
67
|
+
```typescript
|
|
68
|
+
export const processSqs = <Input, Output>(deps: {
|
|
69
|
+
log?: Log
|
|
70
|
+
sqsClient: SQSClient
|
|
71
|
+
configuration: SQSProcessConfiguration
|
|
72
|
+
didItFail: (error: Error) => boolean
|
|
73
|
+
runProcess: (input: Input) => Promise<Optional<Output>>
|
|
74
|
+
getProcessInput: (record: SQSRecord) => Promise<Optional<Input>>
|
|
75
|
+
}) => (event: SQSEvent) => Promise<SQSBatchResponse>
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
| Parameter | Description |
|
|
79
|
+
|-----------|-------------|
|
|
80
|
+
| `log` | Optional logging wrapper from `@othree.io/journal` (`logIOAsync`) |
|
|
81
|
+
| `sqsClient` | AWS SDK v3 `SQSClient` instance |
|
|
82
|
+
| `configuration` | DLQ URL and exponential backoff timeouts |
|
|
83
|
+
| `didItFail` | Error classifier: return `true` for fatal errors (DLQ), `false` for retriable errors (retry with backoff) |
|
|
84
|
+
| `runProcess` | Business logic handler. Return `Optional` with a value for success, `Empty()` for no-op, or throw for error classification |
|
|
85
|
+
| `getProcessInput` | Extracts typed input from the raw `SQSRecord`. Return `Empty()` to mark the message as a failure. The original error from the Optional is preserved in the failure details |
|
|
86
|
+
|
|
87
|
+
### `SQSProcessConfiguration`
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
type SQSProcessConfiguration = Readonly<{
|
|
91
|
+
dlqUrl: string
|
|
92
|
+
exponentialTimeouts: Array<number>
|
|
93
|
+
}>
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
- `dlqUrl` - The URL of the application-level Dead Letter Queue
|
|
97
|
+
- `exponentialTimeouts` - Visibility timeout values (in seconds) indexed by receive count
|
|
98
|
+
|
|
99
|
+
## Result Actions
|
|
100
|
+
|
|
101
|
+
Every processed message is classified into one of four outcomes:
|
|
102
|
+
|
|
103
|
+
| Action | Description | Behavior |
|
|
104
|
+
|--------|-------------|----------|
|
|
105
|
+
| `success` | `runProcess` returned a value | Message acknowledged (removed from queue) |
|
|
106
|
+
| `none` | `runProcess` returned `Empty()` without an error | Message acknowledged |
|
|
107
|
+
| `retry` | `didItFail` returned `false` for the error | Message returned to queue with exponential backoff |
|
|
108
|
+
| `failure` | `didItFail` returned `true`, or `getProcessInput` returned `Empty()` | Error details sent to DLQ, message acknowledged |
|
|
109
|
+
|
|
110
|
+
## Error Handling Flow
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
1. Message received from SQS
|
|
114
|
+
2. getProcessInput extracts typed input
|
|
115
|
+
- Empty Optional -> failure (original error preserved, sent to DLQ)
|
|
116
|
+
- Valid input -> continue
|
|
117
|
+
3. FIFO group check (if MessageGroupId is set)
|
|
118
|
+
- If a previous message in the same group is retrying
|
|
119
|
+
-> cascade retry (skip processing, return to queue)
|
|
120
|
+
4. runProcess executes business logic
|
|
121
|
+
- Returns value -> success (acknowledged)
|
|
122
|
+
- Returns Empty() -> none (acknowledged)
|
|
123
|
+
- Throws error -> continue to classification
|
|
124
|
+
5. didItFail classifies the error
|
|
125
|
+
- Returns true -> failure (sent to DLQ)
|
|
126
|
+
- Returns false -> retry (back to queue with backoff)
|
|
127
|
+
6. SQSBatchResponse returned to Lambda runtime
|
|
128
|
+
- Only retry messages included in batchItemFailures (using messageId)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Exponential Backoff
|
|
132
|
+
|
|
133
|
+
The `exponentialTimeouts` array controls the SQS visibility timeout progression. The timeout is selected using the message's `ApproximateReceiveCount` attribute:
|
|
134
|
+
|
|
135
|
+
```typescript
|
|
136
|
+
exponentialTimeouts: [5, 10, 30]
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
| Receive Count | Visibility Timeout |
|
|
140
|
+
|---------------|-------------------|
|
|
141
|
+
| 1 | 5 seconds |
|
|
142
|
+
| 2 | 10 seconds |
|
|
143
|
+
| 3+ | 30 seconds (clamped to last value) |
|
|
144
|
+
|
|
145
|
+
## FIFO Queue Support
|
|
146
|
+
|
|
147
|
+
Stethoscope groups messages by `MessageGroupId` and processes them sequentially within each group:
|
|
148
|
+
|
|
149
|
+
- Messages without a `MessageGroupId` are treated as ungrouped
|
|
150
|
+
- Within a group, if a message is classified as `retry`, all subsequent messages in that group are also retried
|
|
151
|
+
- Fatal failures (`failure`) do not cascade — since the message is sent to the DLQ and won't be retried, processing continues for the remaining messages in the group
|
|
152
|
+
- This preserves message ordering guarantees required by FIFO queues
|
|
153
|
+
|
|
154
|
+
## Using with Logging
|
|
155
|
+
|
|
156
|
+
Pass `logIOAsync` from `@othree.io/journal` to automatically log inputs, outputs, and errors for each processing step:
|
|
157
|
+
|
|
158
|
+
```typescript
|
|
159
|
+
import { processSqs } from '@othree.io/stethoscope'
|
|
160
|
+
import { logIOAsync } from '@othree.io/journal'
|
|
161
|
+
|
|
162
|
+
const log = logIOAsync({
|
|
163
|
+
now: () => Date.now(),
|
|
164
|
+
log: logger.info.bind(logger),
|
|
165
|
+
logError: logger.error.bind(logger),
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
export const handler = processSqs<MyInput, MyOutput>({
|
|
169
|
+
log,
|
|
170
|
+
sqsClient,
|
|
171
|
+
configuration,
|
|
172
|
+
didItFail,
|
|
173
|
+
runProcess,
|
|
174
|
+
getProcessInput,
|
|
175
|
+
})
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Common Patterns
|
|
179
|
+
|
|
180
|
+
### Classifying Errors
|
|
181
|
+
|
|
182
|
+
```typescript
|
|
183
|
+
const didItFail = (error: Error): boolean => {
|
|
184
|
+
const retryableErrors = ['TimeoutError', 'ServiceUnavailableError', 'ThrottlingError']
|
|
185
|
+
return !retryableErrors.includes(error.name)
|
|
186
|
+
}
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Parsing SNS-wrapped Messages
|
|
190
|
+
|
|
191
|
+
When SQS is subscribed to an SNS topic, the record body contains an SNS envelope:
|
|
192
|
+
|
|
193
|
+
```typescript
|
|
194
|
+
const getProcessInput = async (record: SQSRecord): Promise<Optional<MyInput>> => {
|
|
195
|
+
return TryAsync(async () => {
|
|
196
|
+
const snsMessage = JSON.parse(record.body)
|
|
197
|
+
return JSON.parse(snsMessage.Message)
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## Infrastructure Requirements
|
|
203
|
+
|
|
204
|
+
Stethoscope handles message processing logic, but the AWS infrastructure must be configured correctly for retry and failure handling to work.
|
|
205
|
+
|
|
206
|
+
### Architecture
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
+-----------------+
|
|
210
|
+
| SNS Topic |
|
|
211
|
+
| (event source) |
|
|
212
|
+
+--------+--------+
|
|
213
|
+
|
|
|
214
|
+
v
|
|
215
|
+
+------------------+ +---------------------+
|
|
216
|
+
| Redrive DLQ | <---------- | Events Queue |
|
|
217
|
+
| (max receive | redrive | (main processing) |
|
|
218
|
+
| count exceeded) | policy | |
|
|
219
|
+
+------------------+ +----------+----------+
|
|
220
|
+
|
|
|
221
|
+
| SQS Event Source
|
|
222
|
+
v
|
|
223
|
+
+---------------------+
|
|
224
|
+
| Lambda Handler |
|
|
225
|
+
| (stethoscope) |
|
|
226
|
+
+----------+----------+
|
|
227
|
+
|
|
|
228
|
+
| explicit send on failure
|
|
229
|
+
v
|
|
230
|
+
+---------------------+
|
|
231
|
+
| Application DLQ |
|
|
232
|
+
| (business failures) |
|
|
233
|
+
+---------------------+
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### Two DLQ Mechanisms
|
|
237
|
+
|
|
238
|
+
You need **two separate dead letter queues**:
|
|
239
|
+
|
|
240
|
+
1. **Application DLQ** - For messages that stethoscope explicitly marks as failures
|
|
241
|
+
- Receives messages when `didItFail` returns `true` or when `getProcessInput` returns `Empty()`
|
|
242
|
+
- Contains the original SQS record and serialized error details
|
|
243
|
+
- Lambda must have `sqs:SendMessage` permission to this queue
|
|
244
|
+
|
|
245
|
+
2. **Redrive DLQ** - For messages that exceed the queue's max receive count
|
|
246
|
+
- Configured via the events queue's redrive policy
|
|
247
|
+
- Catches messages that keep timing out or causing Lambda crashes
|
|
248
|
+
- Acts as a safety net for infrastructure-level failures
|
|
249
|
+
|
|
250
|
+
### CDK Example
|
|
251
|
+
|
|
252
|
+
```typescript
|
|
253
|
+
import { Queue } from 'aws-cdk-lib/aws-sqs'
|
|
254
|
+
import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'
|
|
255
|
+
import { Duration } from 'aws-cdk-lib'
|
|
256
|
+
|
|
257
|
+
// Application-level DLQ (stethoscope sends failures here)
|
|
258
|
+
const dlq = new Queue(this, 'DLQ', {
|
|
259
|
+
queueName: 'my-service-dlq',
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
// Infrastructure-level DLQ (max receive count exceeded)
|
|
263
|
+
const redriveDlq = new Queue(this, 'RedriveDLQ', {
|
|
264
|
+
queueName: 'my-service-redrive-dlq',
|
|
265
|
+
})
|
|
266
|
+
|
|
267
|
+
// Main processing queue with redrive policy
|
|
268
|
+
const eventsQueue = new Queue(this, 'EventsQueue', {
|
|
269
|
+
queueName: 'my-service-events',
|
|
270
|
+
visibilityTimeout: Duration.seconds(30),
|
|
271
|
+
deadLetterQueue: {
|
|
272
|
+
queue: redriveDlq,
|
|
273
|
+
maxReceiveCount: 4,
|
|
274
|
+
},
|
|
275
|
+
})
|
|
276
|
+
|
|
277
|
+
// Lambda handler
|
|
278
|
+
const handler = new Function(this, 'Handler', {
|
|
279
|
+
// ... function configuration
|
|
280
|
+
environment: {
|
|
281
|
+
DLQ_URL: dlq.queueUrl,
|
|
282
|
+
},
|
|
283
|
+
})
|
|
284
|
+
|
|
285
|
+
// Grant permissions
|
|
286
|
+
dlq.grantSendMessages(handler)
|
|
287
|
+
eventsQueue.grantConsumeMessages(handler)
|
|
288
|
+
|
|
289
|
+
// Configure SQS event source
|
|
290
|
+
handler.addEventSource(new SqsEventSource(eventsQueue, {
|
|
291
|
+
batchSize: 10,
|
|
292
|
+
maxConcurrency: 10,
|
|
293
|
+
reportBatchItemFailures: true,
|
|
294
|
+
enabled: true,
|
|
295
|
+
}))
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
### Critical: `reportBatchItemFailures`
|
|
299
|
+
|
|
300
|
+
The SQS event source mapping **must** have `reportBatchItemFailures: true`. This setting enables partial batch failure reporting via `SQSBatchResponse`. Without it, any single failure causes the entire batch to be retried.
|
|
301
|
+
|
|
302
|
+
### IAM Permissions
|
|
303
|
+
|
|
304
|
+
The Lambda execution role needs:
|
|
305
|
+
|
|
306
|
+
- `sqs:SendMessage` on the application DLQ (for sending failure details)
|
|
307
|
+
- `sqs:ChangeMessageVisibility` on the events queue (for exponential backoff). This is included in `grantConsumeMessages`
|
|
308
|
+
|
|
309
|
+
## Peer Dependencies
|
|
310
|
+
|
|
311
|
+
| Package | Purpose |
|
|
312
|
+
|---------|---------|
|
|
313
|
+
| `@othree.io/awsome` | SQS operations (send to DLQ, change message visibility, queue URL resolution) |
|
|
314
|
+
| `@othree.io/journal` | Optional structured logging for processing steps |
|
|
315
|
+
| `@othree.io/optional` | `Optional<T>` monad for safe error handling and result classification |
|
|
316
|
+
| `@aws-sdk/client-sqs` | AWS SQS client |
|
package/lib/cjs/aws.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda';
|
|
2
|
+
import { Optional } from '@othree.io/optional';
|
|
3
|
+
import { GetProcessResults, ProcessResultAction, ProcessResults } from './results-processor';
|
|
4
|
+
import { sqs } from '@othree.io/awsome';
|
|
5
|
+
export declare const NoGroupSymbol: unique symbol;
|
|
6
|
+
export type RunProcess<Input, Output> = (input: Input) => Promise<Optional<Output>>;
|
|
7
|
+
export type Key = string | symbol;
|
|
8
|
+
export type MessageResult<Input, Output> = Readonly<{
|
|
9
|
+
message: SQSRecord;
|
|
10
|
+
result: ProcessResults<Input, Output>;
|
|
11
|
+
}>;
|
|
12
|
+
export type Results<Input, Output> = Readonly<{
|
|
13
|
+
[key in ProcessResultAction]: Array<MessageResult<Input, Output>>;
|
|
14
|
+
}>;
|
|
15
|
+
export type GroupResults<Input, Output> = Readonly<{
|
|
16
|
+
group: Key;
|
|
17
|
+
results: Results<Input, Output>;
|
|
18
|
+
}>;
|
|
19
|
+
export type GroupedRecords = Record<Key, Array<SQSRecord>>;
|
|
20
|
+
export declare const groupRecords: (records: Array<SQSRecord>) => GroupedRecords;
|
|
21
|
+
export declare const processSQSEvent: <Input, Output>(deps: {
|
|
22
|
+
process: RunProcess<Input, Output>;
|
|
23
|
+
groupRecords: (records: Array<SQSRecord>) => GroupedRecords;
|
|
24
|
+
getProcessInput: RunProcess<SQSRecord, Input>;
|
|
25
|
+
getProcessResults: GetProcessResults<Input, Output>;
|
|
26
|
+
}) => (event: SQSEvent) => Promise<Array<GroupResults<Input, Output>>>;
|
|
27
|
+
export declare const handleExponentialBackoff: <Input, Output>(deps: {
|
|
28
|
+
changeMessageVisibility: (request: sqs.ChangeMessageVisibilityTimeoutRequest) => Promise<Optional<true>>;
|
|
29
|
+
getQueueUrl: (input: sqs.GetQueueUrlInput) => Optional<sqs.QueueUrlOutput>;
|
|
30
|
+
timeouts: Array<number>;
|
|
31
|
+
}) => (groupResults: Array<GroupResults<Input, Output>>) => Promise<Array<GroupResults<Input, Output>>>;
|
|
32
|
+
export type ErrorMessage<Input> = Readonly<{
|
|
33
|
+
request: Input;
|
|
34
|
+
error: Readonly<{
|
|
35
|
+
name: string;
|
|
36
|
+
message: string;
|
|
37
|
+
stack?: string;
|
|
38
|
+
}>;
|
|
39
|
+
}>;
|
|
40
|
+
export declare const handleErrors: <Input, Output>(deps: {
|
|
41
|
+
notifyError: (payload: ErrorMessage<SQSRecord>) => Promise<Optional<boolean>>;
|
|
42
|
+
}) => (groupResults: Array<GroupResults<Input, Output>>) => Promise<Array<GroupResults<Input, Output>>>;
|
|
43
|
+
export declare const toSQSBatchResponse: <Input, Output>(groupResults: Array<GroupResults<Input, Output>>) => Promise<SQSBatchResponse>;
|
|
44
|
+
export declare const processSQSEventAndHandleResults: <Input, Output>(deps: {
|
|
45
|
+
processSQSEvent: (event: SQSEvent) => Promise<Array<GroupResults<Input, Output>>>;
|
|
46
|
+
handleExponentialBackoff: (groupResults: Array<GroupResults<Input, Output>>) => Promise<Array<GroupResults<Input, Output>>>;
|
|
47
|
+
handleErrors: (groupResults: Array<GroupResults<Input, Output>>) => Promise<Array<GroupResults<Input, Output>>>;
|
|
48
|
+
toSQSBatchResponse: (groupResults: Array<GroupResults<Input, Output>>) => Promise<SQSBatchResponse>;
|
|
49
|
+
}) => (event: SQSEvent) => Promise<SQSBatchResponse>;
|
|
50
|
+
//# sourceMappingURL=aws.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aws.d.ts","sourceRoot":"","sources":["../../src/aws.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAE,QAAQ,EAAE,SAAS,EAAC,MAAM,YAAY,CAAA;AAChE,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EACH,iBAAiB,EAEjB,mBAAmB,EACnB,cAAc,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAC,GAAG,EAAC,MAAM,mBAAmB,CAAA;AAErC,eAAO,MAAM,aAAa,EAAE,OAAO,MAAgC,CAAA;AAInE,MAAM,MAAM,UAAU,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;AAEnF,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAA;AAEjC,MAAM,MAAM,aAAa,CAAC,KAAK,EAAE,MAAM,IAAI,QAAQ,CAAC;IAChD,OAAO,EAAE,SAAS,CAAA;IAClB,MAAM,EAAE,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;CACxC,CAAC,CAAA;AAEF,MAAM,MAAM,OAAO,CAAC,KAAK,EAAE,MAAM,IAAI,QAAQ,CAAC;KACzC,GAAG,IAAI,mBAAmB,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;CACpE,CAAC,CAAA;AAEF,MAAM,MAAM,YAAY,CAAC,KAAK,EAAE,MAAM,IAAI,QAAQ,CAAC;IAC/C,KAAK,EAAE,GAAG,CAAA;IACV,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;CAClC,CAAC,CAAA;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAA;AAkB1D,eAAO,MAAM,YAAY,GAAI,SAAS,KAAK,CAAC,SAAS,CAAC,KAAG,cAexD,CAAA;AAYD,eAAO,MAAM,eAAe,GAAI,KAAK,EAAE,MAAM,EAAE,MAAM;IACjD,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAClC,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,cAAc,CAAA;IAC3D,eAAe,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;IAC7C,iBAAiB,EAAE,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;CACtD,MACU,OAAO,QAAQ,KAAG,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CA4ElE,CAAA;AAEL,eAAO,MAAM,wBAAwB,GAAI,KAAK,EAAE,MAAM,EAAE,MAAM;IAC1D,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,qCAAqC,KAAK,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACxG,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,gBAAgB,KAAK,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAC1E,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CAC1B,MACU,cAAc,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAG,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAqBnG,CAAA;AAEL,MAAM,MAAM,YAAY,CAAC,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO,EAAE,KAAK,CAAA;IACd,KAAK,EAAE,QAAQ,CAAC;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,OAAO,EAAE,MAAM,CAAA;QACf,KAAK,CAAC,EAAE,MAAM,CAAA;KACjB,CAAC,CAAA;CACL,CAAC,CAAA;AAEF,eAAO,MAAM,YAAY,GAAI,KAAK,EAAE,MAAM,EAAE,MAAM;IAC9C,WAAW,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;CAChF,MACU,cAAc,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAG,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAenG,CAAA;AAEL,eAAO,MAAM,kBAAkB,GAAU,KAAK,EAAE,MAAM,EAAE,cAAc,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAG,OAAO,CAAC,gBAAgB,CAUlI,CAAA;AAED,eAAO,MAAM,+BAA+B,GAAI,KAAK,EAAE,MAAM,EAAE,MAAM;IACjE,eAAe,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IACjF,wBAAwB,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAC3H,YAAY,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAC/G,kBAAkB,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;CACtG,MACU,OAAO,QAAQ,KAAG,OAAO,CAAC,gBAAgB,CAKhD,CAAA"}
|
package/lib/cjs/aws.js
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.processSQSEventAndHandleResults = exports.toSQSBatchResponse = exports.handleErrors = exports.handleExponentialBackoff = exports.processSQSEvent = exports.groupRecords = exports.NoGroupSymbol = void 0;
|
|
4
|
+
const optional_1 = require("@othree.io/optional");
|
|
5
|
+
const results_processor_1 = require("./results-processor");
|
|
6
|
+
exports.NoGroupSymbol = Symbol('NoGroupSymbol');
|
|
7
|
+
const LIMIT = 10;
|
|
8
|
+
const InitialValue = Object.freeze({});
|
|
9
|
+
const appendToResults = (accumulated, action, messageResult) => {
|
|
10
|
+
return {
|
|
11
|
+
none: accumulated.none,
|
|
12
|
+
success: accumulated.success,
|
|
13
|
+
failure: accumulated.failure,
|
|
14
|
+
retry: accumulated.retry,
|
|
15
|
+
[action]: [].concat(accumulated[action], messageResult),
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
const groupRecords = (records) => {
|
|
19
|
+
const groupedRecords = records.reduce(((acum, current) => {
|
|
20
|
+
const key = (0, optional_1.Optional)(current.attributes.MessageGroupId).orElse(exports.NoGroupSymbol);
|
|
21
|
+
const existing = (0, optional_1.Optional)(acum[key]);
|
|
22
|
+
const updated = existing
|
|
23
|
+
.map(_ => [].concat(_, current))
|
|
24
|
+
.orElse([current]);
|
|
25
|
+
const result = {};
|
|
26
|
+
Reflect.ownKeys(acum).forEach(k => { result[k] = acum[k]; });
|
|
27
|
+
result[key] = updated;
|
|
28
|
+
return result;
|
|
29
|
+
}), InitialValue);
|
|
30
|
+
return groupedRecords;
|
|
31
|
+
};
|
|
32
|
+
exports.groupRecords = groupRecords;
|
|
33
|
+
const processInBatches = async (items, limit, fn) => {
|
|
34
|
+
const results = [];
|
|
35
|
+
for (let i = 0; i < items.length; i += limit) {
|
|
36
|
+
const batch = items.slice(i, i + limit);
|
|
37
|
+
const batchResults = await Promise.all(batch.map(fn));
|
|
38
|
+
results.push(...batchResults);
|
|
39
|
+
}
|
|
40
|
+
return results;
|
|
41
|
+
};
|
|
42
|
+
const processSQSEvent = (deps) => async (event) => {
|
|
43
|
+
const groupedRecords = deps.groupRecords(event.Records);
|
|
44
|
+
const groupedResults = await processInBatches(Reflect.ownKeys(groupedRecords), LIMIT, async (key) => {
|
|
45
|
+
const values = groupedRecords[key];
|
|
46
|
+
let accumulated = {
|
|
47
|
+
[results_processor_1.ProcessResultAction.none]: [],
|
|
48
|
+
[results_processor_1.ProcessResultAction.failure]: [],
|
|
49
|
+
[results_processor_1.ProcessResultAction.success]: [],
|
|
50
|
+
[results_processor_1.ProcessResultAction.retry]: [],
|
|
51
|
+
};
|
|
52
|
+
for (const current of values) {
|
|
53
|
+
const maybeInput = await deps.getProcessInput(current);
|
|
54
|
+
if (maybeInput.isEmpty) {
|
|
55
|
+
const processResult = {
|
|
56
|
+
request: undefined,
|
|
57
|
+
action: results_processor_1.ProcessResultAction.failure,
|
|
58
|
+
result: undefined,
|
|
59
|
+
error: (0, optional_1.Optional)(maybeInput.getError()).orElse(new Error('Failed to get input')),
|
|
60
|
+
};
|
|
61
|
+
const messageResult = {
|
|
62
|
+
message: current,
|
|
63
|
+
result: processResult,
|
|
64
|
+
};
|
|
65
|
+
accumulated = appendToResults(accumulated, processResult.action, messageResult);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
const input = maybeInput.get();
|
|
69
|
+
/*
|
|
70
|
+
if it is an actual grouped message it means it's a fifo queue and
|
|
71
|
+
if there's already a message for retry (retry action) on a previous message
|
|
72
|
+
then next messages need to be retried.
|
|
73
|
+
Return retry so that the message is placed back into the queue
|
|
74
|
+
*/
|
|
75
|
+
if (key !== exports.NoGroupSymbol && accumulated.retry.length > 0) {
|
|
76
|
+
const messageResult = {
|
|
77
|
+
message: current,
|
|
78
|
+
result: {
|
|
79
|
+
request: maybeInput.orElse(undefined),
|
|
80
|
+
action: results_processor_1.ProcessResultAction.retry,
|
|
81
|
+
result: undefined,
|
|
82
|
+
error: undefined,
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
accumulated = appendToResults(accumulated, messageResult.result.action, messageResult);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const processResult = await deps.process(input)
|
|
89
|
+
.then(maybeResult => deps.getProcessResults(input, maybeResult));
|
|
90
|
+
const messageResult = {
|
|
91
|
+
message: current,
|
|
92
|
+
result: processResult,
|
|
93
|
+
};
|
|
94
|
+
accumulated = appendToResults(accumulated, processResult.action, messageResult);
|
|
95
|
+
}
|
|
96
|
+
const groupResults = {
|
|
97
|
+
group: key,
|
|
98
|
+
results: accumulated,
|
|
99
|
+
};
|
|
100
|
+
return groupResults;
|
|
101
|
+
});
|
|
102
|
+
return groupedResults;
|
|
103
|
+
};
|
|
104
|
+
exports.processSQSEvent = processSQSEvent;
|
|
105
|
+
const handleExponentialBackoff = (deps) => async (groupResults) => {
|
|
106
|
+
await Promise.allSettled(groupResults.map(async (results) => {
|
|
107
|
+
for (const messageResult of results.results.retry) {
|
|
108
|
+
const receiveCount = Number(messageResult.message.attributes.ApproximateReceiveCount);
|
|
109
|
+
const timeoutIndex = receiveCount - 1 >= deps.timeouts.length ? deps.timeouts.length - 1 : receiveCount - 1;
|
|
110
|
+
const visibilityTimeout = deps.timeouts[timeoutIndex];
|
|
111
|
+
await deps.getQueueUrl({
|
|
112
|
+
queueArn: messageResult.message.eventSourceARN
|
|
113
|
+
}).mapAsync(async (queueUrlOutput) => {
|
|
114
|
+
await deps.changeMessageVisibility({
|
|
115
|
+
receiptHandle: messageResult.message.receiptHandle,
|
|
116
|
+
queue: queueUrlOutput.queueUrl,
|
|
117
|
+
timeout: visibilityTimeout,
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
}));
|
|
122
|
+
return groupResults;
|
|
123
|
+
};
|
|
124
|
+
exports.handleExponentialBackoff = handleExponentialBackoff;
|
|
125
|
+
const handleErrors = (deps) => async (groupResults) => {
|
|
126
|
+
await Promise.allSettled(groupResults.map(async (results) => {
|
|
127
|
+
for (const messageResult of results.results.failure) {
|
|
128
|
+
await deps.notifyError({
|
|
129
|
+
request: messageResult.message,
|
|
130
|
+
error: {
|
|
131
|
+
message: messageResult.result.error.message,
|
|
132
|
+
name: messageResult.result.error.name,
|
|
133
|
+
stack: messageResult.result.error.stack,
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}));
|
|
138
|
+
return groupResults;
|
|
139
|
+
};
|
|
140
|
+
exports.handleErrors = handleErrors;
|
|
141
|
+
const toSQSBatchResponse = async (groupResults) => {
|
|
142
|
+
return {
|
|
143
|
+
batchItemFailures: groupResults.map(_ => _.results.retry)
|
|
144
|
+
.flat()
|
|
145
|
+
.map(_ => {
|
|
146
|
+
return {
|
|
147
|
+
itemIdentifier: _.message.messageId,
|
|
148
|
+
};
|
|
149
|
+
}),
|
|
150
|
+
};
|
|
151
|
+
};
|
|
152
|
+
exports.toSQSBatchResponse = toSQSBatchResponse;
|
|
153
|
+
const processSQSEventAndHandleResults = (deps) => async (event) => {
|
|
154
|
+
return deps.processSQSEvent(event)
|
|
155
|
+
.then(deps.handleErrors)
|
|
156
|
+
.then(deps.handleExponentialBackoff)
|
|
157
|
+
.then(deps.toSQSBatchResponse);
|
|
158
|
+
};
|
|
159
|
+
exports.processSQSEventAndHandleResults = processSQSEventAndHandleResults;
|
|
160
|
+
//# sourceMappingURL=aws.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aws.js","sourceRoot":"","sources":["../../src/aws.ts"],"names":[],"mappings":";;;AACA,kDAA4C;AAC5C,2DAK4B;AAGf,QAAA,aAAa,GAAkB,MAAM,CAAC,eAAe,CAAC,CAAA;AAEnE,MAAM,KAAK,GAAG,EAAE,CAAA;AAsBhB,MAAM,YAAY,GAAkC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AAErE,MAAM,eAAe,GAAG,CACpB,WAAmC,EACnC,MAA2B,EAC3B,aAA2C,EACrB,EAAE;IACxB,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,IAAI;QACtB,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,CAAC,MAAM,CAAC,EAAG,EAA0C,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;KACnG,CAAA;AACL,CAAC,CAAA;AAEM,MAAM,YAAY,GAAG,CAAC,OAAyB,EAAkB,EAAE;IACtE,MAAM,cAAc,GAAmB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;QACrE,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,qBAAa,CAAC,CAAA;QAClF,MAAM,QAAQ,GAAG,IAAA,mBAAQ,EAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACpC,MAAM,OAAO,GAAG,QAAQ;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,EAAuB,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;aACrD,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QACtB,MAAM,MAAM,GAAkC,EAAE,CAAA;QAChD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;QAC3D,MAAM,CAAC,GAAG,CAAC,GAAG,OAAO,CAAA;QAErB,OAAO,MAAM,CAAA;IACjB,CAAC,CAAC,EAAE,YAAY,CAAC,CAAA;IAEjB,OAAO,cAAc,CAAA;AACzB,CAAC,CAAA;AAfY,QAAA,YAAY,gBAexB;AAED,MAAM,gBAAgB,GAAG,KAAK,EAAQ,KAAe,EAAE,KAAa,EAAE,EAA2B,EAAqB,EAAE;IACpH,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAA;QACvC,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QACrD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAA;IACjC,CAAC;IACD,OAAO,OAAO,CAAA;AAClB,CAAC,CAAA;AAEM,MAAM,eAAe,GAAG,CAAgB,IAK9C,EAAE,EAAE,CACD,KAAK,EAAE,KAAe,EAA+C,EAAE;IAEnE,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAEvD,MAAM,cAAc,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;QACrG,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAA;QAElC,IAAI,WAAW,GAA2B;YACtC,CAAC,uCAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YAC9B,CAAC,uCAAmB,CAAC,OAAO,CAAC,EAAE,EAAE;YACjC,CAAC,uCAAmB,CAAC,OAAO,CAAC,EAAE,EAAE;YACjC,CAAC,uCAAmB,CAAC,KAAK,CAAC,EAAE,EAAE;SAClC,CAAA;QAED,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YAEtD,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,aAAa,GAA4C;oBAC3D,OAAO,EAAE,SAAU;oBACnB,MAAM,EAAE,uCAAmB,CAAC,OAAO;oBACnC,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,IAAA,mBAAQ,EAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;iBAClF,CAAA;gBAED,MAAM,aAAa,GAAiC;oBAChD,OAAO,EAAE,OAAO;oBAChB,MAAM,EAAE,aAAa;iBACxB,CAAA;gBAED,WAAW,GAAG,eAAe,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;gBAC/E,SAAQ;YACZ,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAA;YAE9B;;;;;eAKG;YACH,IAAI,GAAG,KAAK,qBAAa,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxD,MAAM,aAAa,GAAiC;oBAChD,OAAO,EAAE,OAAO;oBAChB,MAAM,EAAE;wBACJ,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,SAAU,CAAC;wBACtC,MAAM,EAAE,uCAAmB,CAAC,KAAK;wBACjC,MAAM,EAAE,SAAS;wBACjB,KAAK,EAAE,SAAS;qBACnB;iBACJ,CAAA;gBAED,WAAW,GAAG,eAAe,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;gBACtF,SAAQ;YACZ,CAAC;YAED,MAAM,aAAa,GAAkC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;iBACzE,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAA;YAEpE,MAAM,aAAa,GAAiC;gBAChD,OAAO,EAAE,OAAO;gBAChB,MAAM,EAAE,aAAa;aACxB,CAAA;YAED,WAAW,GAAG,eAAe,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;QACnF,CAAC;QAED,MAAM,YAAY,GAAgC;YAC9C,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,WAAW;SACvB,CAAA;QACD,OAAO,YAAY,CAAA;IACvB,CAAC,CAAC,CAAA;IAEF,OAAO,cAAc,CAAA;AACzB,CAAC,CAAA;AAlFQ,QAAA,eAAe,mBAkFvB;AAEE,MAAM,wBAAwB,GAAG,CAAgB,IAIvD,EAAE,EAAE,CACD,KAAK,EAAE,YAAgD,EAA+C,EAAE;IACpG,MAAM,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,OAAO,EAAC,EAAE;QACtD,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAA;YACrF,MAAM,YAAY,GAAG,YAAY,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAA;YAE3G,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;YAErD,MAAM,IAAI,CAAC,WAAW,CAAC;gBACnB,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,cAAc;aACjD,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAC,cAAc,EAAC,EAAE;gBAC/B,MAAM,IAAI,CAAC,uBAAuB,CAAC;oBAC/B,aAAa,EAAE,aAAa,CAAC,OAAO,CAAC,aAAa;oBAClD,KAAK,EAAE,cAAc,CAAC,QAAQ;oBAC9B,OAAO,EAAE,iBAAiB;iBAC7B,CAAC,CAAA;YACN,CAAC,CAAC,CAAA;QACN,CAAC;IACL,CAAC,CAAC,CAAC,CAAA;IAEH,OAAO,YAAY,CAAA;AACvB,CAAC,CAAA;AA1BQ,QAAA,wBAAwB,4BA0BhC;AAWE,MAAM,YAAY,GAAG,CAAgB,IAE3C,EAAE,EAAE,CACD,KAAK,EAAE,YAAgD,EAA+C,EAAE;IACpG,MAAM,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAC,OAAO,EAAC,EAAE;QACtD,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC;gBACnB,OAAO,EAAE,aAAa,CAAC,OAAO;gBAC9B,KAAK,EAAE;oBACH,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,KAAM,CAAC,OAAO;oBAC5C,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,KAAM,CAAC,IAAI;oBACtC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,KAAM,CAAC,KAAK;iBAC3C;aACJ,CAAC,CAAA;QACN,CAAC;IACL,CAAC,CAAC,CAAC,CAAA;IAEH,OAAO,YAAY,CAAA;AACvB,CAAC,CAAA;AAlBQ,QAAA,YAAY,gBAkBpB;AAEE,MAAM,kBAAkB,GAAG,KAAK,EAAiB,YAAgD,EAA6B,EAAE;IACnI,OAAO;QACH,iBAAiB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;aACpD,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,CAAC,EAAE;YACL,OAAO;gBACH,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS;aACtC,CAAA;QACL,CAAC,CAAC;KACT,CAAA;AACL,CAAC,CAAA;AAVY,QAAA,kBAAkB,sBAU9B;AAEM,MAAM,+BAA+B,GAAG,CAAgB,IAK9D,EAAE,EAAE,CACD,KAAK,EAAE,KAAe,EAA6B,EAAE;IACjD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;SAC7B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC;SACnC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;AACtC,CAAC,CAAA;AAXQ,QAAA,+BAA+B,mCAWvC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { RunProcess } from './aws';
|
|
2
|
+
import { SQSRecord } from 'aws-lambda';
|
|
3
|
+
import { SQSClient } from '@aws-sdk/client-sqs';
|
|
4
|
+
import { LoggingProps } from '@othree.io/journal';
|
|
5
|
+
export * from './results-processor';
|
|
6
|
+
export * as aws from './aws';
|
|
7
|
+
export type SQSProcessConfiguration = Readonly<{
|
|
8
|
+
dlqUrl: string;
|
|
9
|
+
exponentialTimeouts: Array<number>;
|
|
10
|
+
}>;
|
|
11
|
+
type Log = <Input, Output>(fn: (...args: Array<Input>) => Promise<Output>) => (context: string, loggingProps?: LoggingProps) => (...args: Array<Input>) => Promise<Output>;
|
|
12
|
+
export declare const processSqs: <Input, Output>(deps: {
|
|
13
|
+
log?: Log;
|
|
14
|
+
sqsClient: SQSClient;
|
|
15
|
+
configuration: SQSProcessConfiguration;
|
|
16
|
+
didItFail: (error: Error) => boolean;
|
|
17
|
+
runProcess: RunProcess<Input, Output>;
|
|
18
|
+
getProcessInput: RunProcess<SQSRecord, Input>;
|
|
19
|
+
}) => (event: import("aws-lambda").SQSEvent) => Promise<import("aws-lambda").SQSBatchResponse>;
|
|
20
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAOH,UAAU,EAEb,MAAM,OAAO,CAAA;AACd,OAAO,EAAC,SAAS,EAAC,MAAM,YAAY,CAAA;AAEpC,OAAO,EAAC,SAAS,EAAC,MAAM,qBAAqB,CAAA;AAE7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAG/C,cAAc,qBAAqB,CAAA;AACnC,OAAO,KAAK,GAAG,MAAM,OAAO,CAAA;AAE5B,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CACrC,CAAC,CAAA;AAEF,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,KAAK,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;AAQ1K,eAAO,MAAM,UAAU,GAAI,KAAK,EAAE,MAAM,EAAE,MAAM;IAC5C,GAAG,CAAC,EAAE,GAAG,CAAA;IACT,SAAS,EAAE,SAAS,CAAA;IACpB,aAAa,EAAE,uBAAuB,CAAA;IACtC,SAAS,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAA;IACpC,UAAU,EAAE,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IACrC,eAAe,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;CAChD,6FAqDA,CAAA"}
|
package/lib/cjs/index.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.processSqs = exports.aws = void 0;
|
|
18
|
+
const aws_1 = require("./aws");
|
|
19
|
+
const results_processor_1 = require("./results-processor");
|
|
20
|
+
const optional_1 = require("@othree.io/optional");
|
|
21
|
+
const awsome_1 = require("@othree.io/awsome");
|
|
22
|
+
__exportStar(require("./results-processor"), exports);
|
|
23
|
+
exports.aws = require("./aws");
|
|
24
|
+
const wrapWithLog = (maybeLog, fn, name) => {
|
|
25
|
+
return maybeLog
|
|
26
|
+
.map(_ => _(fn)(name))
|
|
27
|
+
.orElse(fn);
|
|
28
|
+
};
|
|
29
|
+
const processSqs = (deps) => {
|
|
30
|
+
const maybeLog = (0, optional_1.Optional)(deps.log);
|
|
31
|
+
const processSQSEventFn = wrapWithLog(maybeLog, (0, aws_1.processSQSEvent)({
|
|
32
|
+
process: deps.runProcess,
|
|
33
|
+
groupRecords: aws_1.groupRecords,
|
|
34
|
+
getProcessInput: deps.getProcessInput,
|
|
35
|
+
getProcessResults: (0, results_processor_1.getProcessResults)({ didItFail: deps.didItFail }),
|
|
36
|
+
}), 'processSQSEvent');
|
|
37
|
+
const changeMessageVisibilityTimeout = wrapWithLog(maybeLog, awsome_1.sqs.changeMessageVisibilityTimeout({ client: deps.sqsClient }), 'changeMessageVisibilityTimeout');
|
|
38
|
+
const handleExponentialBackoffFn = wrapWithLog(maybeLog, (0, aws_1.handleExponentialBackoff)({
|
|
39
|
+
changeMessageVisibility: changeMessageVisibilityTimeout,
|
|
40
|
+
getQueueUrl: awsome_1.sqs.getQueueUrl,
|
|
41
|
+
timeouts: deps.configuration.exponentialTimeouts,
|
|
42
|
+
}), 'handleExponentialBackoff');
|
|
43
|
+
const sendDLQMessage = awsome_1.sqs.send({
|
|
44
|
+
client: deps.sqsClient,
|
|
45
|
+
configuration: { QueueUrl: deps.configuration.dlqUrl },
|
|
46
|
+
});
|
|
47
|
+
const handleErrorsFn = wrapWithLog(maybeLog, (0, aws_1.handleErrors)({
|
|
48
|
+
notifyError: sendDLQMessage,
|
|
49
|
+
}), 'handleErrors');
|
|
50
|
+
return wrapWithLog(maybeLog, (0, aws_1.processSQSEventAndHandleResults)({
|
|
51
|
+
processSQSEvent: processSQSEventFn,
|
|
52
|
+
handleExponentialBackoff: handleExponentialBackoffFn,
|
|
53
|
+
handleErrors: handleErrorsFn,
|
|
54
|
+
toSQSBatchResponse: aws_1.toSQSBatchResponse,
|
|
55
|
+
}), 'processSQSEventAndHandleResults');
|
|
56
|
+
};
|
|
57
|
+
exports.processSqs = processSqs;
|
|
58
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,+BASc;AAEd,2DAAqD;AAErD,kDAA4C;AAE5C,8CAAqC;AAErC,sDAAmC;AACnC,+BAA4B;AAS5B,MAAM,WAAW,GAAG,CAAI,QAAuB,EAAE,EAAK,EAAE,IAAY,EAAK,EAAE;IACvE,OAAO,QAAQ;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAS,CAAC,CAAC,IAAI,CAAM,CAAC;SACjC,MAAM,CAAC,EAAE,CAAC,CAAA;AACnB,CAAC,CAAA;AAEM,MAAM,UAAU,GAAG,CAAgB,IAOzC,EAAE,EAAE;IACD,MAAM,QAAQ,GAAG,IAAA,mBAAQ,EAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEnC,MAAM,iBAAiB,GAAG,WAAW,CACjC,QAAQ,EACR,IAAA,qBAAe,EAAgB;QAC3B,OAAO,EAAE,IAAI,CAAC,UAAU;QACxB,YAAY,EAAZ,kBAAY;QACZ,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,iBAAiB,EAAE,IAAA,qCAAiB,EAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;KACtE,CAAC,EACF,iBAAiB,CACpB,CAAA;IAED,MAAM,8BAA8B,GAAG,WAAW,CAC9C,QAAQ,EACR,YAAG,CAAC,8BAA8B,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,EAC9D,gCAAgC,CACnC,CAAA;IAED,MAAM,0BAA0B,GAAG,WAAW,CAC1C,QAAQ,EACR,IAAA,8BAAwB,EAAgB;QACpC,uBAAuB,EAAE,8BAA8B;QACvD,WAAW,EAAE,YAAG,CAAC,WAAW;QAC5B,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,mBAAmB;KACnD,CAAC,EACF,0BAA0B,CAC7B,CAAA;IAED,MAAM,cAAc,GAAG,YAAG,CAAC,IAAI,CAA0B;QACrD,MAAM,EAAE,IAAI,CAAC,SAAS;QACtB,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;KACzD,CAAC,CAAA;IAEF,MAAM,cAAc,GAAG,WAAW,CAC9B,QAAQ,EACR,IAAA,kBAAY,EAAgB;QACxB,WAAW,EAAE,cAAc;KAC9B,CAAC,EACF,cAAc,CACjB,CAAA;IAED,OAAO,WAAW,CACd,QAAQ,EACR,IAAA,qCAA+B,EAAgB;QAC3C,eAAe,EAAE,iBAAiB;QAClC,wBAAwB,EAAE,0BAA0B;QACpD,YAAY,EAAE,cAAc;QAC5B,kBAAkB,EAAlB,wBAAkB;KACrB,CAAC,EACF,iCAAiC,CACpC,CAAA;AACL,CAAC,CAAA;AA5DY,QAAA,UAAU,cA4DtB"}
|