@aws-lambda-powertools/kafka 0.0.0 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +299 -2
- package/lib/cjs/constants.d.ts +10 -0
- package/lib/cjs/constants.d.ts.map +1 -0
- package/lib/cjs/constants.js +12 -0
- package/lib/cjs/consumer.d.ts +42 -0
- package/lib/cjs/consumer.d.ts.map +1 -0
- package/lib/cjs/consumer.js +250 -0
- package/lib/cjs/deserializer/avro.d.ts +9 -0
- package/lib/cjs/deserializer/avro.d.ts.map +1 -0
- package/lib/cjs/deserializer/avro.js +25 -0
- package/lib/cjs/deserializer/json.d.ts +9 -0
- package/lib/cjs/deserializer/json.d.ts.map +1 -0
- package/lib/cjs/deserializer/json.js +25 -0
- package/lib/cjs/deserializer/primitive.d.ts +10 -0
- package/lib/cjs/deserializer/primitive.d.ts.map +1 -0
- package/lib/cjs/deserializer/primitive.js +16 -0
- package/lib/cjs/deserializer/protobuf.d.ts +12 -0
- package/lib/cjs/deserializer/protobuf.d.ts.map +1 -0
- package/lib/cjs/deserializer/protobuf.js +80 -0
- package/lib/cjs/errors.d.ts +27 -0
- package/lib/cjs/errors.d.ts.map +1 -0
- package/lib/cjs/errors.js +44 -0
- package/lib/cjs/index.d.ts +3 -0
- package/lib/cjs/index.d.ts.map +1 -0
- package/lib/cjs/index.js +7 -0
- package/lib/cjs/package.json +1 -0
- package/lib/cjs/types/index.d.ts +2 -0
- package/lib/cjs/types/index.d.ts.map +1 -0
- package/lib/cjs/types/index.js +2 -0
- package/lib/cjs/types/types.d.ts +244 -0
- package/lib/cjs/types/types.d.ts.map +1 -0
- package/lib/cjs/types/types.js +2 -0
- package/lib/esm/constants.d.ts +10 -0
- package/lib/esm/constants.d.ts.map +1 -0
- package/lib/esm/constants.js +9 -0
- package/lib/esm/consumer.d.ts +42 -0
- package/lib/esm/consumer.d.ts.map +1 -0
- package/lib/esm/consumer.js +214 -0
- package/lib/esm/deserializer/avro.d.ts +9 -0
- package/lib/esm/deserializer/avro.d.ts.map +1 -0
- package/lib/esm/deserializer/avro.js +19 -0
- package/lib/esm/deserializer/json.d.ts +9 -0
- package/lib/esm/deserializer/json.d.ts.map +1 -0
- package/lib/esm/deserializer/json.js +22 -0
- package/lib/esm/deserializer/primitive.d.ts +10 -0
- package/lib/esm/deserializer/primitive.d.ts.map +1 -0
- package/lib/esm/deserializer/primitive.js +13 -0
- package/lib/esm/deserializer/protobuf.d.ts +12 -0
- package/lib/esm/deserializer/protobuf.d.ts.map +1 -0
- package/lib/esm/deserializer/protobuf.js +77 -0
- package/lib/esm/errors.d.ts +27 -0
- package/lib/esm/errors.d.ts.map +1 -0
- package/lib/esm/errors.js +38 -0
- package/lib/esm/index.d.ts +3 -0
- package/lib/esm/index.d.ts.map +1 -0
- package/lib/esm/index.js +2 -0
- package/lib/esm/package.json +1 -0
- package/lib/esm/types/index.d.ts +2 -0
- package/lib/esm/types/index.d.ts.map +1 -0
- package/lib/esm/types/index.js +1 -0
- package/lib/esm/types/types.d.ts +244 -0
- package/lib/esm/types/types.d.ts.map +1 -0
- package/lib/esm/types/types.js +1 -0
- package/package.json +77 -19
- package/lib/index.d.ts +0 -0
- package/lib/index.js +0 -3
package/README.md
CHANGED
|
@@ -4,5 +4,302 @@ Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement Serve
|
|
|
4
4
|
|
|
5
5
|
You can use the package in both TypeScript and JavaScript code bases.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
## Intro
|
|
8
|
+
|
|
9
|
+
The Kafka Consumer utility transparently handles message deserialization, provides an intuitive developer experience, and integrates seamlessly with the rest of the Powertools for AWS Lambda ecosystem.
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
To get started, depending on the schema types you want to use, install the library and the corresponding libraries:
|
|
14
|
+
|
|
15
|
+
For JSON schemas:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npm install @aws-lambda-powertools/kafka
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
For Avro schemas:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npm install @aws-lambda-powertools/kafka avro-js
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
For Protobuf schemas:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
npm install @aws-lambda-powertools/kafka protobufjs
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Additionally, if you want to use output parsing with [Standard Schema](https://github.com/standard-schema/standard-schema), you can install [any of the supported libraries](https://standardschema.dev/#what-schema-libraries-implement-the-spec), for example: Zod, Valibot, or ArkType.
|
|
34
|
+
|
|
35
|
+
### Deserialization
|
|
36
|
+
|
|
37
|
+
The Kafka consumer utility transforms raw Kafka events into an intuitive format for processing. To handle messages effectively, you'll need to configure a schema that matches your data format.
|
|
38
|
+
|
|
39
|
+
#### JSON Schema
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
43
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
44
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
45
|
+
|
|
46
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
47
|
+
|
|
48
|
+
const schemaConfig = {
|
|
49
|
+
value: {
|
|
50
|
+
type: SchemaType.JSON,
|
|
51
|
+
},
|
|
52
|
+
} satisfies SchemaConfig;
|
|
53
|
+
|
|
54
|
+
export const handler = kafkaConsumer(async (event, _context) => {
|
|
55
|
+
for (const { value } of event.records) {
|
|
56
|
+
logger.info('received value', { value });
|
|
57
|
+
}
|
|
58
|
+
}, schemaConfig);
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
#### Avro Schema
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
import { readFileSync } from 'node:fs';
|
|
65
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
66
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
67
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
68
|
+
|
|
69
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
70
|
+
|
|
71
|
+
const schemaConfig = {
|
|
72
|
+
value: {
|
|
73
|
+
type: SchemaType.AVRO,
|
|
74
|
+
schema: readFileSync(new URL('./user.avsc', import.meta.url), 'utf8'),
|
|
75
|
+
},
|
|
76
|
+
} satisfies SchemaConfig;
|
|
77
|
+
|
|
78
|
+
export const handler = kafkaConsumer(async (event, _context) => {
|
|
79
|
+
for (const { value } of event.records) {
|
|
80
|
+
logger.info('received value', { value });
|
|
81
|
+
}
|
|
82
|
+
}, schemaConfig);
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
#### Protobuf Schema
|
|
86
|
+
|
|
87
|
+
```ts
|
|
88
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
89
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
90
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
91
|
+
import { User } from './samples/user.es6.generated.js'; // protobuf generated class
|
|
92
|
+
|
|
93
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
94
|
+
|
|
95
|
+
const schemaConfig = {
|
|
96
|
+
value: {
|
|
97
|
+
type: SchemaType.PROTOBUF,
|
|
98
|
+
schema: User,
|
|
99
|
+
},
|
|
100
|
+
} satisfies SchemaConfig;
|
|
101
|
+
|
|
102
|
+
export const handler = kafkaConsumer(async (event, _context) => {
|
|
103
|
+
for (const { value } of event.records) {
|
|
104
|
+
logger.info('received value', { value });
|
|
105
|
+
}
|
|
106
|
+
}, schemaConfig);
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Additional Parsing
|
|
110
|
+
|
|
111
|
+
You can parse deserialized data using your preferred parsing library. This can help you integrate Kafka data with your domain schemas and application architecture, providing type hints, runtime parsing and validation, and advanced data transformations.
|
|
112
|
+
|
|
113
|
+
#### Zod
|
|
114
|
+
|
|
115
|
+
```ts
|
|
116
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
117
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
118
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
119
|
+
import { z } from 'zod/v4';
|
|
120
|
+
|
|
121
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
122
|
+
|
|
123
|
+
const OrderItemSchema = z.object({
|
|
124
|
+
productId: z.string(),
|
|
125
|
+
quantity: z.number().int().positive(),
|
|
126
|
+
price: z.number().positive(),
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
const OrderSchema = z.object({
|
|
130
|
+
id: z.string(),
|
|
131
|
+
customerId: z.string(),
|
|
132
|
+
items: z.array(OrderItemSchema).min(1, 'Order must have at least one item'),
|
|
133
|
+
createdAt: z.iso.datetime(),
|
|
134
|
+
totalAmount: z.number().positive(),
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
const schemaConfig = {
|
|
138
|
+
value: {
|
|
139
|
+
type: SchemaType.JSON,
|
|
140
|
+
parserSchema: OrderSchema,
|
|
141
|
+
},
|
|
142
|
+
} satisfies SchemaConfig;
|
|
143
|
+
|
|
144
|
+
export const handler = kafkaConsumer<unknown, z.infer<typeof OrderSchema>>(
|
|
145
|
+
async (event, _context) => {
|
|
146
|
+
for (const record of event.records) {
|
|
147
|
+
const {
|
|
148
|
+
value: { id, items },
|
|
149
|
+
} = record;
|
|
150
|
+
logger.setCorrelationId(id);
|
|
151
|
+
logger.debug(`order includes ${items.length} items`);
|
|
152
|
+
}
|
|
153
|
+
},
|
|
154
|
+
schemaConfig
|
|
155
|
+
);
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
#### Valibot
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
162
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
163
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
164
|
+
import * as v from 'valibot';
|
|
165
|
+
|
|
166
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
167
|
+
|
|
168
|
+
const OrderItemSchema = v.object({
|
|
169
|
+
productId: v.string(),
|
|
170
|
+
quantity: v.pipe(v.number(), v.integer(), v.toMinValue(1)),
|
|
171
|
+
price: v.pipe(v.number(), v.integer()),
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const OrderSchema = v.object({
|
|
175
|
+
id: v.string(),
|
|
176
|
+
customerId: v.string(),
|
|
177
|
+
items: v.pipe(
|
|
178
|
+
v.array(OrderItemSchema),
|
|
179
|
+
v.minLength(1, 'Order must have at least one item')
|
|
180
|
+
),
|
|
181
|
+
createdAt: v.pipe(v.string(), v.isoDateTime()),
|
|
182
|
+
totalAmount: v.pipe(v.number(), v.toMinValue(0)),
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
const schemaConfig = {
|
|
186
|
+
value: {
|
|
187
|
+
type: SchemaType.JSON,
|
|
188
|
+
parserSchema: OrderSchema,
|
|
189
|
+
},
|
|
190
|
+
} satisfies SchemaConfig;
|
|
191
|
+
|
|
192
|
+
export const handler = kafkaConsumer<unknown, v.InferInput<typeof OrderSchema>>(
|
|
193
|
+
async (event, _context) => {
|
|
194
|
+
for (const record of event.records) {
|
|
195
|
+
const {
|
|
196
|
+
value: { id, items },
|
|
197
|
+
} = record;
|
|
198
|
+
logger.setCorrelationId(id);
|
|
199
|
+
logger.debug(`order includes ${items.length} items`);
|
|
200
|
+
}
|
|
201
|
+
},
|
|
202
|
+
schemaConfig
|
|
203
|
+
);
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
#### ArkType
|
|
207
|
+
|
|
208
|
+
```ts
|
|
209
|
+
import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
210
|
+
import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
|
|
211
|
+
import { Logger } from '@aws-lambda-powertools/logger';
|
|
212
|
+
import { type } from 'arktype';
|
|
213
|
+
|
|
214
|
+
const logger = new Logger({ serviceName: 'kafka-consumer' });
|
|
215
|
+
|
|
216
|
+
const OrderItemSchema = type({
|
|
217
|
+
productId: 'string',
|
|
218
|
+
quantity: 'number.integer >= 1',
|
|
219
|
+
price: 'number.integer',
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
const OrderSchema = type({
|
|
223
|
+
id: 'string',
|
|
224
|
+
customerId: 'string',
|
|
225
|
+
items: OrderItemSchema.array().moreThanLength(0),
|
|
226
|
+
createdAt: 'string.date',
|
|
227
|
+
totalAmount: 'number.integer >= 0',
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const schemaConfig = {
|
|
231
|
+
value: {
|
|
232
|
+
type: SchemaType.JSON,
|
|
233
|
+
parserSchema: OrderSchema,
|
|
234
|
+
},
|
|
235
|
+
} satisfies SchemaConfig;
|
|
236
|
+
|
|
237
|
+
export const handler = kafkaConsumer<unknown, typeof OrderSchema.infer>(
|
|
238
|
+
async (event, _context) => {
|
|
239
|
+
for (const record of event.records) {
|
|
240
|
+
const {
|
|
241
|
+
value: { id, items },
|
|
242
|
+
} = record;
|
|
243
|
+
logger.setCorrelationId(id);
|
|
244
|
+
logger.debug(`order includes ${items.length} items`);
|
|
245
|
+
}
|
|
246
|
+
},
|
|
247
|
+
schemaConfig
|
|
248
|
+
);
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
See the [documentation](https://docs.powertools.aws.dev/lambda/typescript/latest/features/kafka) for more details on how to use the Kafka utility.
|
|
252
|
+
|
|
253
|
+
## Contribute
|
|
254
|
+
|
|
255
|
+
If you are interested in contributing to this project, please refer to our [Contributing Guidelines](https://github.com/aws-powertools/powertools-lambda-typescript/blob/main/CONTRIBUTING.md).
|
|
256
|
+
|
|
257
|
+
## Roadmap
|
|
258
|
+
|
|
259
|
+
The roadmap of Powertools for AWS Lambda (TypeScript) is driven by customers’ demand.
|
|
260
|
+
Help us prioritize upcoming functionalities or utilities by [upvoting existing RFCs and feature requests](https://github.com/aws-powertools/powertools-lambda-typescript/issues), or [creating new ones](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new/choose), in this GitHub repository.
|
|
261
|
+
|
|
262
|
+
## Connect
|
|
263
|
+
|
|
264
|
+
* **Powertools for AWS Lambda on Discord**: `#typescript` - **[Invite link](https://discord.gg/B8zZKbbyET)**
|
|
265
|
+
* **Email**: <aws-lambda-powertools-feedback@amazon.com>
|
|
266
|
+
|
|
267
|
+
## How to support Powertools for AWS Lambda (TypeScript)?
|
|
268
|
+
|
|
269
|
+
### Becoming a reference customer
|
|
270
|
+
|
|
271
|
+
Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://s12d.com/become-reference-pt-ts) issue.
|
|
272
|
+
|
|
273
|
+
The following companies, among others, use Powertools:
|
|
274
|
+
|
|
275
|
+
* [Alma Media](https://www.almamedia.fi)
|
|
276
|
+
* [AppYourself](https://appyourself.net)
|
|
277
|
+
* [Bailey Nelson](https://www.baileynelson.com.au)
|
|
278
|
+
* [Banxware](https://www.banxware.com)
|
|
279
|
+
* [Caylent](https://caylent.com/)
|
|
280
|
+
* [Certible](https://www.certible.com/)
|
|
281
|
+
* [Elva](https://elva-group.com)
|
|
282
|
+
* [Flyweight](https://flyweight.io/)
|
|
283
|
+
* [globaldatanet](https://globaldatanet.com/)
|
|
284
|
+
* [Guild](https://guild.com)
|
|
285
|
+
* [Hashnode](https://hashnode.com/)
|
|
286
|
+
* [Instil](https://instil.co/)
|
|
287
|
+
* [LocalStack](https://localstack.cloud/)
|
|
288
|
+
* [Ours Privacy](https://oursprivacy.com/)
|
|
289
|
+
* [Perfect Post](https://www.perfectpost.fr)
|
|
290
|
+
* [Sennder](https://sennder.com/)
|
|
291
|
+
* [tecRacer GmbH & Co. KG](https://www.tecracer.com/)
|
|
292
|
+
* [Trek10](https://www.trek10.com/)
|
|
293
|
+
* [WeSchool](https://www.weschool.com)
|
|
294
|
+
|
|
295
|
+
### Sharing your work
|
|
296
|
+
|
|
297
|
+
Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has [already shared](https://docs.powertools.aws.dev/lambda/typescript/latest/we_made_this) about Powertools for AWS Lambda (TypeScript).
|
|
298
|
+
|
|
299
|
+
### Using Lambda Layer
|
|
300
|
+
|
|
301
|
+
This helps us understand who uses Powertools for AWS Lambda (TypeScript) in a non-intrusive way, and helps us gain future investments for other Powertools for AWS Lambda languages. When [using Layers](https://docs.powertools.aws.dev/lambda/typescript/latest/getting-started/lambda-layers/), you can add Powertools as a dev dependency to not impact the development process.
|
|
302
|
+
|
|
303
|
+
## License
|
|
304
|
+
|
|
305
|
+
This library is licensed under the MIT-0 License. See the LICENSE file.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SchemaType = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Types of Kafka schema formats.
|
|
6
|
+
*/
|
|
7
|
+
const SchemaType = {
|
|
8
|
+
JSON: 'json',
|
|
9
|
+
AVRO: 'avro',
|
|
10
|
+
PROTOBUF: 'protobuf',
|
|
11
|
+
};
|
|
12
|
+
exports.SchemaType = SchemaType;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { AsyncHandler } from '@aws-lambda-powertools/commons/types';
|
|
2
|
+
import type { Context, Handler } from 'aws-lambda';
|
|
3
|
+
import type { ConsumerRecords, MSKEvent, SchemaConfig } from './types/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Wrap a handler function to automatically deserialize and validate Kafka records from an MSK event.
|
|
6
|
+
*
|
|
7
|
+
* The returned function will:
|
|
8
|
+
* - Deserialize the key and value of each record using the provided schema config.
|
|
9
|
+
* - Validate the deserialized key and value using Zod schemas if provided.
|
|
10
|
+
* - Replace the `records` property in the event with an array of deserialized and validated records.
|
|
11
|
+
* - Call the original handler with the modified event and original context/arguments.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* import { kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
16
|
+
* import { z } from 'zod';
|
|
17
|
+
*
|
|
18
|
+
* const keySchema = z.string();
|
|
19
|
+
* const valueSchema = z.object({
|
|
20
|
+
* id: z.number(),
|
|
21
|
+
* });
|
|
22
|
+
*
|
|
23
|
+
* export const handler = kafkaConsumer<z.infer<keySchema>, z.infer<valueSchema>>(async (event, context) => {
|
|
24
|
+
* // event.records is now an array of deserialized and validated records
|
|
25
|
+
* for (const record of event.records) {
|
|
26
|
+
* console.log(record.key, record.value);
|
|
27
|
+
* }
|
|
28
|
+
* }, {
|
|
29
|
+
* key: { type: 'json', parserSchema: keySchema },
|
|
30
|
+
* value: { type: 'json', parserSchema: valueSchema },
|
|
31
|
+
* });
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* @typeParam K - Optional type of the deserialized key - defaults to `unknown`.
|
|
35
|
+
* @typeParam V - Optional type of the deserialized value - defaults to `unknown`.
|
|
36
|
+
*
|
|
37
|
+
* @param handler - The original handler function to wrap. It should accept the deserialized event as its first argument.
|
|
38
|
+
* @param config - The schema configuration for deserializing and validating record keys and values.
|
|
39
|
+
*/
|
|
40
|
+
declare const kafkaConsumer: <K, V>(handler: AsyncHandler<Handler<ConsumerRecords<K, V>>>, config?: SchemaConfig) => ((event: MSKEvent, context: Context) => Promise<unknown>);
|
|
41
|
+
export { kafkaConsumer };
|
|
42
|
+
//# sourceMappingURL=consumer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;AAsM1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,QAAA,MAAM,aAAa,GAAI,CAAC,EAAE,CAAC,EACzB,SAAS,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EACrD,SAAS,YAAY,KACpB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAwB1D,CAAC;AAEF,OAAO,EAAE,aAAa,EAAE,CAAC"}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.kafkaConsumer = void 0;
|
|
37
|
+
const typeutils_1 = require("@aws-lambda-powertools/commons/typeutils");
|
|
38
|
+
const json_js_1 = require("./deserializer/json.js");
|
|
39
|
+
const primitive_js_1 = require("./deserializer/primitive.js");
|
|
40
|
+
const errors_js_1 = require("./errors.js");
|
|
41
|
+
/**
|
|
42
|
+
* Type guard to assert that the event is a valid {@link MSKEvent | `MSKEvent`}.
|
|
43
|
+
*
|
|
44
|
+
* @param event - The event to validate, expected to be an MSKEvent.
|
|
45
|
+
*/
|
|
46
|
+
const assertIsMSKEvent = (event) => {
|
|
47
|
+
if (!(0, typeutils_1.isRecord)(event) ||
|
|
48
|
+
!(0, typeutils_1.isRecord)(event.records) ||
|
|
49
|
+
!Object.values(event.records).every((arr) => Array.isArray(arr))) {
|
|
50
|
+
throw new errors_js_1.KafkaConsumerError('Event is not a valid MSKEvent. Expected an object with a "records" property.');
|
|
51
|
+
}
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Deserialize Kafka message headers from an array of header objects.
|
|
56
|
+
*
|
|
57
|
+
* It returns `null` if the headers are `null`, or an array of header objects
|
|
58
|
+
* where each header value is decoded as a UTF-8 string.
|
|
59
|
+
*
|
|
60
|
+
* @param headers - An array of header objects, where each object maps header keys (string)
|
|
61
|
+
* to header values (`number[]`), representing the raw bytes of each header value -
|
|
62
|
+
* i.e. `[{ "headerKey": [104, 101, 108, 108, 111] }]`
|
|
63
|
+
*/
|
|
64
|
+
const deserializeHeaders = (headers) => {
|
|
65
|
+
if (headers === null) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
const result = [];
|
|
69
|
+
for (const header of headers) {
|
|
70
|
+
const entries = [];
|
|
71
|
+
for (const [headerKey, headerValue] of Object.entries(header)) {
|
|
72
|
+
entries.push([headerKey, Buffer.from(headerValue).toString('utf-8')]);
|
|
73
|
+
}
|
|
74
|
+
result.push(Object.fromEntries(entries));
|
|
75
|
+
}
|
|
76
|
+
return result;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Deserialize a base64-encoded value using the provided schema configuration.
|
|
80
|
+
*
|
|
81
|
+
* It returns the deserialized value, which may be a string, object, or other type depending on the schema type.
|
|
82
|
+
*
|
|
83
|
+
* @param value - The base64-encoded string to deserialize.
|
|
84
|
+
* @param config - The schema configuration to use for deserialization. See {@link SchemaConfigValue | `SchemaConfigValue`}.
|
|
85
|
+
* If not provided, the value is decoded as a UTF-8 string.
|
|
86
|
+
*/
|
|
87
|
+
const deserialize = ({ value, deserializer, config, schemaMetadata, }) => {
|
|
88
|
+
if (config === undefined) {
|
|
89
|
+
return deserializer(value);
|
|
90
|
+
}
|
|
91
|
+
if (config.type === 'json') {
|
|
92
|
+
return deserializer(value);
|
|
93
|
+
}
|
|
94
|
+
if (config.type === 'avro') {
|
|
95
|
+
if (!config.schema) {
|
|
96
|
+
throw new errors_js_1.KafkaConsumerMissingSchemaError('Schema string is required for avro deserialization');
|
|
97
|
+
}
|
|
98
|
+
return deserializer(value, config.schema);
|
|
99
|
+
}
|
|
100
|
+
if (config.type === 'protobuf') {
|
|
101
|
+
if (!config.schema) {
|
|
102
|
+
throw new errors_js_1.KafkaConsumerMissingSchemaError('Schema string is required for protobuf deserialization');
|
|
103
|
+
}
|
|
104
|
+
return deserializer(value, config.schema, schemaMetadata);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
/**
|
|
108
|
+
* Get the deserializer function based on the provided type.
|
|
109
|
+
*
|
|
110
|
+
* @param type - The type of deserializer to use. Supported types are: `json`, `avro`, `protobuf`, or `undefined`.
|
|
111
|
+
* If `undefined`, it defaults to deserializing as a primitive string.
|
|
112
|
+
*/
|
|
113
|
+
const getDeserializer = async (type) => {
|
|
114
|
+
if (!type) {
|
|
115
|
+
return primitive_js_1.deserialize;
|
|
116
|
+
}
|
|
117
|
+
if (type === 'json') {
|
|
118
|
+
return json_js_1.deserialize;
|
|
119
|
+
}
|
|
120
|
+
if (type === 'protobuf') {
|
|
121
|
+
const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/protobuf.js')));
|
|
122
|
+
return deserializer.deserialize;
|
|
123
|
+
}
|
|
124
|
+
if (type === 'avro') {
|
|
125
|
+
const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/avro.js')));
|
|
126
|
+
return deserializer.deserialize;
|
|
127
|
+
}
|
|
128
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Parse a value against a provided schema using the `~standard` property for validation.
|
|
132
|
+
*
|
|
133
|
+
* @param value - The value to parse against the schema.
|
|
134
|
+
* @param schema - The schema to validate against, which should be a {@link StandardSchemaV1 | `Standard Schema V1`} object.
|
|
135
|
+
*/
|
|
136
|
+
const parseSchema = (value, schema) => {
|
|
137
|
+
const result = schema['~standard'].validate(value);
|
|
138
|
+
/* v8 ignore start */
|
|
139
|
+
if (result instanceof Promise)
|
|
140
|
+
throw new errors_js_1.KafkaConsumerParserError('Schema parsing supports only synchronous validation');
|
|
141
|
+
/* v8 ignore stop */
|
|
142
|
+
if (result.issues) {
|
|
143
|
+
throw new errors_js_1.KafkaConsumerParserError('Schema validation failed', {
|
|
144
|
+
cause: result.issues,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
return result.value;
|
|
148
|
+
};
|
|
149
|
+
/**
|
|
150
|
+
* Deserialize a single record from an MSK event.
|
|
151
|
+
*
|
|
152
|
+
* @param record - A single record from the MSK event.
|
|
153
|
+
* @param config - The schema configuration for deserializing the record's key and value.
|
|
154
|
+
*/
|
|
155
|
+
const deserializeRecord = async (record, config) => {
|
|
156
|
+
const { key, value, headers, valueSchemaMetadata, keySchemaMetadata, ...rest } = record;
|
|
157
|
+
const { key: keyConfig, value: valueConfig } = config || {};
|
|
158
|
+
const deserializerKey = await getDeserializer(keyConfig?.type);
|
|
159
|
+
const deserializerValue = await getDeserializer(valueConfig?.type);
|
|
160
|
+
return {
|
|
161
|
+
...rest,
|
|
162
|
+
get key() {
|
|
163
|
+
if (key === undefined || key === '') {
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
if ((0, typeutils_1.isNull)(key))
|
|
167
|
+
return null;
|
|
168
|
+
const deserializedKey = deserialize({
|
|
169
|
+
value: key,
|
|
170
|
+
deserializer: deserializerKey,
|
|
171
|
+
config: keyConfig,
|
|
172
|
+
schemaMetadata: keySchemaMetadata,
|
|
173
|
+
});
|
|
174
|
+
return keyConfig?.parserSchema
|
|
175
|
+
? parseSchema(deserializedKey, keyConfig.parserSchema)
|
|
176
|
+
: deserializedKey;
|
|
177
|
+
},
|
|
178
|
+
originalKey: key,
|
|
179
|
+
get value() {
|
|
180
|
+
const deserializedValue = deserialize({
|
|
181
|
+
value: value,
|
|
182
|
+
deserializer: deserializerValue,
|
|
183
|
+
config: valueConfig,
|
|
184
|
+
schemaMetadata: valueSchemaMetadata,
|
|
185
|
+
});
|
|
186
|
+
return valueConfig?.parserSchema
|
|
187
|
+
? parseSchema(deserializedValue, valueConfig.parserSchema)
|
|
188
|
+
: deserializedValue;
|
|
189
|
+
},
|
|
190
|
+
originalValue: value,
|
|
191
|
+
get headers() {
|
|
192
|
+
return deserializeHeaders(headers);
|
|
193
|
+
},
|
|
194
|
+
originalHeaders: headers,
|
|
195
|
+
valueSchemaMetadata,
|
|
196
|
+
keySchemaMetadata,
|
|
197
|
+
};
|
|
198
|
+
};
|
|
199
|
+
/**
|
|
200
|
+
* Wrap a handler function to automatically deserialize and validate Kafka records from an MSK event.
|
|
201
|
+
*
|
|
202
|
+
* The returned function will:
|
|
203
|
+
* - Deserialize the key and value of each record using the provided schema config.
|
|
204
|
+
* - Validate the deserialized key and value using Zod schemas if provided.
|
|
205
|
+
* - Replace the `records` property in the event with an array of deserialized and validated records.
|
|
206
|
+
* - Call the original handler with the modified event and original context/arguments.
|
|
207
|
+
*
|
|
208
|
+
* @example
|
|
209
|
+
* ```ts
|
|
210
|
+
* import { kafkaConsumer } from '@aws-lambda-powertools/kafka';
|
|
211
|
+
* import { z } from 'zod';
|
|
212
|
+
*
|
|
213
|
+
* const keySchema = z.string();
|
|
214
|
+
* const valueSchema = z.object({
|
|
215
|
+
* id: z.number(),
|
|
216
|
+
* });
|
|
217
|
+
*
|
|
218
|
+
* export const handler = kafkaConsumer<z.infer<keySchema>, z.infer<valueSchema>>(async (event, context) => {
|
|
219
|
+
* // event.records is now an array of deserialized and validated records
|
|
220
|
+
* for (const record of event.records) {
|
|
221
|
+
* console.log(record.key, record.value);
|
|
222
|
+
* }
|
|
223
|
+
* }, {
|
|
224
|
+
* key: { type: 'json', parserSchema: keySchema },
|
|
225
|
+
* value: { type: 'json', parserSchema: valueSchema },
|
|
226
|
+
* });
|
|
227
|
+
* ```
|
|
228
|
+
*
|
|
229
|
+
* @typeParam K - Optional type of the deserialized key - defaults to `unknown`.
|
|
230
|
+
* @typeParam V - Optional type of the deserialized value - defaults to `unknown`.
|
|
231
|
+
*
|
|
232
|
+
* @param handler - The original handler function to wrap. It should accept the deserialized event as its first argument.
|
|
233
|
+
* @param config - The schema configuration for deserializing and validating record keys and values.
|
|
234
|
+
*/
|
|
235
|
+
const kafkaConsumer = (handler, config) => {
|
|
236
|
+
return async (event, context) => {
|
|
237
|
+
assertIsMSKEvent(event);
|
|
238
|
+
const consumerRecords = [];
|
|
239
|
+
for (const recordsArray of Object.values(event.records)) {
|
|
240
|
+
for (const record of recordsArray) {
|
|
241
|
+
consumerRecords.push((await deserializeRecord(record, config)));
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return handler({
|
|
245
|
+
...event,
|
|
246
|
+
records: consumerRecords,
|
|
247
|
+
}, context);
|
|
248
|
+
};
|
|
249
|
+
};
|
|
250
|
+
exports.kafkaConsumer = kafkaConsumer;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
|
|
3
|
+
*
|
|
4
|
+
* @param data - The base64-encoded string representing the Avro binary data.
|
|
5
|
+
* @param schema - The Avro schema as a JSON string.
|
|
6
|
+
*/
|
|
7
|
+
declare const deserialize: (data: string, schema: string) => unknown;
|
|
8
|
+
export { deserialize };
|
|
9
|
+
//# sourceMappingURL=avro.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,YAUhD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.deserialize = void 0;
|
|
7
|
+
const avro_js_1 = __importDefault(require("avro-js"));
|
|
8
|
+
const errors_js_1 = require("../errors.js");
|
|
9
|
+
/**
|
|
10
|
+
* Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
|
|
11
|
+
*
|
|
12
|
+
* @param data - The base64-encoded string representing the Avro binary data.
|
|
13
|
+
* @param schema - The Avro schema as a JSON string.
|
|
14
|
+
*/
|
|
15
|
+
const deserialize = (data, schema) => {
|
|
16
|
+
try {
|
|
17
|
+
const type = avro_js_1.default.parse(schema);
|
|
18
|
+
const buffer = Buffer.from(data, 'base64');
|
|
19
|
+
return type.fromBuffer(buffer);
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
exports.deserialize = deserialize;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deserialize a base64 encoded string into either a JSON object or plain string
|
|
3
|
+
*
|
|
4
|
+
* @param data - The base64 encoded string to deserialize
|
|
5
|
+
* @returns The deserialized data as either a JSON object or string
|
|
6
|
+
*/
|
|
7
|
+
declare const deserialize: (data: string) => any;
|
|
8
|
+
export { deserialize };
|
|
9
|
+
//# sourceMappingURL=json.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,QAYhC,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|