@aws-lambda-powertools/kafka 0.0.0 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +299 -2
  2. package/lib/cjs/constants.d.ts +10 -0
  3. package/lib/cjs/constants.d.ts.map +1 -0
  4. package/lib/cjs/constants.js +12 -0
  5. package/lib/cjs/consumer.d.ts +42 -0
  6. package/lib/cjs/consumer.d.ts.map +1 -0
  7. package/lib/cjs/consumer.js +250 -0
  8. package/lib/cjs/deserializer/avro.d.ts +9 -0
  9. package/lib/cjs/deserializer/avro.d.ts.map +1 -0
  10. package/lib/cjs/deserializer/avro.js +25 -0
  11. package/lib/cjs/deserializer/json.d.ts +9 -0
  12. package/lib/cjs/deserializer/json.d.ts.map +1 -0
  13. package/lib/cjs/deserializer/json.js +25 -0
  14. package/lib/cjs/deserializer/primitive.d.ts +10 -0
  15. package/lib/cjs/deserializer/primitive.d.ts.map +1 -0
  16. package/lib/cjs/deserializer/primitive.js +16 -0
  17. package/lib/cjs/deserializer/protobuf.d.ts +12 -0
  18. package/lib/cjs/deserializer/protobuf.d.ts.map +1 -0
  19. package/lib/cjs/deserializer/protobuf.js +80 -0
  20. package/lib/cjs/errors.d.ts +27 -0
  21. package/lib/cjs/errors.d.ts.map +1 -0
  22. package/lib/cjs/errors.js +44 -0
  23. package/lib/cjs/index.d.ts +3 -0
  24. package/lib/cjs/index.d.ts.map +1 -0
  25. package/lib/cjs/index.js +7 -0
  26. package/lib/cjs/package.json +1 -0
  27. package/lib/cjs/types/index.d.ts +2 -0
  28. package/lib/cjs/types/index.d.ts.map +1 -0
  29. package/lib/cjs/types/index.js +2 -0
  30. package/lib/cjs/types/types.d.ts +244 -0
  31. package/lib/cjs/types/types.d.ts.map +1 -0
  32. package/lib/cjs/types/types.js +2 -0
  33. package/lib/esm/constants.d.ts +10 -0
  34. package/lib/esm/constants.d.ts.map +1 -0
  35. package/lib/esm/constants.js +9 -0
  36. package/lib/esm/consumer.d.ts +42 -0
  37. package/lib/esm/consumer.d.ts.map +1 -0
  38. package/lib/esm/consumer.js +214 -0
  39. package/lib/esm/deserializer/avro.d.ts +9 -0
  40. package/lib/esm/deserializer/avro.d.ts.map +1 -0
  41. package/lib/esm/deserializer/avro.js +19 -0
  42. package/lib/esm/deserializer/json.d.ts +9 -0
  43. package/lib/esm/deserializer/json.d.ts.map +1 -0
  44. package/lib/esm/deserializer/json.js +22 -0
  45. package/lib/esm/deserializer/primitive.d.ts +10 -0
  46. package/lib/esm/deserializer/primitive.d.ts.map +1 -0
  47. package/lib/esm/deserializer/primitive.js +13 -0
  48. package/lib/esm/deserializer/protobuf.d.ts +12 -0
  49. package/lib/esm/deserializer/protobuf.d.ts.map +1 -0
  50. package/lib/esm/deserializer/protobuf.js +77 -0
  51. package/lib/esm/errors.d.ts +27 -0
  52. package/lib/esm/errors.d.ts.map +1 -0
  53. package/lib/esm/errors.js +38 -0
  54. package/lib/esm/index.d.ts +3 -0
  55. package/lib/esm/index.d.ts.map +1 -0
  56. package/lib/esm/index.js +2 -0
  57. package/lib/esm/package.json +1 -0
  58. package/lib/esm/types/index.d.ts +2 -0
  59. package/lib/esm/types/index.d.ts.map +1 -0
  60. package/lib/esm/types/index.js +1 -0
  61. package/lib/esm/types/types.d.ts +244 -0
  62. package/lib/esm/types/types.d.ts.map +1 -0
  63. package/lib/esm/types/types.js +1 -0
  64. package/package.json +77 -19
  65. package/lib/index.d.ts +0 -0
  66. package/lib/index.js +0 -3
package/README.md CHANGED
@@ -4,5 +4,302 @@ Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement Serve
4
4
 
5
5
  You can use the package in both TypeScript and JavaScript code bases.
6
6
 
7
- > [!WARNING]
8
- > **This version is a placeholder and under active development.** APIs and features may change frequently without notice.
7
+ ## Intro
8
+
9
+ The Kafka Consumer utility transparently handles message deserialization, provides an intuitive developer experience, and integrates seamlessly with the rest of the Powertools for AWS Lambda ecosystem.
10
+
11
+ ## Usage
12
+
13
+ To get started, depending on the schema types you want to use, install the library and the corresponding libraries:
14
+
15
+ For JSON schemas:
16
+
17
+ ```bash
18
+ npm install @aws-lambda-powertools/kafka
19
+ ```
20
+
21
+ For Avro schemas:
22
+
23
+ ```bash
24
+ npm install @aws-lambda-powertools/kafka avro-js
25
+ ```
26
+
27
+ For Protobuf schemas:
28
+
29
+ ```bash
30
+ npm install @aws-lambda-powertools/kafka protobufjs
31
+ ```
32
+
33
+ Additionally, if you want to use output parsing with [Standard Schema](https://github.com/standard-schema/standard-schema), you can install [any of the supported libraries](https://standardschema.dev/#what-schema-libraries-implement-the-spec), for example: Zod, Valibot, or ArkType.
34
+
35
+ ### Deserialization
36
+
37
+ The Kafka consumer utility transforms raw Kafka events into an intuitive format for processing. To handle messages effectively, you'll need to configure a schema that matches your data format.
38
+
39
+ #### JSON Schema
40
+
41
+ ```ts
42
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
43
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
44
+ import { Logger } from '@aws-lambda-powertools/logger';
45
+
46
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
47
+
48
+ const schemaConfig = {
49
+ value: {
50
+ type: SchemaType.JSON,
51
+ },
52
+ } satisfies SchemaConfig;
53
+
54
+ export const handler = kafkaConsumer(async (event, _context) => {
55
+ for (const { value } of event.records) {
56
+ logger.info('received value', { value });
57
+ }
58
+ }, schemaConfig);
59
+ ```
60
+
61
+ #### Avro Schema
62
+
63
+ ```ts
64
+ import { readFileSync } from 'node:fs';
65
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
66
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
67
+ import { Logger } from '@aws-lambda-powertools/logger';
68
+
69
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
70
+
71
+ const schemaConfig = {
72
+ value: {
73
+ type: SchemaType.AVRO,
74
+ schema: readFileSync(new URL('./user.avsc', import.meta.url), 'utf8'),
75
+ },
76
+ } satisfies SchemaConfig;
77
+
78
+ export const handler = kafkaConsumer(async (event, _context) => {
79
+ for (const { value } of event.records) {
80
+ logger.info('received value', { value });
81
+ }
82
+ }, schemaConfig);
83
+ ```
84
+
85
+ #### Protobuf Schema
86
+
87
+ ```ts
88
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
89
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
90
+ import { Logger } from '@aws-lambda-powertools/logger';
91
+ import { User } from './samples/user.es6.generated.js'; // protobuf generated class
92
+
93
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
94
+
95
+ const schemaConfig = {
96
+ value: {
97
+ type: SchemaType.PROTOBUF,
98
+ schema: User,
99
+ },
100
+ } satisfies SchemaConfig;
101
+
102
+ export const handler = kafkaConsumer(async (event, _context) => {
103
+ for (const { value } of event.records) {
104
+ logger.info('received value', { value });
105
+ }
106
+ }, schemaConfig);
107
+ ```
108
+
109
+ ### Additional Parsing
110
+
111
+ You can parse deserialized data using your preferred parsing library. This can help you integrate Kafka data with your domain schemas and application architecture, providing type hints, runtime parsing and validation, and advanced data transformations.
112
+
113
+ #### Zod
114
+
115
+ ```ts
116
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
117
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
118
+ import { Logger } from '@aws-lambda-powertools/logger';
119
+ import { z } from 'zod/v4';
120
+
121
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
122
+
123
+ const OrderItemSchema = z.object({
124
+ productId: z.string(),
125
+ quantity: z.number().int().positive(),
126
+ price: z.number().positive(),
127
+ });
128
+
129
+ const OrderSchema = z.object({
130
+ id: z.string(),
131
+ customerId: z.string(),
132
+ items: z.array(OrderItemSchema).min(1, 'Order must have at least one item'),
133
+ createdAt: z.iso.datetime(),
134
+ totalAmount: z.number().positive(),
135
+ });
136
+
137
+ const schemaConfig = {
138
+ value: {
139
+ type: SchemaType.JSON,
140
+ parserSchema: OrderSchema,
141
+ },
142
+ } satisfies SchemaConfig;
143
+
144
+ export const handler = kafkaConsumer<unknown, z.infer<typeof OrderSchema>>(
145
+ async (event, _context) => {
146
+ for (const record of event.records) {
147
+ const {
148
+ value: { id, items },
149
+ } = record;
150
+ logger.setCorrelationId(id);
151
+ logger.debug(`order includes ${items.length} items`);
152
+ }
153
+ },
154
+ schemaConfig
155
+ );
156
+ ```
157
+
158
+ #### Valibot
159
+
160
+ ```ts
161
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
162
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
163
+ import { Logger } from '@aws-lambda-powertools/logger';
164
+ import * as v from 'valibot';
165
+
166
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
167
+
168
+ const OrderItemSchema = v.object({
169
+ productId: v.string(),
170
+ quantity: v.pipe(v.number(), v.integer(), v.toMinValue(1)),
171
+ price: v.pipe(v.number(), v.integer()),
172
+ });
173
+
174
+ const OrderSchema = v.object({
175
+ id: v.string(),
176
+ customerId: v.string(),
177
+ items: v.pipe(
178
+ v.array(OrderItemSchema),
179
+ v.minLength(1, 'Order must have at least one item')
180
+ ),
181
+ createdAt: v.pipe(v.string(), v.isoDateTime()),
182
+ totalAmount: v.pipe(v.number(), v.toMinValue(0)),
183
+ });
184
+
185
+ const schemaConfig = {
186
+ value: {
187
+ type: SchemaType.JSON,
188
+ parserSchema: OrderSchema,
189
+ },
190
+ } satisfies SchemaConfig;
191
+
192
+ export const handler = kafkaConsumer<unknown, v.InferInput<typeof OrderSchema>>(
193
+ async (event, _context) => {
194
+ for (const record of event.records) {
195
+ const {
196
+ value: { id, items },
197
+ } = record;
198
+ logger.setCorrelationId(id);
199
+ logger.debug(`order includes ${items.length} items`);
200
+ }
201
+ },
202
+ schemaConfig
203
+ );
204
+ ```
205
+
206
+ #### ArkType
207
+
208
+ ```ts
209
+ import { SchemaType, kafkaConsumer } from '@aws-lambda-powertools/kafka';
210
+ import type { SchemaConfig } from '@aws-lambda-powertools/kafka/types';
211
+ import { Logger } from '@aws-lambda-powertools/logger';
212
+ import { type } from 'arktype';
213
+
214
+ const logger = new Logger({ serviceName: 'kafka-consumer' });
215
+
216
+ const OrderItemSchema = type({
217
+ productId: 'string',
218
+ quantity: 'number.integer >= 1',
219
+ price: 'number.integer',
220
+ });
221
+
222
+ const OrderSchema = type({
223
+ id: 'string',
224
+ customerId: 'string',
225
+ items: OrderItemSchema.array().moreThanLength(0),
226
+ createdAt: 'string.date',
227
+ totalAmount: 'number.integer >= 0',
228
+ });
229
+
230
+ const schemaConfig = {
231
+ value: {
232
+ type: SchemaType.JSON,
233
+ parserSchema: OrderSchema,
234
+ },
235
+ } satisfies SchemaConfig;
236
+
237
+ export const handler = kafkaConsumer<unknown, typeof OrderSchema.infer>(
238
+ async (event, _context) => {
239
+ for (const record of event.records) {
240
+ const {
241
+ value: { id, items },
242
+ } = record;
243
+ logger.setCorrelationId(id);
244
+ logger.debug(`order includes ${items.length} items`);
245
+ }
246
+ },
247
+ schemaConfig
248
+ );
249
+ ```
250
+
251
+ See the [documentation](https://docs.powertools.aws.dev/lambda/typescript/latest/features/kafka) for more details on how to use the Kafka utility.
252
+
253
+ ## Contribute
254
+
255
+ If you are interested in contributing to this project, please refer to our [Contributing Guidelines](https://github.com/aws-powertools/powertools-lambda-typescript/blob/main/CONTRIBUTING.md).
256
+
257
+ ## Roadmap
258
+
259
+ The roadmap of Powertools for AWS Lambda (TypeScript) is driven by customers’ demand.
260
+ Help us prioritize upcoming functionalities or utilities by [upvoting existing RFCs and feature requests](https://github.com/aws-powertools/powertools-lambda-typescript/issues), or [creating new ones](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new/choose), in this GitHub repository.
261
+
262
+ ## Connect
263
+
264
+ * **Powertools for AWS Lambda on Discord**: `#typescript` - **[Invite link](https://discord.gg/B8zZKbbyET)**
265
+ * **Email**: <aws-lambda-powertools-feedback@amazon.com>
266
+
267
+ ## How to support Powertools for AWS Lambda (TypeScript)?
268
+
269
+ ### Becoming a reference customer
270
+
271
+ Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://s12d.com/become-reference-pt-ts) issue.
272
+
273
+ The following companies, among others, use Powertools:
274
+
275
+ * [Alma Media](https://www.almamedia.fi)
276
+ * [AppYourself](https://appyourself.net)
277
+ * [Bailey Nelson](https://www.baileynelson.com.au)
278
+ * [Banxware](https://www.banxware.com)
279
+ * [Caylent](https://caylent.com/)
280
+ * [Certible](https://www.certible.com/)
281
+ * [Elva](https://elva-group.com)
282
+ * [Flyweight](https://flyweight.io/)
283
+ * [globaldatanet](https://globaldatanet.com/)
284
+ * [Guild](https://guild.com)
285
+ * [Hashnode](https://hashnode.com/)
286
+ * [Instil](https://instil.co/)
287
+ * [LocalStack](https://localstack.cloud/)
288
+ * [Ours Privacy](https://oursprivacy.com/)
289
+ * [Perfect Post](https://www.perfectpost.fr)
290
+ * [Sennder](https://sennder.com/)
291
+ * [tecRacer GmbH & Co. KG](https://www.tecracer.com/)
292
+ * [Trek10](https://www.trek10.com/)
293
+ * [WeSchool](https://www.weschool.com)
294
+
295
+ ### Sharing your work
296
+
297
+ Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has [already shared](https://docs.powertools.aws.dev/lambda/typescript/latest/we_made_this) about Powertools for AWS Lambda (TypeScript).
298
+
299
+ ### Using Lambda Layer
300
+
301
+ This helps us understand who uses Powertools for AWS Lambda (TypeScript) in a non-intrusive way, and helps us gain future investments for other Powertools for AWS Lambda languages. When [using Layers](https://docs.powertools.aws.dev/lambda/typescript/latest/getting-started/lambda-layers/), you can add Powertools as a dev dependency to not impact the development process.
302
+
303
+ ## License
304
+
305
+ This library is licensed under the MIT-0 License. See the LICENSE file.
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Types of Kafka schema formats.
3
+ */
4
+ declare const SchemaType: {
5
+ readonly JSON: "json";
6
+ readonly AVRO: "avro";
7
+ readonly PROTOBUF: "protobuf";
8
+ };
9
+ export { SchemaType };
10
+ //# sourceMappingURL=constants.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,CAAC"}
@@ -0,0 +1,12 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SchemaType = void 0;
4
+ /**
5
+ * Types of Kafka schema formats.
6
+ */
7
+ const SchemaType = {
8
+ JSON: 'json',
9
+ AVRO: 'avro',
10
+ PROTOBUF: 'protobuf',
11
+ };
12
+ exports.SchemaType = SchemaType;
@@ -0,0 +1,42 @@
1
+ import type { AsyncHandler } from '@aws-lambda-powertools/commons/types';
2
+ import type { Context, Handler } from 'aws-lambda';
3
+ import type { ConsumerRecords, MSKEvent, SchemaConfig } from './types/types.js';
4
+ /**
5
+ * Wrap a handler function to automatically deserialize and validate Kafka records from an MSK event.
6
+ *
7
+ * The returned function will:
8
+ * - Deserialize the key and value of each record using the provided schema config.
9
+ * - Validate the deserialized key and value using Zod schemas if provided.
10
+ * - Replace the `records` property in the event with an array of deserialized and validated records.
11
+ * - Call the original handler with the modified event and original context/arguments.
12
+ *
13
+ * @example
14
+ * ```ts
15
+ * import { kafkaConsumer } from '@aws-lambda-powertools/kafka';
16
+ * import { z } from 'zod';
17
+ *
18
+ * const keySchema = z.string();
19
+ * const valueSchema = z.object({
20
+ * id: z.number(),
21
+ * });
22
+ *
23
+ * export const handler = kafkaConsumer<z.infer<keySchema>, z.infer<valueSchema>>(async (event, context) => {
24
+ * // event.records is now an array of deserialized and validated records
25
+ * for (const record of event.records) {
26
+ * console.log(record.key, record.value);
27
+ * }
28
+ * }, {
29
+ * key: { type: 'json', parserSchema: keySchema },
30
+ * value: { type: 'json', parserSchema: valueSchema },
31
+ * });
32
+ * ```
33
+ *
34
+ * @typeParam K - Optional type of the deserialized key - defaults to `unknown`.
35
+ * @typeParam V - Optional type of the deserialized value - defaults to `unknown`.
36
+ *
37
+ * @param handler - The original handler function to wrap. It should accept the deserialized event as its first argument.
38
+ * @param config - The schema configuration for deserializing and validating record keys and values.
39
+ */
40
+ declare const kafkaConsumer: <K, V>(handler: AsyncHandler<Handler<ConsumerRecords<K, V>>>, config?: SchemaConfig) => ((event: MSKEvent, context: Context) => Promise<unknown>);
41
+ export { kafkaConsumer };
42
+ //# sourceMappingURL=consumer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;AAsM1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,QAAA,MAAM,aAAa,GAAI,CAAC,EAAE,CAAC,EACzB,SAAS,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EACrD,SAAS,YAAY,KACpB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAwB1D,CAAC;AAEF,OAAO,EAAE,aAAa,EAAE,CAAC"}
@@ -0,0 +1,250 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.kafkaConsumer = void 0;
37
+ const typeutils_1 = require("@aws-lambda-powertools/commons/typeutils");
38
+ const json_js_1 = require("./deserializer/json.js");
39
+ const primitive_js_1 = require("./deserializer/primitive.js");
40
+ const errors_js_1 = require("./errors.js");
41
+ /**
42
+ * Type guard to assert that the event is a valid {@link MSKEvent | `MSKEvent`}.
43
+ *
44
+ * @param event - The event to validate, expected to be an MSKEvent.
45
+ */
46
+ const assertIsMSKEvent = (event) => {
47
+ if (!(0, typeutils_1.isRecord)(event) ||
48
+ !(0, typeutils_1.isRecord)(event.records) ||
49
+ !Object.values(event.records).every((arr) => Array.isArray(arr))) {
50
+ throw new errors_js_1.KafkaConsumerError('Event is not a valid MSKEvent. Expected an object with a "records" property.');
51
+ }
52
+ return true;
53
+ };
54
+ /**
55
+ * Deserialize Kafka message headers from an array of header objects.
56
+ *
57
+ * It returns `null` if the headers are `null`, or an array of header objects
58
+ * where each header value is decoded as a UTF-8 string.
59
+ *
60
+ * @param headers - An array of header objects, where each object maps header keys (string)
61
+ * to header values (`number[]`), representing the raw bytes of each header value -
62
+ * i.e. `[{ "headerKey": [104, 101, 108, 108, 111] }]`
63
+ */
64
+ const deserializeHeaders = (headers) => {
65
+ if (headers === null) {
66
+ return null;
67
+ }
68
+ const result = [];
69
+ for (const header of headers) {
70
+ const entries = [];
71
+ for (const [headerKey, headerValue] of Object.entries(header)) {
72
+ entries.push([headerKey, Buffer.from(headerValue).toString('utf-8')]);
73
+ }
74
+ result.push(Object.fromEntries(entries));
75
+ }
76
+ return result;
77
+ };
78
+ /**
79
+ * Deserialize a base64-encoded value using the provided schema configuration.
80
+ *
81
+ * It returns the deserialized value, which may be a string, object, or other type depending on the schema type.
82
+ *
83
+ * @param value - The base64-encoded string to deserialize.
84
+ * @param config - The schema configuration to use for deserialization. See {@link SchemaConfigValue | `SchemaConfigValue`}.
85
+ * If not provided, the value is decoded as a UTF-8 string.
86
+ */
87
+ const deserialize = ({ value, deserializer, config, schemaMetadata, }) => {
88
+ if (config === undefined) {
89
+ return deserializer(value);
90
+ }
91
+ if (config.type === 'json') {
92
+ return deserializer(value);
93
+ }
94
+ if (config.type === 'avro') {
95
+ if (!config.schema) {
96
+ throw new errors_js_1.KafkaConsumerMissingSchemaError('Schema string is required for avro deserialization');
97
+ }
98
+ return deserializer(value, config.schema);
99
+ }
100
+ if (config.type === 'protobuf') {
101
+ if (!config.schema) {
102
+ throw new errors_js_1.KafkaConsumerMissingSchemaError('Schema string is required for protobuf deserialization');
103
+ }
104
+ return deserializer(value, config.schema, schemaMetadata);
105
+ }
106
+ };
107
+ /**
108
+ * Get the deserializer function based on the provided type.
109
+ *
110
+ * @param type - The type of deserializer to use. Supported types are: `json`, `avro`, `protobuf`, or `undefined`.
111
+ * If `undefined`, it defaults to deserializing as a primitive string.
112
+ */
113
+ const getDeserializer = async (type) => {
114
+ if (!type) {
115
+ return primitive_js_1.deserialize;
116
+ }
117
+ if (type === 'json') {
118
+ return json_js_1.deserialize;
119
+ }
120
+ if (type === 'protobuf') {
121
+ const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/protobuf.js')));
122
+ return deserializer.deserialize;
123
+ }
124
+ if (type === 'avro') {
125
+ const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/avro.js')));
126
+ return deserializer.deserialize;
127
+ }
128
+ throw new errors_js_1.KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
129
+ };
130
+ /**
131
+ * Parse a value against a provided schema using the `~standard` property for validation.
132
+ *
133
+ * @param value - The value to parse against the schema.
134
+ * @param schema - The schema to validate against, which should be a {@link StandardSchemaV1 | `Standard Schema V1`} object.
135
+ */
136
+ const parseSchema = (value, schema) => {
137
+ const result = schema['~standard'].validate(value);
138
+ /* v8 ignore start */
139
+ if (result instanceof Promise)
140
+ throw new errors_js_1.KafkaConsumerParserError('Schema parsing supports only synchronous validation');
141
+ /* v8 ignore stop */
142
+ if (result.issues) {
143
+ throw new errors_js_1.KafkaConsumerParserError('Schema validation failed', {
144
+ cause: result.issues,
145
+ });
146
+ }
147
+ return result.value;
148
+ };
149
+ /**
150
+ * Deserialize a single record from an MSK event.
151
+ *
152
+ * @param record - A single record from the MSK event.
153
+ * @param config - The schema configuration for deserializing the record's key and value.
154
+ */
155
+ const deserializeRecord = async (record, config) => {
156
+ const { key, value, headers, valueSchemaMetadata, keySchemaMetadata, ...rest } = record;
157
+ const { key: keyConfig, value: valueConfig } = config || {};
158
+ const deserializerKey = await getDeserializer(keyConfig?.type);
159
+ const deserializerValue = await getDeserializer(valueConfig?.type);
160
+ return {
161
+ ...rest,
162
+ get key() {
163
+ if (key === undefined || key === '') {
164
+ return undefined;
165
+ }
166
+ if ((0, typeutils_1.isNull)(key))
167
+ return null;
168
+ const deserializedKey = deserialize({
169
+ value: key,
170
+ deserializer: deserializerKey,
171
+ config: keyConfig,
172
+ schemaMetadata: keySchemaMetadata,
173
+ });
174
+ return keyConfig?.parserSchema
175
+ ? parseSchema(deserializedKey, keyConfig.parserSchema)
176
+ : deserializedKey;
177
+ },
178
+ originalKey: key,
179
+ get value() {
180
+ const deserializedValue = deserialize({
181
+ value: value,
182
+ deserializer: deserializerValue,
183
+ config: valueConfig,
184
+ schemaMetadata: valueSchemaMetadata,
185
+ });
186
+ return valueConfig?.parserSchema
187
+ ? parseSchema(deserializedValue, valueConfig.parserSchema)
188
+ : deserializedValue;
189
+ },
190
+ originalValue: value,
191
+ get headers() {
192
+ return deserializeHeaders(headers);
193
+ },
194
+ originalHeaders: headers,
195
+ valueSchemaMetadata,
196
+ keySchemaMetadata,
197
+ };
198
+ };
199
+ /**
200
+ * Wrap a handler function to automatically deserialize and validate Kafka records from an MSK event.
201
+ *
202
+ * The returned function will:
203
+ * - Deserialize the key and value of each record using the provided schema config.
204
+ * - Validate the deserialized key and value using Zod schemas if provided.
205
+ * - Replace the `records` property in the event with an array of deserialized and validated records.
206
+ * - Call the original handler with the modified event and original context/arguments.
207
+ *
208
+ * @example
209
+ * ```ts
210
+ * import { kafkaConsumer } from '@aws-lambda-powertools/kafka';
211
+ * import { z } from 'zod';
212
+ *
213
+ * const keySchema = z.string();
214
+ * const valueSchema = z.object({
215
+ * id: z.number(),
216
+ * });
217
+ *
218
+ * export const handler = kafkaConsumer<z.infer<keySchema>, z.infer<valueSchema>>(async (event, context) => {
219
+ * // event.records is now an array of deserialized and validated records
220
+ * for (const record of event.records) {
221
+ * console.log(record.key, record.value);
222
+ * }
223
+ * }, {
224
+ * key: { type: 'json', parserSchema: keySchema },
225
+ * value: { type: 'json', parserSchema: valueSchema },
226
+ * });
227
+ * ```
228
+ *
229
+ * @typeParam K - Optional type of the deserialized key - defaults to `unknown`.
230
+ * @typeParam V - Optional type of the deserialized value - defaults to `unknown`.
231
+ *
232
+ * @param handler - The original handler function to wrap. It should accept the deserialized event as its first argument.
233
+ * @param config - The schema configuration for deserializing and validating record keys and values.
234
+ */
235
+ const kafkaConsumer = (handler, config) => {
236
+ return async (event, context) => {
237
+ assertIsMSKEvent(event);
238
+ const consumerRecords = [];
239
+ for (const recordsArray of Object.values(event.records)) {
240
+ for (const record of recordsArray) {
241
+ consumerRecords.push((await deserializeRecord(record, config)));
242
+ }
243
+ }
244
+ return handler({
245
+ ...event,
246
+ records: consumerRecords,
247
+ }, context);
248
+ };
249
+ };
250
+ exports.kafkaConsumer = kafkaConsumer;
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
3
+ *
4
+ * @param data - The base64-encoded string representing the Avro binary data.
5
+ * @param schema - The Avro schema as a JSON string.
6
+ */
7
+ declare const deserialize: (data: string, schema: string) => unknown;
8
+ export { deserialize };
9
+ //# sourceMappingURL=avro.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,YAUhD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
@@ -0,0 +1,25 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.deserialize = void 0;
7
+ const avro_js_1 = __importDefault(require("avro-js"));
8
+ const errors_js_1 = require("../errors.js");
9
+ /**
10
+ * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
11
+ *
12
+ * @param data - The base64-encoded string representing the Avro binary data.
13
+ * @param schema - The Avro schema as a JSON string.
14
+ */
15
+ const deserialize = (data, schema) => {
16
+ try {
17
+ const type = avro_js_1.default.parse(schema);
18
+ const buffer = Buffer.from(data, 'base64');
19
+ return type.fromBuffer(buffer);
20
+ }
21
+ catch (error) {
22
+ throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
23
+ }
24
+ };
25
+ exports.deserialize = deserialize;
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Deserialize a base64 encoded string into either a JSON object or plain string
3
+ *
4
+ * @param data - The base64 encoded string to deserialize
5
+ * @returns The deserialized data as either a JSON object or string
6
+ */
7
+ declare const deserialize: (data: string) => any;
8
+ export { deserialize };
9
+ //# sourceMappingURL=json.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,QAYhC,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}