@aws-lambda-powertools/kafka 2.34.0 → 2.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -261,7 +261,7 @@ Help us prioritize upcoming functionalities or utilities by [upvoting existing R
261
261
 
262
262
  ## Connect
263
263
 
264
- * **Powertools for AWS Lambda on Discord**: `#typescript` - **[Invite link](https://discord.gg/B8zZKbbyET)**
264
+ * **Powertools for AWS Lambda on GitHub Discussions**: [Join the conversation](https://github.com/aws-powertools/powertools-lambda-typescript/discussions)
265
265
  * **Email**: <aws-lambda-powertools-feedback@amazon.com>
266
266
 
267
267
  ## How to support Powertools for AWS Lambda (TypeScript)?
@@ -119,12 +119,12 @@ const getDeserializer = async (type) => {
119
119
  return json_js_1.deserialize;
120
120
  }
121
121
  if (type === 'protobuf') {
122
- const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/protobuf.js')));
123
- return deserializer.deserialize;
122
+ const { createDeserializer } = await Promise.resolve().then(() => __importStar(require('./deserializer/protobuf.js')));
123
+ return (await createDeserializer());
124
124
  }
125
125
  if (type === 'avro') {
126
- const deserializer = await Promise.resolve().then(() => __importStar(require('./deserializer/avro.js')));
127
- return deserializer.deserialize;
126
+ const { createDeserializer } = await Promise.resolve().then(() => __importStar(require('./deserializer/avro.js')));
127
+ return (await createDeserializer());
128
128
  }
129
129
  throw new errors_js_1.KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
130
130
  };
@@ -1,9 +1,12 @@
1
+ import type { AvroDeserializer } from '../types/types.js';
1
2
  /**
2
- * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
3
+ * Create an Avro deserializer, importing `avro-js` on first use.
3
4
  *
4
- * @param data - The base64-encoded string representing the Avro binary data.
5
- * @param schema - The Avro schema as a JSON string.
5
+ * The `avro-js` dependency is only required when deserializing Avro messages,
6
+ * so it's resolved at runtime through a non-literal specifier. This keeps
7
+ * bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
8
+ * it isn't installed - in builds that never use Avro.
6
9
  */
7
- declare const deserialize: (data: string, schema: string) => unknown;
8
- export { deserialize };
10
+ declare const createDeserializer: () => Promise<AvroDeserializer>;
11
+ export { createDeserializer };
9
12
  //# sourceMappingURL=avro.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,YAUhD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
1
+ {"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAI1D;;;;;;;GAOG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,gBAAgB,CA2B5D,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
@@ -1,25 +1,71 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
5
35
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.deserialize = void 0;
7
- const avro_js_1 = __importDefault(require("avro-js"));
36
+ exports.createDeserializer = void 0;
8
37
  const errors_js_1 = require("../errors.js");
38
+ let deserializer;
9
39
  /**
10
- * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
40
+ * Create an Avro deserializer, importing `avro-js` on first use.
11
41
  *
12
- * @param data - The base64-encoded string representing the Avro binary data.
13
- * @param schema - The Avro schema as a JSON string.
42
+ * The `avro-js` dependency is only required when deserializing Avro messages,
43
+ * so it's resolved at runtime through a non-literal specifier. This keeps
44
+ * bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
45
+ * it isn't installed - in builds that never use Avro.
14
46
  */
15
- const deserialize = (data, schema) => {
16
- try {
17
- const type = avro_js_1.default.parse(schema);
18
- const buffer = Buffer.from(data, 'base64');
19
- return type.fromBuffer(buffer);
20
- }
21
- catch (error) {
22
- throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
47
+ const createDeserializer = async () => {
48
+ if (deserializer !== undefined) {
49
+ return deserializer;
23
50
  }
51
+ const moduleName = 'avro-js';
52
+ const { default: avro } = await Promise.resolve(`${moduleName}`).then(s => __importStar(require(s)));
53
+ /**
54
+ * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
55
+ *
56
+ * @param data - The base64-encoded string representing the Avro binary data.
57
+ * @param schema - The Avro schema as a JSON string.
58
+ */
59
+ deserializer = (data, schema) => {
60
+ try {
61
+ const type = avro.parse(schema);
62
+ const buffer = Buffer.from(data, 'base64');
63
+ return type.fromBuffer(buffer);
64
+ }
65
+ catch (error) {
66
+ throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
67
+ }
68
+ };
69
+ return deserializer;
24
70
  };
25
- exports.deserialize = deserialize;
71
+ exports.createDeserializer = createDeserializer;
@@ -1,12 +1,13 @@
1
- import type { ProtobufMessage, SchemaMetadata } from '../types/types.js';
1
+ import type { ProtobufDeserializer } from '../types/types.js';
2
2
  /**
3
- * Deserialize a Protobuf message from a base64-encoded string.
3
+ * Create a Protobuf deserializer, importing `protobufjs` on first use.
4
4
  *
5
- * @template T - The type of the deserialized message object.
6
- *
7
- * @param data - The base64-encoded string representing the Protobuf binary data.
8
- * @param messageType - The Protobuf message type definition - see {@link Message | `Message`} from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
5
+ * The `protobufjs` dependency is only required when deserializing Protobuf
6
+ * messages, so it's resolved at runtime through a non-literal specifier. This
7
+ * keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
8
+ * failing when it isn't installed - in builds that never use Protobuf. See
9
+ * {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
9
10
  */
10
- declare const deserialize: <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
11
- export { deserialize };
11
+ declare const createDeserializer: () => Promise<ProtobufDeserializer>;
12
+ export { createDeserializer };
12
13
  //# sourceMappingURL=protobuf.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOzE;;;;;;;GAOG;AACH,QAAA,MAAM,WAAW,GAAI,CAAC,EACpB,MAAM,MAAM,EACZ,aAAa,eAAe,CAAC,CAAC,CAAC,EAC/B,gBAAgB,cAAc,KAC7B,CAiDF,CAAC;AAuBF,OAAO,EAAE,WAAW,EAAE,CAAC"}
1
+ {"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EAGrB,MAAM,mBAAmB,CAAC;AAI3B;;;;;;;;GAQG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,oBAAoB,CAuGhE,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
@@ -1,80 +1,130 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
2
35
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.deserialize = void 0;
4
- const protobufjs_1 = require("protobufjs");
36
+ exports.createDeserializer = void 0;
5
37
  const errors_js_1 = require("../errors.js");
38
+ let deserializer;
6
39
  /**
7
- * Default order of varint types used in Protobuf to attempt deserializing Confluent Schema Registry messages.
8
- */
9
- const varintOrder = ['int32', 'sint32'];
10
- /**
11
- * Deserialize a Protobuf message from a base64-encoded string.
12
- *
13
- * @template T - The type of the deserialized message object.
40
+ * Create a Protobuf deserializer, importing `protobufjs` on first use.
14
41
  *
15
- * @param data - The base64-encoded string representing the Protobuf binary data.
16
- * @param messageType - The Protobuf message type definition - see {@link Message | `Message`} from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
42
+ * The `protobufjs` dependency is only required when deserializing Protobuf
43
+ * messages, so it's resolved at runtime through a non-literal specifier. This
44
+ * keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
45
+ * failing when it isn't installed - in builds that never use Protobuf. See
46
+ * {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
17
47
  */
18
- const deserialize = (data, messageType, schemaMetadata) => {
19
- const buffer = Buffer.from(data, 'base64');
20
- try {
21
- if (schemaMetadata.schemaId === undefined) {
22
- return messageType.decode(buffer, buffer.length);
23
- }
48
+ const createDeserializer = async () => {
49
+ if (deserializer !== undefined) {
50
+ return deserializer;
51
+ }
52
+ const moduleName = 'protobufjs';
53
+ const { BufferReader } = await Promise.resolve(`${moduleName}`).then(s => __importStar(require(s)));
54
+ /**
55
+ * Default order of varint types used in Protobuf to attempt deserializing Confluent Schema Registry messages.
56
+ */
57
+ const varintOrder = ['int32', 'sint32'];
58
+ /**
59
+ * Clip the Confluent Schema Registry buffer to remove the index bytes.
60
+ *
61
+ * @param buffer - The buffer to clip.
62
+ * @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
63
+ */
64
+ const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
65
+ const reader = new BufferReader(buffer);
24
66
  /**
25
- * If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
26
- *
27
- * When this is the case, we know the schema is coming from Glue Schema Registry,
28
- * and the first byte of the buffer is a magic byte that we need to remove before
29
- * decoding the message.
67
+ * Read the first varint byte to get the index count or 0.
68
+ * Doing so, also advances the reader position to the next byte after the index count.
30
69
  */
31
- if (schemaMetadata.schemaId.length > 10) {
32
- // remove the first byte from the buffer
33
- const reader = new protobufjs_1.BufferReader(buffer);
34
- reader.uint32();
35
- return messageType.decode(reader);
36
- }
37
- }
38
- catch (error) {
39
- throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
40
- }
70
+ const indexCount = intType === 'int32' ? reader.int32() : reader.sint32();
71
+ // Skip the index bytes
72
+ reader.skip(indexCount);
73
+ return reader;
74
+ };
41
75
  /**
42
- * If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
43
- * so we need to remove the MessageIndex bytes.
44
- * We don't know the type of the index, so we try both `int32` and `sint32`. If both fail, we throw an error.
76
+ * Deserialize a Protobuf message from a base64-encoded string.
77
+ *
78
+ * @template T - The type of the deserialized message object.
79
+ *
80
+ * @param data - The base64-encoded string representing the Protobuf binary data.
81
+ * @param messageType - The Protobuf message type definition - see the `Message` type from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
45
82
  */
46
- try {
47
- const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[0]);
48
- return messageType.decode(newBuffer);
49
- }
50
- catch (error) {
83
+ deserializer = (data, messageType, schemaMetadata) => {
84
+ const buffer = Buffer.from(data, 'base64');
51
85
  try {
52
- const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
53
- const decoded = messageType.decode(newBuffer);
54
- // swap varint order if the first attempt failed so we can use the correct one for subsequent messages
55
- varintOrder.reverse();
56
- return decoded;
86
+ if (schemaMetadata.schemaId === undefined) {
87
+ return messageType.decode(buffer, buffer.length);
88
+ }
89
+ /**
90
+ * If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
91
+ *
92
+ * When this is the case, we know the schema is coming from Glue Schema Registry,
93
+ * and the first byte of the buffer is a magic byte that we need to remove before
94
+ * decoding the message.
95
+ */
96
+ if (schemaMetadata.schemaId.length > 10) {
97
+ // remove the first byte from the buffer
98
+ const reader = new BufferReader(buffer);
99
+ reader.uint32();
100
+ return messageType.decode(reader);
101
+ }
57
102
  }
58
- catch {
103
+ catch (error) {
59
104
  throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
60
105
  }
61
- }
62
- };
63
- exports.deserialize = deserialize;
64
- /**
65
- * Clip the Confluent Schema Registry buffer to remove the index bytes.
66
- *
67
- * @param buffer - The buffer to clip.
68
- * @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
69
- */
70
- const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
71
- const reader = new protobufjs_1.BufferReader(buffer);
72
- /**
73
- * Read the first varint byte to get the index count or 0.
74
- * Doing so, also advances the reader position to the next byte after the index count.
75
- */
76
- const indexCount = intType === 'int32' ? reader.int32() : reader.sint32();
77
- // Skip the index bytes
78
- reader.skip(indexCount);
79
- return reader;
106
+ /**
107
+ * If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
108
+ * so we need to remove the MessageIndex bytes.
109
+ * We don't know the type of the index, so we try both `int32` and `sint32`. If both fail, we throw an error.
110
+ */
111
+ try {
112
+ const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[0]);
113
+ return messageType.decode(newBuffer);
114
+ }
115
+ catch (error) {
116
+ try {
117
+ const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
118
+ const decoded = messageType.decode(newBuffer);
119
+ // swap varint order if the first attempt failed so we can use the correct one for subsequent messages
120
+ varintOrder.reverse();
121
+ return decoded;
122
+ }
123
+ catch {
124
+ throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
125
+ }
126
+ }
127
+ };
128
+ return deserializer;
80
129
  };
130
+ exports.createDeserializer = createDeserializer;
@@ -234,11 +234,13 @@ type ProtobufMessage<T = unknown> = {
234
234
  decode(reader: Reader | Uint8Array, length?: number): T;
235
235
  };
236
236
  type Deserializer = (input: string | null, schema?: unknown, schemaMetadata?: SchemaMetadata) => unknown;
237
+ type AvroDeserializer = (data: string, schema: string) => unknown;
238
+ type ProtobufDeserializer = <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
237
239
  type DeserializeOptions = {
238
240
  value: string | null;
239
241
  deserializer: Deserializer;
240
242
  config?: SchemaConfigValue;
241
243
  schemaMetadata: SchemaMetadata;
242
244
  };
243
- export type { ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
245
+ export type { AvroDeserializer, ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufDeserializer, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
244
246
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,UAAU,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,cAAc,CAAC,EAAE,cAAc,KAC5B,OAAO,CAAC;AAEb,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,CAAC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,UAAU,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,cAAc,CAAC,EAAE,cAAc,KAC5B,OAAO,CAAC;AAEb,KAAK,gBAAgB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;AAElE,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAC5B,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,EAC/B,cAAc,EAAE,cAAc,KAC3B,CAAC,CAAC;AAEP,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,CAAC"}
@@ -83,12 +83,12 @@ const getDeserializer = async (type) => {
83
83
  return deserializeJson;
84
84
  }
85
85
  if (type === 'protobuf') {
86
- const deserializer = await import('./deserializer/protobuf.js');
87
- return deserializer.deserialize;
86
+ const { createDeserializer } = await import('./deserializer/protobuf.js');
87
+ return (await createDeserializer());
88
88
  }
89
89
  if (type === 'avro') {
90
- const deserializer = await import('./deserializer/avro.js');
91
- return deserializer.deserialize;
90
+ const { createDeserializer } = await import('./deserializer/avro.js');
91
+ return (await createDeserializer());
92
92
  }
93
93
  throw new KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
94
94
  };
@@ -1,9 +1,12 @@
1
+ import type { AvroDeserializer } from '../types/types.js';
1
2
  /**
2
- * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
3
+ * Create an Avro deserializer, importing `avro-js` on first use.
3
4
  *
4
- * @param data - The base64-encoded string representing the Avro binary data.
5
- * @param schema - The Avro schema as a JSON string.
5
+ * The `avro-js` dependency is only required when deserializing Avro messages,
6
+ * so it's resolved at runtime through a non-literal specifier. This keeps
7
+ * bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
8
+ * it isn't installed - in builds that never use Avro.
6
9
  */
7
- declare const deserialize: (data: string, schema: string) => unknown;
8
- export { deserialize };
10
+ declare const createDeserializer: () => Promise<AvroDeserializer>;
11
+ export { createDeserializer };
9
12
  //# sourceMappingURL=avro.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,EAAE,QAAQ,MAAM,YAUhD,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
1
+ {"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAI1D;;;;;;;GAOG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,gBAAgB,CA2B5D,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
@@ -1,19 +1,35 @@
1
- import avro from 'avro-js';
2
1
  import { KafkaConsumerDeserializationError } from '../errors.js';
2
+ let deserializer;
3
3
  /**
4
- * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
4
+ * Create an Avro deserializer, importing `avro-js` on first use.
5
5
  *
6
- * @param data - The base64-encoded string representing the Avro binary data.
7
- * @param schema - The Avro schema as a JSON string.
6
+ * The `avro-js` dependency is only required when deserializing Avro messages,
7
+ * so it's resolved at runtime through a non-literal specifier. This keeps
8
+ * bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
9
+ * it isn't installed - in builds that never use Avro.
8
10
  */
9
- const deserialize = (data, schema) => {
10
- try {
11
- const type = avro.parse(schema);
12
- const buffer = Buffer.from(data, 'base64');
13
- return type.fromBuffer(buffer);
14
- }
15
- catch (error) {
16
- throw new KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
11
+ const createDeserializer = async () => {
12
+ if (deserializer !== undefined) {
13
+ return deserializer;
17
14
  }
15
+ const moduleName = 'avro-js';
16
+ const { default: avro } = await import(moduleName);
17
+ /**
18
+ * Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
19
+ *
20
+ * @param data - The base64-encoded string representing the Avro binary data.
21
+ * @param schema - The Avro schema as a JSON string.
22
+ */
23
+ deserializer = (data, schema) => {
24
+ try {
25
+ const type = avro.parse(schema);
26
+ const buffer = Buffer.from(data, 'base64');
27
+ return type.fromBuffer(buffer);
28
+ }
29
+ catch (error) {
30
+ throw new KafkaConsumerDeserializationError(`Failed to deserialize Avro message: ${error}, message: ${data}, schema: ${schema}`);
31
+ }
32
+ };
33
+ return deserializer;
18
34
  };
19
- export { deserialize };
35
+ export { createDeserializer };
@@ -1,12 +1,13 @@
1
- import type { ProtobufMessage, SchemaMetadata } from '../types/types.js';
1
+ import type { ProtobufDeserializer } from '../types/types.js';
2
2
  /**
3
- * Deserialize a Protobuf message from a base64-encoded string.
3
+ * Create a Protobuf deserializer, importing `protobufjs` on first use.
4
4
  *
5
- * @template T - The type of the deserialized message object.
6
- *
7
- * @param data - The base64-encoded string representing the Protobuf binary data.
8
- * @param messageType - The Protobuf message type definition - see {@link Message | `Message`} from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
5
+ * The `protobufjs` dependency is only required when deserializing Protobuf
6
+ * messages, so it's resolved at runtime through a non-literal specifier. This
7
+ * keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
8
+ * failing when it isn't installed - in builds that never use Protobuf. See
9
+ * {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
9
10
  */
10
- declare const deserialize: <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
11
- export { deserialize };
11
+ declare const createDeserializer: () => Promise<ProtobufDeserializer>;
12
+ export { createDeserializer };
12
13
  //# sourceMappingURL=protobuf.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOzE;;;;;;;GAOG;AACH,QAAA,MAAM,WAAW,GAAI,CAAC,EACpB,MAAM,MAAM,EACZ,aAAa,eAAe,CAAC,CAAC,CAAC,EAC/B,gBAAgB,cAAc,KAC7B,CAiDF,CAAC;AAuBF,OAAO,EAAE,WAAW,EAAE,CAAC"}
1
+ {"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EAGrB,MAAM,mBAAmB,CAAC;AAI3B;;;;;;;;GAQG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,oBAAoB,CAuGhE,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
@@ -1,77 +1,94 @@
1
- import { BufferReader } from 'protobufjs';
2
1
  import { KafkaConsumerDeserializationError } from '../errors.js';
2
+ let deserializer;
3
3
  /**
4
- * Default order of varint types used in Protobuf to attempt deserializing Confluent Schema Registry messages.
5
- */
6
- const varintOrder = ['int32', 'sint32'];
7
- /**
8
- * Deserialize a Protobuf message from a base64-encoded string.
4
+ * Create a Protobuf deserializer, importing `protobufjs` on first use.
9
5
  *
10
- * @template T - The type of the deserialized message object.
11
- *
12
- * @param data - The base64-encoded string representing the Protobuf binary data.
13
- * @param messageType - The Protobuf message type definition - see {@link Message | `Message`} from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
6
+ * The `protobufjs` dependency is only required when deserializing Protobuf
7
+ * messages, so it's resolved at runtime through a non-literal specifier. This
8
+ * keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
9
+ * failing when it isn't installed - in builds that never use Protobuf. See
10
+ * {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
14
11
  */
15
- const deserialize = (data, messageType, schemaMetadata) => {
16
- const buffer = Buffer.from(data, 'base64');
17
- try {
18
- if (schemaMetadata.schemaId === undefined) {
19
- return messageType.decode(buffer, buffer.length);
20
- }
12
+ const createDeserializer = async () => {
13
+ if (deserializer !== undefined) {
14
+ return deserializer;
15
+ }
16
+ const moduleName = 'protobufjs';
17
+ const { BufferReader } = await import(moduleName);
18
+ /**
19
+ * Default order of varint types used in Protobuf to attempt deserializing Confluent Schema Registry messages.
20
+ */
21
+ const varintOrder = ['int32', 'sint32'];
22
+ /**
23
+ * Clip the Confluent Schema Registry buffer to remove the index bytes.
24
+ *
25
+ * @param buffer - The buffer to clip.
26
+ * @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
27
+ */
28
+ const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
29
+ const reader = new BufferReader(buffer);
21
30
  /**
22
- * If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
23
- *
24
- * When this is the case, we know the schema is coming from Glue Schema Registry,
25
- * and the first byte of the buffer is a magic byte that we need to remove before
26
- * decoding the message.
31
+ * Read the first varint byte to get the index count or 0.
32
+ * Doing so, also advances the reader position to the next byte after the index count.
27
33
  */
28
- if (schemaMetadata.schemaId.length > 10) {
29
- // remove the first byte from the buffer
30
- const reader = new BufferReader(buffer);
31
- reader.uint32();
32
- return messageType.decode(reader);
33
- }
34
- }
35
- catch (error) {
36
- throw new KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
37
- }
34
+ const indexCount = intType === 'int32' ? reader.int32() : reader.sint32();
35
+ // Skip the index bytes
36
+ reader.skip(indexCount);
37
+ return reader;
38
+ };
38
39
  /**
39
- * If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
40
- * so we need to remove the MessageIndex bytes.
41
- * We don't know the type of the index, so we try both `int32` and `sint32`. If both fail, we throw an error.
40
+ * Deserialize a Protobuf message from a base64-encoded string.
41
+ *
42
+ * @template T - The type of the deserialized message object.
43
+ *
44
+ * @param data - The base64-encoded string representing the Protobuf binary data.
45
+ * @param messageType - The Protobuf message type definition - see the `Message` type from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
42
46
  */
43
- try {
44
- const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[0]);
45
- return messageType.decode(newBuffer);
46
- }
47
- catch (error) {
47
+ deserializer = (data, messageType, schemaMetadata) => {
48
+ const buffer = Buffer.from(data, 'base64');
48
49
  try {
49
- const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
50
- const decoded = messageType.decode(newBuffer);
51
- // swap varint order if the first attempt failed so we can use the correct one for subsequent messages
52
- varintOrder.reverse();
53
- return decoded;
50
+ if (schemaMetadata.schemaId === undefined) {
51
+ return messageType.decode(buffer, buffer.length);
52
+ }
53
+ /**
54
+ * If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
55
+ *
56
+ * When this is the case, we know the schema is coming from Glue Schema Registry,
57
+ * and the first byte of the buffer is a magic byte that we need to remove before
58
+ * decoding the message.
59
+ */
60
+ if (schemaMetadata.schemaId.length > 10) {
61
+ // remove the first byte from the buffer
62
+ const reader = new BufferReader(buffer);
63
+ reader.uint32();
64
+ return messageType.decode(reader);
65
+ }
54
66
  }
55
- catch {
67
+ catch (error) {
56
68
  throw new KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
57
69
  }
58
- }
59
- };
60
- /**
61
- * Clip the Confluent Schema Registry buffer to remove the index bytes.
62
- *
63
- * @param buffer - The buffer to clip.
64
- * @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
65
- */
66
- const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
67
- const reader = new BufferReader(buffer);
68
- /**
69
- * Read the first varint byte to get the index count or 0.
70
- * Doing so, also advances the reader position to the next byte after the index count.
71
- */
72
- const indexCount = intType === 'int32' ? reader.int32() : reader.sint32();
73
- // Skip the index bytes
74
- reader.skip(indexCount);
75
- return reader;
70
+ /**
71
+ * If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
72
+ * so we need to remove the MessageIndex bytes.
73
+ * We don't know the type of the index, so we try both `int32` and `sint32`. If both fail, we throw an error.
74
+ */
75
+ try {
76
+ const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[0]);
77
+ return messageType.decode(newBuffer);
78
+ }
79
+ catch (error) {
80
+ try {
81
+ const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
82
+ const decoded = messageType.decode(newBuffer);
83
+ // swap varint order if the first attempt failed so we can use the correct one for subsequent messages
84
+ varintOrder.reverse();
85
+ return decoded;
86
+ }
87
+ catch {
88
+ throw new KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
89
+ }
90
+ }
91
+ };
92
+ return deserializer;
76
93
  };
77
- export { deserialize };
94
+ export { createDeserializer };
@@ -234,11 +234,13 @@ type ProtobufMessage<T = unknown> = {
234
234
  decode(reader: Reader | Uint8Array, length?: number): T;
235
235
  };
236
236
  type Deserializer = (input: string | null, schema?: unknown, schemaMetadata?: SchemaMetadata) => unknown;
237
+ type AvroDeserializer = (data: string, schema: string) => unknown;
238
+ type ProtobufDeserializer = <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
237
239
  type DeserializeOptions = {
238
240
  value: string | null;
239
241
  deserializer: Deserializer;
240
242
  config?: SchemaConfigValue;
241
243
  schemaMetadata: SchemaMetadata;
242
244
  };
243
- export type { ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
245
+ export type { AvroDeserializer, ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufDeserializer, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
244
246
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,UAAU,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,cAAc,CAAC,EAAE,cAAc,KAC5B,OAAO,CAAC;AAEb,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,CAAC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,UAAU,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,cAAc,CAAC,EAAE,cAAc,KAC5B,OAAO,CAAC;AAEb,KAAK,gBAAgB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;AAElE,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAC5B,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,EAC/B,cAAc,EAAE,cAAc,KAC3B,CAAC,CAAC;AAEP,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aws-lambda-powertools/kafka",
3
- "version": "2.34.0",
3
+ "version": "2.36.0",
4
4
  "description": "Utility to easily handle message deserialization and parsing of Kafka events in AWS Lambda functions",
5
5
  "author": {
6
6
  "name": "Amazon Web Services",
@@ -27,12 +27,14 @@
27
27
  "nodejs"
28
28
  ],
29
29
  "dependencies": {
30
- "@aws-lambda-powertools/commons": "2.34.0",
30
+ "@aws-lambda-powertools/commons": "2.36.0",
31
31
  "@standard-schema/spec": "^1.1.0"
32
32
  },
33
33
  "peerDependencies": {
34
- "arktype": "^2.2.3",
35
- "valibot": "^1.4.2",
34
+ "arktype": "^2.2.7",
35
+ "avro-js": "^1.12.1",
36
+ "protobufjs": "^8.8.0",
37
+ "valibot": "^1.5.0",
36
38
  "zod": "^3.25.0 || ^4.0.0"
37
39
  },
38
40
  "peerDependenciesMeta": {
@@ -44,6 +46,12 @@
44
46
  },
45
47
  "arktype": {
46
48
  "optional": true
49
+ },
50
+ "avro-js": {
51
+ "optional": true
52
+ },
53
+ "protobufjs": {
54
+ "optional": true
47
55
  }
48
56
  },
49
57
  "files": [