@aws-lambda-powertools/kafka 2.35.0 → 2.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/constants.d.ts +7 -1
- package/lib/cjs/constants.d.ts.map +1 -1
- package/lib/cjs/constants.js +8 -1
- package/lib/cjs/consumer.d.ts.map +1 -1
- package/lib/cjs/consumer.js +7 -5
- package/lib/cjs/deserializer/avro.d.ts +8 -5
- package/lib/cjs/deserializer/avro.d.ts.map +1 -1
- package/lib/cjs/deserializer/avro.js +100 -16
- package/lib/cjs/deserializer/json.d.ts.map +1 -1
- package/lib/cjs/deserializer/json.js +2 -4
- package/lib/cjs/deserializer/protobuf.d.ts +9 -8
- package/lib/cjs/deserializer/protobuf.d.ts.map +1 -1
- package/lib/cjs/deserializer/protobuf.js +98 -66
- package/lib/cjs/index.d.ts +1 -1
- package/lib/cjs/index.d.ts.map +1 -1
- package/lib/cjs/index.js +2 -1
- package/lib/cjs/types/index.d.ts +1 -1
- package/lib/cjs/types/index.d.ts.map +1 -1
- package/lib/cjs/types/types.d.ts +19 -3
- package/lib/cjs/types/types.d.ts.map +1 -1
- package/lib/esm/constants.d.ts +7 -1
- package/lib/esm/constants.d.ts.map +1 -1
- package/lib/esm/constants.js +7 -1
- package/lib/esm/consumer.d.ts.map +1 -1
- package/lib/esm/consumer.js +7 -5
- package/lib/esm/deserializer/avro.d.ts +8 -5
- package/lib/esm/deserializer/avro.d.ts.map +1 -1
- package/lib/esm/deserializer/avro.js +66 -12
- package/lib/esm/deserializer/json.d.ts.map +1 -1
- package/lib/esm/deserializer/json.js +2 -4
- package/lib/esm/deserializer/protobuf.d.ts +9 -8
- package/lib/esm/deserializer/protobuf.d.ts.map +1 -1
- package/lib/esm/deserializer/protobuf.js +64 -65
- package/lib/esm/index.d.ts +1 -1
- package/lib/esm/index.d.ts.map +1 -1
- package/lib/esm/index.js +1 -1
- package/lib/esm/types/index.d.ts +1 -1
- package/lib/esm/types/index.d.ts.map +1 -1
- package/lib/esm/types/types.d.ts +19 -3
- package/lib/esm/types/types.d.ts.map +1 -1
- package/package.json +12 -4
package/lib/cjs/constants.d.ts
CHANGED
|
@@ -6,5 +6,11 @@ declare const SchemaType: {
|
|
|
6
6
|
readonly AVRO: "avro";
|
|
7
7
|
readonly PROTOBUF: "protobuf";
|
|
8
8
|
};
|
|
9
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Wire formats supported for Kafka payloads.
|
|
11
|
+
*/
|
|
12
|
+
declare const WireFormat: {
|
|
13
|
+
readonly CONFLUENT: "confluent";
|
|
14
|
+
};
|
|
15
|
+
export { SchemaType, WireFormat };
|
|
10
16
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX;;GAEG;AACH,QAAA,MAAM,UAAU;;CAEN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
|
package/lib/cjs/constants.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SchemaType = void 0;
|
|
3
|
+
exports.WireFormat = exports.SchemaType = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* Types of Kafka schema formats.
|
|
6
6
|
*/
|
|
@@ -10,3 +10,10 @@ const SchemaType = {
|
|
|
10
10
|
PROTOBUF: 'protobuf',
|
|
11
11
|
};
|
|
12
12
|
exports.SchemaType = SchemaType;
|
|
13
|
+
/**
|
|
14
|
+
* Wire formats supported for Kafka payloads.
|
|
15
|
+
*/
|
|
16
|
+
const WireFormat = {
|
|
17
|
+
CONFLUENT: 'confluent',
|
|
18
|
+
};
|
|
19
|
+
exports.WireFormat = WireFormat;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;AA6M1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,QAAA,MAAM,aAAa,GAAI,CAAC,EAAE,CAAC,EACzB,SAAS,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EACrD,SAAS,YAAY,KACpB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAwB1D,CAAC;AAEF,OAAO,EAAE,aAAa,EAAE,CAAC"}
|
package/lib/cjs/consumer.js
CHANGED
|
@@ -95,7 +95,9 @@ const deserialize = ({ value, deserializer, config, schemaMetadata, }) => {
|
|
|
95
95
|
if (!config.schema) {
|
|
96
96
|
throw new errors_js_1.KafkaConsumerMissingSchemaError('Schema string is required for avro deserialization');
|
|
97
97
|
}
|
|
98
|
-
return deserializer(value, config.schema
|
|
98
|
+
return deserializer(value, config.schema, {
|
|
99
|
+
wireFormat: config.wireFormat,
|
|
100
|
+
});
|
|
99
101
|
}
|
|
100
102
|
/* v8 ignore else -- @preserve */
|
|
101
103
|
if (config.type === 'protobuf') {
|
|
@@ -119,12 +121,12 @@ const getDeserializer = async (type) => {
|
|
|
119
121
|
return json_js_1.deserialize;
|
|
120
122
|
}
|
|
121
123
|
if (type === 'protobuf') {
|
|
122
|
-
const
|
|
123
|
-
return
|
|
124
|
+
const { createDeserializer } = await Promise.resolve().then(() => __importStar(require('./deserializer/protobuf.js')));
|
|
125
|
+
return (await createDeserializer());
|
|
124
126
|
}
|
|
125
127
|
if (type === 'avro') {
|
|
126
|
-
const
|
|
127
|
-
return
|
|
128
|
+
const { createDeserializer } = await Promise.resolve().then(() => __importStar(require('./deserializer/avro.js')));
|
|
129
|
+
return (await createDeserializer());
|
|
128
130
|
}
|
|
129
131
|
throw new errors_js_1.KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
|
|
130
132
|
};
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import type { AvroDeserializer } from '../types/types.js';
|
|
1
2
|
/**
|
|
2
|
-
*
|
|
3
|
+
* Create an Avro deserializer, importing `avro-js` on first use.
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
5
|
+
* The `avro-js` dependency is only required when deserializing Avro messages,
|
|
6
|
+
* so it's resolved at runtime through a non-literal specifier. This keeps
|
|
7
|
+
* bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
|
|
8
|
+
* it isn't installed - in builds that never use Avro.
|
|
6
9
|
*/
|
|
7
|
-
declare const
|
|
8
|
-
export {
|
|
10
|
+
declare const createDeserializer: () => Promise<AvroDeserializer>;
|
|
11
|
+
export { createDeserializer };
|
|
9
12
|
//# sourceMappingURL=avro.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAoC1D;;;;;;;GAOG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,gBAAgB,CA2C5D,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -1,25 +1,109 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
5
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
7
|
-
const
|
|
36
|
+
exports.createDeserializer = void 0;
|
|
37
|
+
const constants_js_1 = require("../constants.js");
|
|
8
38
|
const errors_js_1 = require("../errors.js");
|
|
39
|
+
const CONFLUENT_HEADER_SIZE = 5;
|
|
40
|
+
const CONFLUENT_MAGIC_BYTE = 0x00;
|
|
41
|
+
let deserializer;
|
|
9
42
|
/**
|
|
10
|
-
*
|
|
43
|
+
* Strip the Confluent wire format header (magic byte and 4-byte schema ID) from an Avro payload.
|
|
11
44
|
*
|
|
12
|
-
* @param
|
|
13
|
-
* @param
|
|
45
|
+
* @param buffer - The decoded Avro payload.
|
|
46
|
+
* @param wireFormat - The wire format of the payload. If not provided, the payload is returned as is.
|
|
14
47
|
*/
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
48
|
+
const stripWireFormatHeader = (buffer, wireFormat) => {
|
|
49
|
+
if (wireFormat === undefined) {
|
|
50
|
+
return buffer;
|
|
51
|
+
}
|
|
52
|
+
// Guards untyped (plain JavaScript) callers
|
|
53
|
+
if (wireFormat !== constants_js_1.WireFormat.CONFLUENT) {
|
|
54
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Unsupported Avro wire format: ${wireFormat}`);
|
|
55
|
+
}
|
|
56
|
+
if (buffer.length < CONFLUENT_HEADER_SIZE) {
|
|
57
|
+
throw new errors_js_1.KafkaConsumerDeserializationError('Invalid Confluent wire format: payload must contain a 5-byte header');
|
|
58
|
+
}
|
|
59
|
+
if (buffer[0] !== CONFLUENT_MAGIC_BYTE) {
|
|
60
|
+
throw new errors_js_1.KafkaConsumerDeserializationError('Invalid Confluent wire format: expected magic byte 0x00');
|
|
20
61
|
}
|
|
21
|
-
|
|
22
|
-
|
|
62
|
+
return buffer.subarray(CONFLUENT_HEADER_SIZE);
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Create an Avro deserializer, importing `avro-js` on first use.
|
|
66
|
+
*
|
|
67
|
+
* The `avro-js` dependency is only required when deserializing Avro messages,
|
|
68
|
+
* so it's resolved at runtime through a non-literal specifier. This keeps
|
|
69
|
+
* bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
|
|
70
|
+
* it isn't installed - in builds that never use Avro.
|
|
71
|
+
*/
|
|
72
|
+
const createDeserializer = async () => {
|
|
73
|
+
if (deserializer !== undefined) {
|
|
74
|
+
return deserializer;
|
|
23
75
|
}
|
|
76
|
+
const moduleName = 'avro-js';
|
|
77
|
+
const { default: avro } = await Promise.resolve(`${moduleName}`).then(s => __importStar(require(s)));
|
|
78
|
+
/**
|
|
79
|
+
* Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
|
|
80
|
+
*
|
|
81
|
+
* @param data - The base64-encoded string representing the Avro binary data.
|
|
82
|
+
* @param schema - The Avro schema as a JSON string.
|
|
83
|
+
* @param options - Deserialization options. If `options.wireFormat` is `confluent`, the 5-byte header is stripped.
|
|
84
|
+
*/
|
|
85
|
+
deserializer = (data, schema, options) => {
|
|
86
|
+
const decodedPayload = Buffer.from(data, 'base64');
|
|
87
|
+
const buffer = stripWireFormatHeader(decodedPayload, options?.wireFormat);
|
|
88
|
+
let type;
|
|
89
|
+
try {
|
|
90
|
+
type = avro.parse(schema);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message (base64-decoded payload length: ${decodedPayload.length} bytes): invalid Avro schema`);
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
// Most Confluent serializers write top-level bytes without the Avro length prefix, so return them as-is
|
|
97
|
+
if (options?.wireFormat === constants_js_1.WireFormat.CONFLUENT &&
|
|
98
|
+
type.getName(true) === 'bytes') {
|
|
99
|
+
return buffer;
|
|
100
|
+
}
|
|
101
|
+
return type.fromBuffer(buffer);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Avro message (base64-decoded payload length: ${decodedPayload.length} bytes): ${error}`);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
return deserializer;
|
|
24
108
|
};
|
|
25
|
-
exports.
|
|
109
|
+
exports.createDeserializer = createDeserializer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,QAUhC,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
|
@@ -15,10 +15,8 @@ const deserialize = (data) => {
|
|
|
15
15
|
// we assume it's a JSON but it can also be a string, we don't know
|
|
16
16
|
return JSON.parse(plainText);
|
|
17
17
|
}
|
|
18
|
-
catch
|
|
19
|
-
// If JSON parsing fails,
|
|
20
|
-
// in case we could not parse it we return the base64 decoded value
|
|
21
|
-
console.error(`Failed to parse JSON from base64 value: ${data}`, error);
|
|
18
|
+
catch {
|
|
19
|
+
// If JSON parsing fails, return the base64 decoded value
|
|
22
20
|
return plainText;
|
|
23
21
|
}
|
|
24
22
|
};
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ProtobufDeserializer } from '../types/types.js';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Create a Protobuf deserializer, importing `protobufjs` on first use.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* The `protobufjs` dependency is only required when deserializing Protobuf
|
|
6
|
+
* messages, so it's resolved at runtime through a non-literal specifier. This
|
|
7
|
+
* keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
|
|
8
|
+
* failing when it isn't installed - in builds that never use Protobuf. See
|
|
9
|
+
* {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
9
10
|
*/
|
|
10
|
-
declare const
|
|
11
|
-
export {
|
|
11
|
+
declare const createDeserializer: () => Promise<ProtobufDeserializer>;
|
|
12
|
+
export { createDeserializer };
|
|
12
13
|
//# sourceMappingURL=protobuf.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EAGrB,MAAM,mBAAmB,CAAC;AAI3B;;;;;;;;GAQG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,oBAAoB,CA4EhE,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -1,80 +1,112 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
const protobufjs_1 = require("protobufjs");
|
|
36
|
+
exports.createDeserializer = void 0;
|
|
5
37
|
const errors_js_1 = require("../errors.js");
|
|
38
|
+
let deserializer;
|
|
6
39
|
/**
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
const varintOrder = ['int32', 'sint32'];
|
|
10
|
-
/**
|
|
11
|
-
* Deserialize a Protobuf message from a base64-encoded string.
|
|
40
|
+
* Create a Protobuf deserializer, importing `protobufjs` on first use.
|
|
12
41
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
42
|
+
* The `protobufjs` dependency is only required when deserializing Protobuf
|
|
43
|
+
* messages, so it's resolved at runtime through a non-literal specifier. This
|
|
44
|
+
* keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
|
|
45
|
+
* failing when it isn't installed - in builds that never use Protobuf. See
|
|
46
|
+
* {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
17
47
|
*/
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
if (schemaMetadata.schemaId === undefined) {
|
|
22
|
-
return messageType.decode(buffer, buffer.length);
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
|
|
26
|
-
*
|
|
27
|
-
* When this is the case, we know the schema is coming from Glue Schema Registry,
|
|
28
|
-
* and the first byte of the buffer is a magic byte that we need to remove before
|
|
29
|
-
* decoding the message.
|
|
30
|
-
*/
|
|
31
|
-
if (schemaMetadata.schemaId.length > 10) {
|
|
32
|
-
// remove the first byte from the buffer
|
|
33
|
-
const reader = new protobufjs_1.BufferReader(buffer);
|
|
34
|
-
reader.uint32();
|
|
35
|
-
return messageType.decode(reader);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
catch (error) {
|
|
39
|
-
throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
|
|
48
|
+
const createDeserializer = async () => {
|
|
49
|
+
if (deserializer !== undefined) {
|
|
50
|
+
return deserializer;
|
|
40
51
|
}
|
|
52
|
+
const moduleName = 'protobufjs';
|
|
53
|
+
const { BufferReader } = await Promise.resolve(`${moduleName}`).then(s => __importStar(require(s)));
|
|
41
54
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
55
|
+
* Skip the Confluent Schema Registry message indexes at the start of the buffer.
|
|
56
|
+
*
|
|
57
|
+
* Confluent writes the indexes as an array of zigzag varints: the array length
|
|
58
|
+
* first, then each index. The common case `[0]` is written as a single `0` byte.
|
|
59
|
+
*
|
|
60
|
+
* @param buffer - The buffer to read from, positioned after the schema ID.
|
|
45
61
|
*/
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
try {
|
|
52
|
-
const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
|
|
53
|
-
const decoded = messageType.decode(newBuffer);
|
|
54
|
-
// swap varint order if the first attempt failed so we can use the correct one for subsequent messages
|
|
55
|
-
varintOrder.reverse();
|
|
56
|
-
return decoded;
|
|
62
|
+
const skipConfluentMessageIndexes = (buffer) => {
|
|
63
|
+
const reader = new BufferReader(buffer);
|
|
64
|
+
const indexCount = reader.sint32();
|
|
65
|
+
if (indexCount < 0) {
|
|
66
|
+
throw new Error(`Invalid Confluent message index array length: ${indexCount}`);
|
|
57
67
|
}
|
|
58
|
-
|
|
59
|
-
|
|
68
|
+
for (let i = 0; i < indexCount; i++) {
|
|
69
|
+
reader.sint32();
|
|
60
70
|
}
|
|
61
|
-
|
|
62
|
-
};
|
|
63
|
-
exports.deserialize = deserialize;
|
|
64
|
-
/**
|
|
65
|
-
* Clip the Confluent Schema Registry buffer to remove the index bytes.
|
|
66
|
-
*
|
|
67
|
-
* @param buffer - The buffer to clip.
|
|
68
|
-
* @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
|
|
69
|
-
*/
|
|
70
|
-
const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
|
|
71
|
-
const reader = new protobufjs_1.BufferReader(buffer);
|
|
71
|
+
return reader;
|
|
72
|
+
};
|
|
72
73
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
74
|
+
* Deserialize a Protobuf message from a base64-encoded string.
|
|
75
|
+
*
|
|
76
|
+
* @template T - The type of the deserialized message object.
|
|
77
|
+
*
|
|
78
|
+
* @param data - The base64-encoded string representing the Protobuf binary data.
|
|
79
|
+
* @param messageType - The Protobuf message type definition - see the `Message` type from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
75
80
|
*/
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
deserializer = (data, messageType, schemaMetadata) => {
|
|
82
|
+
const buffer = Buffer.from(data, 'base64');
|
|
83
|
+
try {
|
|
84
|
+
if (schemaMetadata.schemaId === undefined) {
|
|
85
|
+
return messageType.decode(buffer, buffer.length);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
|
|
89
|
+
*
|
|
90
|
+
* When this is the case, we know the schema is coming from Glue Schema Registry,
|
|
91
|
+
* and the first byte of the buffer is a magic byte that we need to remove before
|
|
92
|
+
* decoding the message.
|
|
93
|
+
*/
|
|
94
|
+
if (schemaMetadata.schemaId.length > 10) {
|
|
95
|
+
// remove the first byte from the buffer
|
|
96
|
+
const reader = new BufferReader(buffer);
|
|
97
|
+
reader.uint32();
|
|
98
|
+
return messageType.decode(reader);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
|
|
102
|
+
* so we need to remove the message index bytes before decoding.
|
|
103
|
+
*/
|
|
104
|
+
return messageType.decode(skipConfluentMessageIndexes(buffer));
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
throw new errors_js_1.KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message (base64-decoded payload length: ${buffer.length} bytes): ${error}`);
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
return deserializer;
|
|
80
111
|
};
|
|
112
|
+
exports.createDeserializer = createDeserializer;
|
package/lib/cjs/index.d.ts
CHANGED
package/lib/cjs/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}
|
package/lib/cjs/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.kafkaConsumer = exports.SchemaType = void 0;
|
|
3
|
+
exports.kafkaConsumer = exports.WireFormat = exports.SchemaType = void 0;
|
|
4
4
|
var constants_js_1 = require("./constants.js");
|
|
5
5
|
Object.defineProperty(exports, "SchemaType", { enumerable: true, get: function () { return constants_js_1.SchemaType; } });
|
|
6
|
+
Object.defineProperty(exports, "WireFormat", { enumerable: true, get: function () { return constants_js_1.WireFormat; } });
|
|
6
7
|
var consumer_js_1 = require("./consumer.js");
|
|
7
8
|
Object.defineProperty(exports, "kafkaConsumer", { enumerable: true, get: function () { return consumer_js_1.kafkaConsumer; } });
|
package/lib/cjs/types/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export type { ConsumerRecord, ConsumerRecords, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, } from './types.js';
|
|
1
|
+
export type { ConsumerRecord, ConsumerRecords, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, WireFormat, } from './types.js';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,cAAc,EACd,eAAe,EACf,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,cAAc,EACd,eAAe,EACf,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,UAAU,GACX,MAAM,YAAY,CAAC"}
|
package/lib/cjs/types/types.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
2
2
|
import type { Reader } from 'protobufjs';
|
|
3
|
-
import type { SchemaType as SchemaTypeMap } from '../constants.js';
|
|
3
|
+
import type { SchemaType as SchemaTypeMap, WireFormat as WireFormatMap } from '../constants.js';
|
|
4
4
|
/**
|
|
5
5
|
* Union type for supported schema types (JSON, Avro, Protobuf).
|
|
6
6
|
*/
|
|
7
7
|
type SchemaType = (typeof SchemaTypeMap)[keyof typeof SchemaTypeMap];
|
|
8
|
+
/**
|
|
9
|
+
* Union type for supported wire formats.
|
|
10
|
+
*/
|
|
11
|
+
type WireFormat = (typeof WireFormatMap)[keyof typeof WireFormatMap];
|
|
8
12
|
/**
|
|
9
13
|
* Metadata about the schema used for the field.
|
|
10
14
|
*/
|
|
@@ -115,6 +119,10 @@ type AvroConfig = {
|
|
|
115
119
|
* Avro schema definition as a string
|
|
116
120
|
*/
|
|
117
121
|
schema: string;
|
|
122
|
+
/**
|
|
123
|
+
* Optional wire format of the payload; `confluent` strips the 5-byte Confluent header before decoding.
|
|
124
|
+
*/
|
|
125
|
+
wireFormat?: WireFormat;
|
|
118
126
|
/**
|
|
119
127
|
* Optional {@link https://github.com/standard-schema/standard-schema | Standard Schema} for runtime validation
|
|
120
128
|
*/
|
|
@@ -233,12 +241,20 @@ type MSKEvent = {
|
|
|
233
241
|
type ProtobufMessage<T = unknown> = {
|
|
234
242
|
decode(reader: Reader | Uint8Array, length?: number): T;
|
|
235
243
|
};
|
|
236
|
-
type Deserializer = (input: string | null, schema?: unknown,
|
|
244
|
+
type Deserializer = (input: string | null, schema?: unknown, options?: SchemaMetadata | AvroDeserializerOptions) => unknown;
|
|
245
|
+
type AvroDeserializerOptions = {
|
|
246
|
+
/**
|
|
247
|
+
* Wire format of the payload; `confluent` strips the 5-byte Confluent header before decoding.
|
|
248
|
+
*/
|
|
249
|
+
wireFormat?: WireFormat;
|
|
250
|
+
};
|
|
251
|
+
type AvroDeserializer = (data: string, schema: string, options?: AvroDeserializerOptions) => unknown;
|
|
252
|
+
type ProtobufDeserializer = <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
|
|
237
253
|
type DeserializeOptions = {
|
|
238
254
|
value: string | null;
|
|
239
255
|
deserializer: Deserializer;
|
|
240
256
|
config?: SchemaConfigValue;
|
|
241
257
|
schemaMetadata: SchemaMetadata;
|
|
242
258
|
};
|
|
243
|
-
export type { ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
|
|
259
|
+
export type { AvroDeserializer, AvroDeserializerOptions, ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufDeserializer, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, WireFormat, };
|
|
244
260
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EACV,UAAU,IAAI,aAAa,EAC3B,UAAU,IAAI,aAAa,EAC5B,MAAM,iBAAiB,CAAC;AAEzB;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE,cAAc,GAAG,uBAAuB,KAC/C,OAAO,CAAC;AAEb,KAAK,uBAAuB,GAAG;IAC7B;;OAEG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,uBAAuB,KAC9B,OAAO,CAAC;AAEb,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAC5B,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,EAC/B,cAAc,EAAE,cAAc,KAC3B,CAAC,CAAC;AAEP,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,UAAU,GACX,CAAC"}
|
package/lib/esm/constants.d.ts
CHANGED
|
@@ -6,5 +6,11 @@ declare const SchemaType: {
|
|
|
6
6
|
readonly AVRO: "avro";
|
|
7
7
|
readonly PROTOBUF: "protobuf";
|
|
8
8
|
};
|
|
9
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Wire formats supported for Kafka payloads.
|
|
11
|
+
*/
|
|
12
|
+
declare const WireFormat: {
|
|
13
|
+
readonly CONFLUENT: "confluent";
|
|
14
|
+
};
|
|
15
|
+
export { SchemaType, WireFormat };
|
|
10
16
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,QAAA,MAAM,UAAU;;;;CAIN,CAAC;AAEX;;GAEG;AACH,QAAA,MAAM,UAAU;;CAEN,CAAC;AAEX,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
|
package/lib/esm/constants.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../../src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAGzE,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AASnD,OAAO,KAAK,EAEV,eAAe,EAIf,QAAQ,EACR,YAAY,EAEb,MAAM,kBAAkB,CAAC;AA6M1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,QAAA,MAAM,aAAa,GAAI,CAAC,EAAE,CAAC,EACzB,SAAS,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EACrD,SAAS,YAAY,KACpB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAwB1D,CAAC;AAEF,OAAO,EAAE,aAAa,EAAE,CAAC"}
|
package/lib/esm/consumer.js
CHANGED
|
@@ -59,7 +59,9 @@ const deserialize = ({ value, deserializer, config, schemaMetadata, }) => {
|
|
|
59
59
|
if (!config.schema) {
|
|
60
60
|
throw new KafkaConsumerMissingSchemaError('Schema string is required for avro deserialization');
|
|
61
61
|
}
|
|
62
|
-
return deserializer(value, config.schema
|
|
62
|
+
return deserializer(value, config.schema, {
|
|
63
|
+
wireFormat: config.wireFormat,
|
|
64
|
+
});
|
|
63
65
|
}
|
|
64
66
|
/* v8 ignore else -- @preserve */
|
|
65
67
|
if (config.type === 'protobuf') {
|
|
@@ -83,12 +85,12 @@ const getDeserializer = async (type) => {
|
|
|
83
85
|
return deserializeJson;
|
|
84
86
|
}
|
|
85
87
|
if (type === 'protobuf') {
|
|
86
|
-
const
|
|
87
|
-
return
|
|
88
|
+
const { createDeserializer } = await import('./deserializer/protobuf.js');
|
|
89
|
+
return (await createDeserializer());
|
|
88
90
|
}
|
|
89
91
|
if (type === 'avro') {
|
|
90
|
-
const
|
|
91
|
-
return
|
|
92
|
+
const { createDeserializer } = await import('./deserializer/avro.js');
|
|
93
|
+
return (await createDeserializer());
|
|
92
94
|
}
|
|
93
95
|
throw new KafkaConsumerDeserializationError(`Unsupported deserialization type: ${type}. Supported types are: json, avro, protobuf.`);
|
|
94
96
|
};
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import type { AvroDeserializer } from '../types/types.js';
|
|
1
2
|
/**
|
|
2
|
-
*
|
|
3
|
+
* Create an Avro deserializer, importing `avro-js` on first use.
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
5
|
+
* The `avro-js` dependency is only required when deserializing Avro messages,
|
|
6
|
+
* so it's resolved at runtime through a non-literal specifier. This keeps
|
|
7
|
+
* bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
|
|
8
|
+
* it isn't installed - in builds that never use Avro.
|
|
6
9
|
*/
|
|
7
|
-
declare const
|
|
8
|
-
export {
|
|
10
|
+
declare const createDeserializer: () => Promise<AvroDeserializer>;
|
|
11
|
+
export { createDeserializer };
|
|
9
12
|
//# sourceMappingURL=avro.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"avro.d.ts","sourceRoot":"","sources":["../../../src/deserializer/avro.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAoC1D;;;;;;;GAOG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,gBAAgB,CA2C5D,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -1,19 +1,73 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { WireFormat } from '../constants.js';
|
|
2
2
|
import { KafkaConsumerDeserializationError } from '../errors.js';
|
|
3
|
+
const CONFLUENT_HEADER_SIZE = 5;
|
|
4
|
+
const CONFLUENT_MAGIC_BYTE = 0x00;
|
|
5
|
+
let deserializer;
|
|
3
6
|
/**
|
|
4
|
-
*
|
|
7
|
+
* Strip the Confluent wire format header (magic byte and 4-byte schema ID) from an Avro payload.
|
|
5
8
|
*
|
|
6
|
-
* @param
|
|
7
|
-
* @param
|
|
9
|
+
* @param buffer - The decoded Avro payload.
|
|
10
|
+
* @param wireFormat - The wire format of the payload. If not provided, the payload is returned as is.
|
|
8
11
|
*/
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const buffer = Buffer.from(data, 'base64');
|
|
13
|
-
return type.fromBuffer(buffer);
|
|
12
|
+
const stripWireFormatHeader = (buffer, wireFormat) => {
|
|
13
|
+
if (wireFormat === undefined) {
|
|
14
|
+
return buffer;
|
|
14
15
|
}
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
// Guards untyped (plain JavaScript) callers
|
|
17
|
+
if (wireFormat !== WireFormat.CONFLUENT) {
|
|
18
|
+
throw new KafkaConsumerDeserializationError(`Unsupported Avro wire format: ${wireFormat}`);
|
|
17
19
|
}
|
|
20
|
+
if (buffer.length < CONFLUENT_HEADER_SIZE) {
|
|
21
|
+
throw new KafkaConsumerDeserializationError('Invalid Confluent wire format: payload must contain a 5-byte header');
|
|
22
|
+
}
|
|
23
|
+
if (buffer[0] !== CONFLUENT_MAGIC_BYTE) {
|
|
24
|
+
throw new KafkaConsumerDeserializationError('Invalid Confluent wire format: expected magic byte 0x00');
|
|
25
|
+
}
|
|
26
|
+
return buffer.subarray(CONFLUENT_HEADER_SIZE);
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Create an Avro deserializer, importing `avro-js` on first use.
|
|
30
|
+
*
|
|
31
|
+
* The `avro-js` dependency is only required when deserializing Avro messages,
|
|
32
|
+
* so it's resolved at runtime through a non-literal specifier. This keeps
|
|
33
|
+
* bundlers (e.g. esbuild) from eagerly including `avro-js` - and failing when
|
|
34
|
+
* it isn't installed - in builds that never use Avro.
|
|
35
|
+
*/
|
|
36
|
+
const createDeserializer = async () => {
|
|
37
|
+
if (deserializer !== undefined) {
|
|
38
|
+
return deserializer;
|
|
39
|
+
}
|
|
40
|
+
const moduleName = 'avro-js';
|
|
41
|
+
const { default: avro } = await import(moduleName);
|
|
42
|
+
/**
|
|
43
|
+
* Deserialize an Avro message from a base64-encoded string using the provided Avro schema.
|
|
44
|
+
*
|
|
45
|
+
* @param data - The base64-encoded string representing the Avro binary data.
|
|
46
|
+
* @param schema - The Avro schema as a JSON string.
|
|
47
|
+
* @param options - Deserialization options. If `options.wireFormat` is `confluent`, the 5-byte header is stripped.
|
|
48
|
+
*/
|
|
49
|
+
deserializer = (data, schema, options) => {
|
|
50
|
+
const decodedPayload = Buffer.from(data, 'base64');
|
|
51
|
+
const buffer = stripWireFormatHeader(decodedPayload, options?.wireFormat);
|
|
52
|
+
let type;
|
|
53
|
+
try {
|
|
54
|
+
type = avro.parse(schema);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
throw new KafkaConsumerDeserializationError(`Failed to deserialize Avro message (base64-decoded payload length: ${decodedPayload.length} bytes): invalid Avro schema`);
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
// Most Confluent serializers write top-level bytes without the Avro length prefix, so return them as-is
|
|
61
|
+
if (options?.wireFormat === WireFormat.CONFLUENT &&
|
|
62
|
+
type.getName(true) === 'bytes') {
|
|
63
|
+
return buffer;
|
|
64
|
+
}
|
|
65
|
+
return type.fromBuffer(buffer);
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
throw new KafkaConsumerDeserializationError(`Failed to deserialize Avro message (base64-decoded payload length: ${decodedPayload.length} bytes): ${error}`);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
return deserializer;
|
|
18
72
|
};
|
|
19
|
-
export {
|
|
73
|
+
export { createDeserializer };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/deserializer/json.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,QAAA,MAAM,WAAW,GAAI,MAAM,MAAM,QAUhC,CAAC;AAEF,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
|
@@ -12,10 +12,8 @@ const deserialize = (data) => {
|
|
|
12
12
|
// we assume it's a JSON but it can also be a string, we don't know
|
|
13
13
|
return JSON.parse(plainText);
|
|
14
14
|
}
|
|
15
|
-
catch
|
|
16
|
-
// If JSON parsing fails,
|
|
17
|
-
// in case we could not parse it we return the base64 decoded value
|
|
18
|
-
console.error(`Failed to parse JSON from base64 value: ${data}`, error);
|
|
15
|
+
catch {
|
|
16
|
+
// If JSON parsing fails, return the base64 decoded value
|
|
19
17
|
return plainText;
|
|
20
18
|
}
|
|
21
19
|
};
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ProtobufDeserializer } from '../types/types.js';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Create a Protobuf deserializer, importing `protobufjs` on first use.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* The `protobufjs` dependency is only required when deserializing Protobuf
|
|
6
|
+
* messages, so it's resolved at runtime through a non-literal specifier. This
|
|
7
|
+
* keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
|
|
8
|
+
* failing when it isn't installed - in builds that never use Protobuf. See
|
|
9
|
+
* {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
9
10
|
*/
|
|
10
|
-
declare const
|
|
11
|
-
export {
|
|
11
|
+
declare const createDeserializer: () => Promise<ProtobufDeserializer>;
|
|
12
|
+
export { createDeserializer };
|
|
12
13
|
//# sourceMappingURL=protobuf.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"protobuf.d.ts","sourceRoot":"","sources":["../../../src/deserializer/protobuf.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EAGrB,MAAM,mBAAmB,CAAC;AAI3B;;;;;;;;GAQG;AACH,QAAA,MAAM,kBAAkB,QAAa,OAAO,CAAC,oBAAoB,CA4EhE,CAAC;AAEF,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -1,77 +1,76 @@
|
|
|
1
|
-
import { BufferReader } from 'protobufjs';
|
|
2
1
|
import { KafkaConsumerDeserializationError } from '../errors.js';
|
|
2
|
+
let deserializer;
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*/
|
|
6
|
-
const varintOrder = ['int32', 'sint32'];
|
|
7
|
-
/**
|
|
8
|
-
* Deserialize a Protobuf message from a base64-encoded string.
|
|
4
|
+
* Create a Protobuf deserializer, importing `protobufjs` on first use.
|
|
9
5
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
6
|
+
* The `protobufjs` dependency is only required when deserializing Protobuf
|
|
7
|
+
* messages, so it's resolved at runtime through a non-literal specifier. This
|
|
8
|
+
* keeps bundlers (e.g. esbuild) from eagerly including `protobufjs` - and
|
|
9
|
+
* failing when it isn't installed - in builds that never use Protobuf. See
|
|
10
|
+
* {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
14
11
|
*/
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
if (schemaMetadata.schemaId === undefined) {
|
|
19
|
-
return messageType.decode(buffer, buffer.length);
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
|
|
23
|
-
*
|
|
24
|
-
* When this is the case, we know the schema is coming from Glue Schema Registry,
|
|
25
|
-
* and the first byte of the buffer is a magic byte that we need to remove before
|
|
26
|
-
* decoding the message.
|
|
27
|
-
*/
|
|
28
|
-
if (schemaMetadata.schemaId.length > 10) {
|
|
29
|
-
// remove the first byte from the buffer
|
|
30
|
-
const reader = new BufferReader(buffer);
|
|
31
|
-
reader.uint32();
|
|
32
|
-
return messageType.decode(reader);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
catch (error) {
|
|
36
|
-
throw new KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`);
|
|
12
|
+
const createDeserializer = async () => {
|
|
13
|
+
if (deserializer !== undefined) {
|
|
14
|
+
return deserializer;
|
|
37
15
|
}
|
|
16
|
+
const moduleName = 'protobufjs';
|
|
17
|
+
const { BufferReader } = await import(moduleName);
|
|
38
18
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
19
|
+
* Skip the Confluent Schema Registry message indexes at the start of the buffer.
|
|
20
|
+
*
|
|
21
|
+
* Confluent writes the indexes as an array of zigzag varints: the array length
|
|
22
|
+
* first, then each index. The common case `[0]` is written as a single `0` byte.
|
|
23
|
+
*
|
|
24
|
+
* @param buffer - The buffer to read from, positioned after the schema ID.
|
|
42
25
|
*/
|
|
43
|
-
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
try {
|
|
49
|
-
const newBuffer = clipConfluentSchemaRegistryBuffer(buffer, varintOrder[1]);
|
|
50
|
-
const decoded = messageType.decode(newBuffer);
|
|
51
|
-
// swap varint order if the first attempt failed so we can use the correct one for subsequent messages
|
|
52
|
-
varintOrder.reverse();
|
|
53
|
-
return decoded;
|
|
26
|
+
const skipConfluentMessageIndexes = (buffer) => {
|
|
27
|
+
const reader = new BufferReader(buffer);
|
|
28
|
+
const indexCount = reader.sint32();
|
|
29
|
+
if (indexCount < 0) {
|
|
30
|
+
throw new Error(`Invalid Confluent message index array length: ${indexCount}`);
|
|
54
31
|
}
|
|
55
|
-
|
|
56
|
-
|
|
32
|
+
for (let i = 0; i < indexCount; i++) {
|
|
33
|
+
reader.sint32();
|
|
57
34
|
}
|
|
58
|
-
|
|
59
|
-
};
|
|
60
|
-
/**
|
|
61
|
-
* Clip the Confluent Schema Registry buffer to remove the index bytes.
|
|
62
|
-
*
|
|
63
|
-
* @param buffer - The buffer to clip.
|
|
64
|
-
* @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
|
|
65
|
-
*/
|
|
66
|
-
const clipConfluentSchemaRegistryBuffer = (buffer, intType) => {
|
|
67
|
-
const reader = new BufferReader(buffer);
|
|
35
|
+
return reader;
|
|
36
|
+
};
|
|
68
37
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
38
|
+
* Deserialize a Protobuf message from a base64-encoded string.
|
|
39
|
+
*
|
|
40
|
+
* @template T - The type of the deserialized message object.
|
|
41
|
+
*
|
|
42
|
+
* @param data - The base64-encoded string representing the Protobuf binary data.
|
|
43
|
+
* @param messageType - The Protobuf message type definition - see the `Message` type from {@link https://www.npmjs.com/package/protobufjs | `protobufjs`}.
|
|
71
44
|
*/
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
45
|
+
deserializer = (data, messageType, schemaMetadata) => {
|
|
46
|
+
const buffer = Buffer.from(data, 'base64');
|
|
47
|
+
try {
|
|
48
|
+
if (schemaMetadata.schemaId === undefined) {
|
|
49
|
+
return messageType.decode(buffer, buffer.length);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* If `schemaId` is longer than 10 chars, it's an UUID, otherwise it's a numeric ID.
|
|
53
|
+
*
|
|
54
|
+
* When this is the case, we know the schema is coming from Glue Schema Registry,
|
|
55
|
+
* and the first byte of the buffer is a magic byte that we need to remove before
|
|
56
|
+
* decoding the message.
|
|
57
|
+
*/
|
|
58
|
+
if (schemaMetadata.schemaId.length > 10) {
|
|
59
|
+
// remove the first byte from the buffer
|
|
60
|
+
const reader = new BufferReader(buffer);
|
|
61
|
+
reader.uint32();
|
|
62
|
+
return messageType.decode(reader);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
|
|
66
|
+
* so we need to remove the message index bytes before decoding.
|
|
67
|
+
*/
|
|
68
|
+
return messageType.decode(skipConfluentMessageIndexes(buffer));
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
throw new KafkaConsumerDeserializationError(`Failed to deserialize Protobuf message (base64-decoded payload length: ${buffer.length} bytes): ${error}`);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
return deserializer;
|
|
76
75
|
};
|
|
77
|
-
export {
|
|
76
|
+
export { createDeserializer };
|
package/lib/esm/index.d.ts
CHANGED
package/lib/esm/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}
|
package/lib/esm/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { SchemaType } from './constants.js';
|
|
1
|
+
export { SchemaType, WireFormat } from './constants.js';
|
|
2
2
|
export { kafkaConsumer } from './consumer.js';
|
package/lib/esm/types/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export type { ConsumerRecord, ConsumerRecords, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, } from './types.js';
|
|
1
|
+
export type { ConsumerRecord, ConsumerRecords, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, WireFormat, } from './types.js';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,cAAc,EACd,eAAe,EACf,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,GACX,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,cAAc,EACd,eAAe,EACf,QAAQ,EACR,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,UAAU,GACX,MAAM,YAAY,CAAC"}
|
package/lib/esm/types/types.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
2
2
|
import type { Reader } from 'protobufjs';
|
|
3
|
-
import type { SchemaType as SchemaTypeMap } from '../constants.js';
|
|
3
|
+
import type { SchemaType as SchemaTypeMap, WireFormat as WireFormatMap } from '../constants.js';
|
|
4
4
|
/**
|
|
5
5
|
* Union type for supported schema types (JSON, Avro, Protobuf).
|
|
6
6
|
*/
|
|
7
7
|
type SchemaType = (typeof SchemaTypeMap)[keyof typeof SchemaTypeMap];
|
|
8
|
+
/**
|
|
9
|
+
* Union type for supported wire formats.
|
|
10
|
+
*/
|
|
11
|
+
type WireFormat = (typeof WireFormatMap)[keyof typeof WireFormatMap];
|
|
8
12
|
/**
|
|
9
13
|
* Metadata about the schema used for the field.
|
|
10
14
|
*/
|
|
@@ -115,6 +119,10 @@ type AvroConfig = {
|
|
|
115
119
|
* Avro schema definition as a string
|
|
116
120
|
*/
|
|
117
121
|
schema: string;
|
|
122
|
+
/**
|
|
123
|
+
* Optional wire format of the payload; `confluent` strips the 5-byte Confluent header before decoding.
|
|
124
|
+
*/
|
|
125
|
+
wireFormat?: WireFormat;
|
|
118
126
|
/**
|
|
119
127
|
* Optional {@link https://github.com/standard-schema/standard-schema | Standard Schema} for runtime validation
|
|
120
128
|
*/
|
|
@@ -233,12 +241,20 @@ type MSKEvent = {
|
|
|
233
241
|
type ProtobufMessage<T = unknown> = {
|
|
234
242
|
decode(reader: Reader | Uint8Array, length?: number): T;
|
|
235
243
|
};
|
|
236
|
-
type Deserializer = (input: string | null, schema?: unknown,
|
|
244
|
+
type Deserializer = (input: string | null, schema?: unknown, options?: SchemaMetadata | AvroDeserializerOptions) => unknown;
|
|
245
|
+
type AvroDeserializerOptions = {
|
|
246
|
+
/**
|
|
247
|
+
* Wire format of the payload; `confluent` strips the 5-byte Confluent header before decoding.
|
|
248
|
+
*/
|
|
249
|
+
wireFormat?: WireFormat;
|
|
250
|
+
};
|
|
251
|
+
type AvroDeserializer = (data: string, schema: string, options?: AvroDeserializerOptions) => unknown;
|
|
252
|
+
type ProtobufDeserializer = <T>(data: string, messageType: ProtobufMessage<T>, schemaMetadata: SchemaMetadata) => T;
|
|
237
253
|
type DeserializeOptions = {
|
|
238
254
|
value: string | null;
|
|
239
255
|
deserializer: Deserializer;
|
|
240
256
|
config?: SchemaConfigValue;
|
|
241
257
|
schemaMetadata: SchemaMetadata;
|
|
242
258
|
};
|
|
243
|
-
export type { ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, };
|
|
259
|
+
export type { AvroDeserializer, AvroDeserializerOptions, ConsumerRecord, ConsumerRecords, DeserializeOptions, Deserializer, MSKEvent, ProtobufDeserializer, ProtobufMessage, Record, RecordHeader, SchemaConfig, SchemaConfigValue, SchemaMetadata, SchemaType, WireFormat, };
|
|
244
260
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EACV,UAAU,IAAI,aAAa,EAC3B,UAAU,IAAI,aAAa,EAC5B,MAAM,iBAAiB,CAAC;AAEzB;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAC;AAErE;;GAEG;AACH,KAAK,cAAc,GAAG;IACpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,EAAE,CAAC,IAAI;IAC1B;;OAEG;IACH,GAAG,EAAE,CAAC,CAAC;IACP;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,CAAC;IAC3C;;OAEG;IACH,eAAe,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IACxC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI;IAC/C;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACtC,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9B;;GAEG;AACH,KAAK,iBAAiB,GAAG,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;AAE3E;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,GAAG;IAChB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,CAAC;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,cAAc,CAAC,CAAC,IAAI;IACvB;;OAEG;IACH,IAAI,EAAE,OAAO,aAAa,CAAC,QAAQ,CAAC;IACpC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3B;;OAEG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;;OAGG;IACH,KAAK,EAAE,iBAAiB,CAAC;IACzB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,KAAK,YAAY,GAAG;IAClB;;OAEG;IACH,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,KAAK,MAAM,GAAG;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,aAAa,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACjD;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB;;OAEG;IACH,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,mBAAmB,EAAE,cAAc,CAAC;IACpC;;OAEG;IACH,iBAAiB,EAAE,cAAc,CAAC;CACnC,CAAC;AAEF;;;GAGG;AACH,KAAK,QAAQ,GAAG;IACd;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,EAAE;QACP,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,IAAI;IAClC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;CACzD,CAAC;AAEF,KAAK,YAAY,GAAG,CAClB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,CAAC,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE,cAAc,GAAG,uBAAuB,KAC/C,OAAO,CAAC;AAEb,KAAK,uBAAuB,GAAG;IAC7B;;OAEG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,uBAAuB,KAC9B,OAAO,CAAC;AAEb,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAC5B,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,EAC/B,cAAc,EAAE,cAAc,KAC3B,CAAC,CAAC;AAEP,KAAK,kBAAkB,GAAG;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,UAAU,GACX,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aws-lambda-powertools/kafka",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.37.0",
|
|
4
4
|
"description": "Utility to easily handle message deserialization and parsing of Kafka events in AWS Lambda functions",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Amazon Web Services",
|
|
@@ -27,12 +27,14 @@
|
|
|
27
27
|
"nodejs"
|
|
28
28
|
],
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@aws-lambda-powertools/commons": "2.
|
|
30
|
+
"@aws-lambda-powertools/commons": "2.37.0",
|
|
31
31
|
"@standard-schema/spec": "^1.1.0"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
34
|
-
"arktype": "^2.2.
|
|
35
|
-
"
|
|
34
|
+
"arktype": "^2.2.9",
|
|
35
|
+
"avro-js": "^1.12.1",
|
|
36
|
+
"protobufjs": "^8.8.1",
|
|
37
|
+
"valibot": "^1.5.0",
|
|
36
38
|
"zod": "^3.25.0 || ^4.0.0"
|
|
37
39
|
},
|
|
38
40
|
"peerDependenciesMeta": {
|
|
@@ -44,6 +46,12 @@
|
|
|
44
46
|
},
|
|
45
47
|
"arktype": {
|
|
46
48
|
"optional": true
|
|
49
|
+
},
|
|
50
|
+
"avro-js": {
|
|
51
|
+
"optional": true
|
|
52
|
+
},
|
|
53
|
+
"protobufjs": {
|
|
54
|
+
"optional": true
|
|
47
55
|
}
|
|
48
56
|
},
|
|
49
57
|
"files": [
|