@platformatic/kafka 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -632,7 +632,7 @@ export class MessagesStream extends Readable {
|
|
|
632
632
|
consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
|
|
633
633
|
}
|
|
634
634
|
catch (error) {
|
|
635
|
-
const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
|
|
635
|
+
const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error);
|
|
636
636
|
if (shouldDestroy) {
|
|
637
637
|
diagnosticContext.error = error;
|
|
638
638
|
consumerReceivesChannel.error.publish(diagnosticContext);
|
|
@@ -32,7 +32,7 @@ export interface PreferredReadReplica {
|
|
|
32
32
|
node: number;
|
|
33
33
|
expiresAt: number;
|
|
34
34
|
}
|
|
35
|
-
export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
|
|
35
|
+
export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
|
|
36
36
|
export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
|
|
37
37
|
export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
|
|
38
38
|
export declare const MessagesStreamModes: {
|
|
@@ -11,7 +11,6 @@ export function defaultProtobufTypeMapper(_, type, context) {
|
|
|
11
11
|
// Confluent Schema Registry convention
|
|
12
12
|
return `${context.topic}-${type}`;
|
|
13
13
|
}
|
|
14
|
-
// TODO(ShogunPanda): Authentication support
|
|
15
14
|
export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
16
15
|
#url;
|
|
17
16
|
#schemas;
|
|
@@ -19,6 +18,7 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
|
19
18
|
#protobufTypeMapper;
|
|
20
19
|
#jsonValidateSend;
|
|
21
20
|
#jsonAjv;
|
|
21
|
+
#pendingFetches;
|
|
22
22
|
#auth;
|
|
23
23
|
constructor(options) {
|
|
24
24
|
super();
|
|
@@ -28,6 +28,7 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
|
28
28
|
this.#jsonValidateSend = options.jsonValidateSend ?? false;
|
|
29
29
|
this.#jsonAjv = new Ajv2020({ allErrors: true, coerceTypes: false, strict: true, ...options.jsonAjvOptions });
|
|
30
30
|
this.#auth = options.auth;
|
|
31
|
+
this.#pendingFetches = new Map();
|
|
31
32
|
}
|
|
32
33
|
getSchemaId(message, type) {
|
|
33
34
|
if (message === null) {
|
|
@@ -45,42 +46,15 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
|
45
46
|
return this.#schemas.get(id);
|
|
46
47
|
}
|
|
47
48
|
async fetchSchema(id, callback) {
|
|
49
|
+
let fetch = this.#pendingFetches.get(id);
|
|
50
|
+
if (!fetch) {
|
|
51
|
+
fetch = this.#fetchSchema(id).finally(() => {
|
|
52
|
+
this.#pendingFetches.delete(id);
|
|
53
|
+
});
|
|
54
|
+
this.#pendingFetches.set(id, fetch);
|
|
55
|
+
}
|
|
48
56
|
try {
|
|
49
|
-
|
|
50
|
-
if (this.#auth) {
|
|
51
|
-
if (this.#auth.token) {
|
|
52
|
-
const token = await getCredential('token', this.#auth.token);
|
|
53
|
-
requestInit.headers = {
|
|
54
|
-
Authorization: `Bearer ${token}`
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
else {
|
|
58
|
-
const username = await getCredential('username', this.#auth.username);
|
|
59
|
-
const password = await getCredential('password', this.#auth.password);
|
|
60
|
-
requestInit.headers = {
|
|
61
|
-
Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
|
|
66
|
-
if (!response.ok) {
|
|
67
|
-
throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
|
|
68
|
-
}
|
|
69
|
-
const responseBody = (await response.json());
|
|
70
|
-
const { schema } = responseBody;
|
|
71
|
-
const schemaType = responseBody.schemaType ?? 'AVRO';
|
|
72
|
-
switch (schemaType) {
|
|
73
|
-
case 'AVRO':
|
|
74
|
-
this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
|
|
75
|
-
break;
|
|
76
|
-
case 'PROTOBUF':
|
|
77
|
-
this.#protobufParse ??= this.#loadProtobuf();
|
|
78
|
-
this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
|
|
79
|
-
break;
|
|
80
|
-
case 'JSON':
|
|
81
|
-
this.#schemas.set(id, { id, type: 'json', schema: this.#jsonAjv.compile(JSON.parse(schema)) });
|
|
82
|
-
break;
|
|
83
|
-
}
|
|
57
|
+
await fetch;
|
|
84
58
|
process.nextTick(callback);
|
|
85
59
|
}
|
|
86
60
|
catch (err) {
|
|
@@ -139,6 +113,43 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
|
139
113
|
registry.fetchSchema(schemaId, callback);
|
|
140
114
|
};
|
|
141
115
|
}
|
|
116
|
+
async #fetchSchema(id) {
|
|
117
|
+
const requestInit = {};
|
|
118
|
+
if (this.#auth) {
|
|
119
|
+
if (this.#auth.token) {
|
|
120
|
+
const token = await getCredential('token', this.#auth.token);
|
|
121
|
+
requestInit.headers = {
|
|
122
|
+
Authorization: `Bearer ${token}`
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
const username = await getCredential('username', this.#auth.username);
|
|
127
|
+
const password = await getCredential('password', this.#auth.password);
|
|
128
|
+
requestInit.headers = {
|
|
129
|
+
Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
|
|
134
|
+
if (!response.ok) {
|
|
135
|
+
throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
|
|
136
|
+
}
|
|
137
|
+
const responseBody = (await response.json());
|
|
138
|
+
const { schema } = responseBody;
|
|
139
|
+
const schemaType = responseBody.schemaType ?? 'AVRO';
|
|
140
|
+
switch (schemaType) {
|
|
141
|
+
case 'AVRO':
|
|
142
|
+
this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
|
|
143
|
+
break;
|
|
144
|
+
case 'PROTOBUF':
|
|
145
|
+
this.#protobufParse ??= this.#loadProtobuf();
|
|
146
|
+
this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
|
|
147
|
+
break;
|
|
148
|
+
case 'JSON':
|
|
149
|
+
this.#schemas.set(id, { id, type: 'json', schema: this.#jsonAjv.compile(JSON.parse(schema)) });
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
142
153
|
#schemaSerializer(type, fallbackSerializer, data, headers, message) {
|
|
143
154
|
/* c8 ignore next 3 - Hard to test */
|
|
144
155
|
if (typeof data === 'undefined') {
|
|
@@ -32,7 +32,7 @@ export interface PreferredReadReplica {
|
|
|
32
32
|
node: number;
|
|
33
33
|
expiresAt: number;
|
|
34
34
|
}
|
|
35
|
-
export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
|
|
35
|
+
export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
|
|
36
36
|
export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
|
|
37
37
|
export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
|
|
38
38
|
export declare const MessagesStreamModes: {
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export const name = "@platformatic/kafka";
|
|
2
|
-
export const version = "2.
|
|
2
|
+
export const version = "2.7.0";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platformatic/kafka",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "Modern and performant client for Apache Kafka",
|
|
5
5
|
"homepage": "https://github.com/platformatic/kafka",
|
|
6
6
|
"author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
|