@platformatic/kafka 2.2.2 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/apis/admin/create-topics-v7.js +1 -0
- package/dist/apis/callbacks.d.ts +1 -3
- package/dist/apis/callbacks.js +26 -11
- package/dist/clients/admin/admin.js +330 -356
- package/dist/clients/base/base.js +8 -1
- package/dist/clients/consumer/consumer.js +23 -2
- package/dist/clients/consumer/messages-stream.js +5 -1
- package/dist/clients/consumer/utils.d.ts +1 -0
- package/dist/clients/consumer/utils.js +6 -0
- package/dist/clients/producer/producer.js +53 -5
- package/dist/clients/serde.d.ts +1 -1
- package/dist/errors.d.ts +2 -0
- package/dist/errors.js +6 -0
- package/dist/protocol/reader.js +4 -3
- package/dist/registries/abstract.d.ts +2 -2
- package/dist/registries/confluent-schema-registry.d.ts +1 -1
- package/dist/registries/confluent-schema-registry.js +3 -0
- package/dist/typescript-4/dist/apis/callbacks.d.ts +1 -3
- package/dist/typescript-4/dist/clients/consumer/utils.d.ts +2 -1
- package/dist/typescript-4/dist/clients/serde.d.ts +1 -1
- package/dist/typescript-4/dist/errors.d.ts +2 -0
- package/dist/typescript-4/dist/registries/abstract.d.ts +2 -2
- package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -2
|
@@ -419,7 +419,13 @@ export class Base extends TypedEventEmitter {
|
|
|
419
419
|
retryCallback(error);
|
|
420
420
|
return;
|
|
421
421
|
}
|
|
422
|
-
|
|
422
|
+
// includeTopicAuthorizedOperations: false. The response's
|
|
423
|
+
// topic_authorized_operations bitfield is parsed but never read
|
|
424
|
+
// by the producer/consumer, and requesting it makes IAM brokers
|
|
425
|
+
// (e.g. MSK) authorize every topic operation and emit a CloudTrail
|
|
426
|
+
// AccessDenied per denied op (CreateTopic/DeleteTopic/...) — pure
|
|
427
|
+
// audit-log noise for permissions the client never exercises.
|
|
428
|
+
api(connection, topicsToFetch, autocreateTopics, false, retryCallback);
|
|
423
429
|
});
|
|
424
430
|
}, attempt);
|
|
425
431
|
}, (error, metadata) => {
|
|
@@ -455,6 +461,7 @@ export class Base extends TypedEventEmitter {
|
|
|
455
461
|
}
|
|
456
462
|
else {
|
|
457
463
|
this.#metadata.lastUpdate = lastUpdate;
|
|
464
|
+
this.#metadata.controllerId = metadata.controllerId;
|
|
458
465
|
}
|
|
459
466
|
const brokers = new Map();
|
|
460
467
|
// This should never change, but we act defensively here
|
|
@@ -15,7 +15,7 @@ import { MessagesStream } from "./messages-stream.js";
|
|
|
15
15
|
import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
|
|
16
16
|
import { roundRobinAssigner } from "./partitions-assigners.js";
|
|
17
17
|
import { TopicsMap } from "./topics-map.js";
|
|
18
|
-
import { partitionKey } from "./utils.js";
|
|
18
|
+
import { nextFetchEpoch, partitionKey } from "./utils.js";
|
|
19
19
|
export class Consumer extends Base {
|
|
20
20
|
groupId;
|
|
21
21
|
groupInstanceId;
|
|
@@ -36,6 +36,7 @@ export class Consumer extends Base {
|
|
|
36
36
|
#memberEpoch;
|
|
37
37
|
#groupRemoteAssignor;
|
|
38
38
|
#clientRack;
|
|
39
|
+
#fetchSessions;
|
|
39
40
|
#preferredReadReplicas;
|
|
40
41
|
#streams;
|
|
41
42
|
#lagMonitoring;
|
|
@@ -82,6 +83,7 @@ export class Consumer extends Base {
|
|
|
82
83
|
this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
|
|
83
84
|
this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
|
|
84
85
|
this.#clientRack = this[kOptions].clientRack ?? '';
|
|
86
|
+
this.#fetchSessions = new Map();
|
|
85
87
|
this.#preferredReadReplicas = new Map();
|
|
86
88
|
this.#streamContext = options.streamContext ?? options.context;
|
|
87
89
|
this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
|
|
@@ -144,6 +146,7 @@ export class Consumer extends Base {
|
|
|
144
146
|
return;
|
|
145
147
|
}
|
|
146
148
|
this.topics.clear();
|
|
149
|
+
this.#fetchSessions.clear();
|
|
147
150
|
if (this[kPrometheus]) {
|
|
148
151
|
ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
|
|
149
152
|
}
|
|
@@ -492,6 +495,7 @@ export class Consumer extends Base {
|
|
|
492
495
|
pool.get(broker, (error, connection) => {
|
|
493
496
|
if (error) {
|
|
494
497
|
this.#clearPreferredReadReplicas(options.topics, topicIds);
|
|
498
|
+
this.#fetchSessions.delete(node);
|
|
495
499
|
// When a connection was not available (either interrupted or not available) we
|
|
496
500
|
// reset the leader epoch in the options so that when connection is re-established again we can continue
|
|
497
501
|
for (const topic of options.topics) {
|
|
@@ -508,11 +512,20 @@ export class Consumer extends Base {
|
|
|
508
512
|
retryCallback(error);
|
|
509
513
|
return;
|
|
510
514
|
}
|
|
511
|
-
|
|
515
|
+
const session = this.#fetchSessions.get(node);
|
|
516
|
+
const sessionId = session?.id ?? 0;
|
|
517
|
+
const sessionEpoch = session?.epoch ?? 0;
|
|
518
|
+
api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, options.topics, [], this.#clientRack, (error, result) => {
|
|
512
519
|
if (error) {
|
|
513
520
|
this.#clearPreferredReadReplicas(options.topics, topicIds);
|
|
514
521
|
const genericError = error;
|
|
522
|
+
if (genericError.findBy?.('apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
|
|
523
|
+
genericError.findBy?.('apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
|
|
524
|
+
genericError.findBy?.('apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
|
|
525
|
+
this.#fetchSessions.delete(node);
|
|
526
|
+
}
|
|
515
527
|
if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
|
|
528
|
+
this.#fetchSessions.delete(node);
|
|
516
529
|
this.clearMetadata();
|
|
517
530
|
for (const topic of options.topics) {
|
|
518
531
|
for (const partition of topic.partitions) {
|
|
@@ -524,6 +537,7 @@ export class Consumer extends Base {
|
|
|
524
537
|
}
|
|
525
538
|
else {
|
|
526
539
|
this.#updatePreferredReadReplicas(metadata, topicIds, result);
|
|
540
|
+
this.#updateFetchSession(node, sessionEpoch, result.sessionId);
|
|
527
541
|
}
|
|
528
542
|
retryCallback(error, result);
|
|
529
543
|
});
|
|
@@ -542,6 +556,13 @@ export class Consumer extends Base {
|
|
|
542
556
|
callback(error, response);
|
|
543
557
|
}, 0);
|
|
544
558
|
}
|
|
559
|
+
#updateFetchSession(node, sentEpoch, responseSessionId) {
|
|
560
|
+
if (!responseSessionId) {
|
|
561
|
+
this.#fetchSessions.delete(node);
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
this.#fetchSessions.set(node, { id: responseSessionId, epoch: nextFetchEpoch(sentEpoch) });
|
|
565
|
+
}
|
|
545
566
|
#commit(options, callback, rejoinAttempts = 0) {
|
|
546
567
|
this.#performGroupOperation('commit', (connection, groupCallback) => {
|
|
547
568
|
const topics = new Map();
|
|
@@ -302,7 +302,11 @@ export class MessagesStream extends Readable {
|
|
|
302
302
|
return super[Symbol.asyncIterator]();
|
|
303
303
|
}
|
|
304
304
|
_construct(callback) {
|
|
305
|
-
this.#
|
|
305
|
+
this.#refreshOffsetsInflight = true;
|
|
306
|
+
this.#refreshOffsets(error => {
|
|
307
|
+
this.#refreshOffsetsInflight = false;
|
|
308
|
+
callback(error ?? undefined);
|
|
309
|
+
});
|
|
306
310
|
}
|
|
307
311
|
_destroy(error, callback) {
|
|
308
312
|
if (this.#autocommitInterval) {
|
|
@@ -685,9 +685,16 @@ export class Producer extends Base {
|
|
|
685
685
|
}
|
|
686
686
|
// Track nodes so that we can get their ID for delayed write reporting
|
|
687
687
|
const nodes = [];
|
|
688
|
+
const sequenceSnapshot = produceOptions.sequences ? new NumericMap() : undefined;
|
|
689
|
+
if (sequenceSnapshot) {
|
|
690
|
+
for (const [partitionKey, messagesCount] of messagesPerPartition) {
|
|
691
|
+
sequenceSnapshot.set(partitionKey, this.#sequences.postIncrement(partitionKey, messagesCount, 0));
|
|
692
|
+
}
|
|
693
|
+
produceOptions = { ...produceOptions, sequences: sequenceSnapshot };
|
|
694
|
+
}
|
|
688
695
|
runConcurrentCallbacks('Producing messages failed.', messagesByDestination, ([destination, destinationMessages], concurrentCallback) => {
|
|
689
696
|
nodes.push(destination);
|
|
690
|
-
this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
|
|
697
|
+
this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, true, concurrentCallback);
|
|
691
698
|
}, (error, apiResults) => {
|
|
692
699
|
this.#metricsProducedMessages?.inc(messages.length);
|
|
693
700
|
const results = {};
|
|
@@ -710,14 +717,16 @@ export class Producer extends Base {
|
|
|
710
717
|
partition: partitionResponse.index,
|
|
711
718
|
offset: partitionResponse.baseOffset
|
|
712
719
|
});
|
|
713
|
-
const partitionKey = `${name}:${partitionResponse.index}`;
|
|
714
|
-
this.#sequences.postIncrement(partitionKey, messagesPerPartition.get(partitionKey), 0);
|
|
715
720
|
}
|
|
716
721
|
}
|
|
717
722
|
results.offsets = topics;
|
|
718
723
|
}
|
|
719
724
|
}
|
|
720
725
|
if (error) {
|
|
726
|
+
if (sendOptions.idempotent && !this.#transaction) {
|
|
727
|
+
this.#producerInfo = undefined;
|
|
728
|
+
this.#sequences.clear();
|
|
729
|
+
}
|
|
721
730
|
;
|
|
722
731
|
error.produced = results;
|
|
723
732
|
callback(error);
|
|
@@ -728,7 +737,7 @@ export class Producer extends Base {
|
|
|
728
737
|
});
|
|
729
738
|
});
|
|
730
739
|
}
|
|
731
|
-
#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, callback) {
|
|
740
|
+
#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, recoverOnProducerStateLoss, callback) {
|
|
732
741
|
// Get the metadata with the topic/partitions informations
|
|
733
742
|
this[kMetadata]({ topics, autocreateTopics }, (error, metadata) => {
|
|
734
743
|
if (error) {
|
|
@@ -762,7 +771,17 @@ export class Producer extends Base {
|
|
|
762
771
|
const hasStaleMetadata = kafkaError?.findBy('hasStaleMetadata', true);
|
|
763
772
|
if (hasStaleMetadata && repeatOnStaleMetadata) {
|
|
764
773
|
this.clearMetadata();
|
|
765
|
-
this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
|
|
774
|
+
this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, recoverOnProducerStateLoss, callback);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
if (recoverOnProducerStateLoss && !this.#transaction && this.#isProducerStateLost(error)) {
|
|
778
|
+
this.#recoverIdempotentProducer(messages, produceOptions, (error, recoveredProduceOptions) => {
|
|
779
|
+
if (error) {
|
|
780
|
+
callback(error);
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, recoveredProduceOptions, false, callback);
|
|
784
|
+
});
|
|
766
785
|
return;
|
|
767
786
|
}
|
|
768
787
|
callback(error, results);
|
|
@@ -778,6 +797,35 @@ export class Producer extends Base {
|
|
|
778
797
|
});
|
|
779
798
|
});
|
|
780
799
|
}
|
|
800
|
+
#isProducerStateLost(error) {
|
|
801
|
+
const kafkaError = error;
|
|
802
|
+
return !!(GenericError.isGenericError(error) &&
|
|
803
|
+
(kafkaError.findBy('apiId', 'UNKNOWN_PRODUCER_ID') || kafkaError.findBy('apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER')));
|
|
804
|
+
}
|
|
805
|
+
#recoverIdempotentProducer(messages, produceOptions, callback) {
|
|
806
|
+
this.#producerInfo = undefined;
|
|
807
|
+
this.#sequences.clear();
|
|
808
|
+
this.initIdempotentProducer({}, (error, producerInfo) => {
|
|
809
|
+
if (error) {
|
|
810
|
+
callback(error);
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
const sequences = new NumericMap();
|
|
814
|
+
const messagesPerPartition = new NumericMap();
|
|
815
|
+
for (const { topic, partition } of messages) {
|
|
816
|
+
messagesPerPartition.postIncrement(`${topic}:${partition}`, 1, 0);
|
|
817
|
+
}
|
|
818
|
+
for (const [partitionKey, messagesCount] of messagesPerPartition) {
|
|
819
|
+
sequences.set(partitionKey, this.#sequences.postIncrement(partitionKey, messagesCount, 0));
|
|
820
|
+
}
|
|
821
|
+
callback(null, {
|
|
822
|
+
...produceOptions,
|
|
823
|
+
producerId: producerInfo.producerId,
|
|
824
|
+
producerEpoch: producerInfo.producerEpoch,
|
|
825
|
+
sequences
|
|
826
|
+
});
|
|
827
|
+
});
|
|
828
|
+
}
|
|
781
829
|
#getCoordinatorConnection(callback) {
|
|
782
830
|
if (this.#coordinatorId === undefined) {
|
|
783
831
|
this[kTransactionFindCoordinator](error => {
|
package/dist/clients/serde.d.ts
CHANGED
|
@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
|
|
|
17
17
|
headerValue: Deserializer<HeaderValue>;
|
|
18
18
|
}
|
|
19
19
|
export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
|
|
20
|
-
export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
|
|
20
|
+
export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
|
|
21
21
|
export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
|
|
22
22
|
export declare function stringSerializer(data?: string): Buffer | undefined;
|
|
23
23
|
export declare function stringDeserializer(data?: string | Buffer): string | undefined;
|
package/dist/errors.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export declare class GenericError extends Error {
|
|
|
14
14
|
static isGenericError(error: Error): error is GenericError | MultipleErrors;
|
|
15
15
|
constructor(code: ErrorCode, message: string, { cause, ...rest }?: ErrorProperties);
|
|
16
16
|
findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
|
|
17
|
+
toString(): string;
|
|
17
18
|
}
|
|
18
19
|
export declare class MultipleErrors extends AggregateError {
|
|
19
20
|
code: string;
|
|
@@ -25,6 +26,7 @@ export declare class MultipleErrors extends AggregateError {
|
|
|
25
26
|
static isMultipleErrors(error: Error): error is MultipleErrors;
|
|
26
27
|
constructor(message: string, errors: Error[], { cause, ...rest }?: ErrorProperties);
|
|
27
28
|
findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
|
|
29
|
+
toString(): string;
|
|
28
30
|
}
|
|
29
31
|
export * from './protocol/errors.ts';
|
|
30
32
|
export declare class AuthenticationError extends GenericError {
|
package/dist/errors.js
CHANGED
|
@@ -45,6 +45,9 @@ export class GenericError extends Error {
|
|
|
45
45
|
}
|
|
46
46
|
return null;
|
|
47
47
|
}
|
|
48
|
+
toString() {
|
|
49
|
+
return `Error ${this.code}: ${this.message}`;
|
|
50
|
+
}
|
|
48
51
|
}
|
|
49
52
|
export class MultipleErrors extends AggregateError {
|
|
50
53
|
code;
|
|
@@ -91,6 +94,9 @@ export class MultipleErrors extends AggregateError {
|
|
|
91
94
|
}
|
|
92
95
|
return null;
|
|
93
96
|
}
|
|
97
|
+
toString() {
|
|
98
|
+
return `Error ${this.code}: ${this.message} (caused by: ${this.errors.map(error => error.toString()).join(', ')})`;
|
|
99
|
+
}
|
|
94
100
|
}
|
|
95
101
|
export * from "./protocol/errors.js";
|
|
96
102
|
export class AuthenticationError extends GenericError {
|
package/dist/protocol/reader.js
CHANGED
|
@@ -302,9 +302,10 @@ export class Reader {
|
|
|
302
302
|
}
|
|
303
303
|
// TODO: Tagged fields are not supported yet
|
|
304
304
|
readTaggedFields() {
|
|
305
|
-
const length = this.
|
|
306
|
-
|
|
307
|
-
this.
|
|
305
|
+
const length = this.readUnsignedVarInt();
|
|
306
|
+
for (let i = 0; i < length; i++) {
|
|
307
|
+
this.readUnsignedVarInt();
|
|
308
|
+
this.skip(this.readUnsignedVarInt());
|
|
308
309
|
}
|
|
309
310
|
}
|
|
310
311
|
}
|
|
@@ -4,7 +4,7 @@ import { type MessageToProduce } from '../protocol/records.ts';
|
|
|
4
4
|
export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
|
|
5
5
|
get(id: Id): Schema | undefined;
|
|
6
6
|
fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
|
|
7
|
-
getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
|
|
7
|
+
getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
|
|
8
8
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
|
9
9
|
getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
|
|
10
10
|
getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
|
|
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
|
|
|
14
14
|
export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
|
|
15
15
|
get(_: Id): Schema | undefined;
|
|
16
16
|
fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
|
|
17
|
-
getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
|
|
17
|
+
getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
|
|
18
18
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
|
19
19
|
getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
|
|
20
20
|
getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
|
|
@@ -30,7 +30,7 @@ export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPay
|
|
|
30
30
|
export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
|
|
31
31
|
#private;
|
|
32
32
|
constructor(options: ConfluentSchemaRegistryOptions);
|
|
33
|
-
getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
|
|
33
|
+
getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
|
|
34
34
|
get(id: number): Schema | undefined;
|
|
35
35
|
fetchSchema(id: number, callback: Callback<void>): Promise<void>;
|
|
36
36
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
|
@@ -27,6 +27,9 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
|
|
|
27
27
|
this.#auth = options.auth;
|
|
28
28
|
}
|
|
29
29
|
getSchemaId(message, type) {
|
|
30
|
+
if (message === null) {
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
30
33
|
if (Buffer.isBuffer(message)) {
|
|
31
34
|
if (type !== 'value') {
|
|
32
35
|
return undefined;
|
|
@@ -6,6 +6,4 @@ export type CallbackWithPromise<ReturnType> = Callback<ReturnType> & {
|
|
|
6
6
|
[kCallbackPromise]?: Promise<ReturnType>;
|
|
7
7
|
};
|
|
8
8
|
export declare function createPromisifiedCallback<ReturnType>(): CallbackWithPromise<ReturnType>;
|
|
9
|
-
export declare function runConcurrentCallbacks<
|
|
10
|
-
export declare function runConcurrentCallbacks<ReturnType, V>(errorMessage: string, collection: Set<V>, operation: (item: V, cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;
|
|
11
|
-
export declare function runConcurrentCallbacks<ReturnType, V>(errorMessage: string, collection: V[], operation: (item: V, cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;
|
|
9
|
+
export declare function runConcurrentCallbacks<R, V>(errorMessage: string, collection: Iterable<V>, operation: (item: V, cb: Callback<R>) => void, callback: Callback<R[]>): void;
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
export declare function partitionKey(topic: string, partition: number): string;
|
|
1
|
+
export declare function partitionKey(topic: string, partition: number): string;
|
|
2
|
+
export declare function nextFetchEpoch(epoch: number): number;
|
|
@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
|
|
|
17
17
|
headerValue: Deserializer<HeaderValue>;
|
|
18
18
|
}
|
|
19
19
|
export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
|
|
20
|
-
export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
|
|
20
|
+
export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
|
|
21
21
|
export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
|
|
22
22
|
export declare function stringSerializer(data?: string): Buffer | undefined;
|
|
23
23
|
export declare function stringDeserializer(data?: string | Buffer): string | undefined;
|
|
@@ -17,6 +17,7 @@ export declare class GenericError extends Error {
|
|
|
17
17
|
...rest
|
|
18
18
|
}?: ErrorProperties);
|
|
19
19
|
findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
|
|
20
|
+
toString(): string;
|
|
20
21
|
}
|
|
21
22
|
export declare class MultipleErrors extends AggregateError {
|
|
22
23
|
code: string;
|
|
@@ -31,6 +32,7 @@ export declare class MultipleErrors extends AggregateError {
|
|
|
31
32
|
...rest
|
|
32
33
|
}?: ErrorProperties);
|
|
33
34
|
findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
|
|
35
|
+
toString(): string;
|
|
34
36
|
}
|
|
35
37
|
export * from "./protocol/errors";
|
|
36
38
|
export declare class AuthenticationError extends GenericError {
|
|
@@ -4,7 +4,7 @@ import { type MessageToProduce } from "../protocol/records";
|
|
|
4
4
|
export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
|
|
5
5
|
get(id: Id): Schema | undefined;
|
|
6
6
|
fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
|
|
7
|
-
getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
|
|
7
|
+
getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
|
|
8
8
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
|
9
9
|
getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
|
|
10
10
|
getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
|
|
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
|
|
|
14
14
|
export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
|
|
15
15
|
get(_: Id): Schema | undefined;
|
|
16
16
|
fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
|
|
17
|
-
getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
|
|
17
|
+
getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
|
|
18
18
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
|
19
19
|
getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
|
|
20
20
|
getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
|
|
@@ -30,7 +30,7 @@ export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPay
|
|
|
30
30
|
export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
|
|
31
31
|
#private;
|
|
32
32
|
constructor(options: ConfluentSchemaRegistryOptions);
|
|
33
|
-
getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
|
|
33
|
+
getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
|
|
34
34
|
get(id: number): Schema | undefined;
|
|
35
35
|
fetchSchema(id: number, callback: Callback<void>): Promise<void>;
|
|
36
36
|
getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export const name = "@platformatic/kafka";
|
|
2
|
-
export const version = "2.
|
|
2
|
+
export const version = "2.3.0";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platformatic/kafka",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "Modern and performant client for Apache Kafka",
|
|
5
5
|
"homepage": "https://github.com/platformatic/kafka",
|
|
6
6
|
"author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
|
|
@@ -91,7 +91,7 @@
|
|
|
91
91
|
"typecheck": "tsc -p . --noEmit",
|
|
92
92
|
"format": "prettier -w benchmarks playground regression src test",
|
|
93
93
|
"test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
|
|
94
|
-
"test:ci": "
|
|
94
|
+
"test:ci": "./scripts/node --test --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-lines=90 --test-coverage-branches=90 --test-coverage-functions=90 --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
|
|
95
95
|
"test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
|
|
96
96
|
"test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
|
|
97
97
|
"test:regression": "npm run test:integrity",
|