kafka-ts 1.1.6 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/index.d.ts +122 -121
- package/dist/api/index.js +24 -1
- package/dist/broker.d.ts +0 -1
- package/dist/broker.js +5 -8
- package/dist/cluster.d.ts +2 -2
- package/dist/cluster.js +48 -24
- package/dist/connection.js +9 -13
- package/dist/consumer/consumer-group.d.ts +2 -1
- package/dist/consumer/consumer-group.js +75 -36
- package/dist/consumer/consumer.d.ts +3 -0
- package/dist/consumer/consumer.js +47 -13
- package/dist/producer/producer.d.ts +3 -1
- package/dist/producer/producer.js +85 -86
- package/dist/utils/error.d.ts +1 -4
- package/dist/utils/error.js +5 -9
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.js +20 -5
- package/package.json +1 -1
package/dist/api/index.d.ts
CHANGED
|
@@ -451,125 +451,126 @@ export declare const API: {
|
|
|
451
451
|
};
|
|
452
452
|
export declare const getApiName: <Request, Response>(api: Api<Request, Response>) => string;
|
|
453
453
|
export declare const API_ERROR: {
|
|
454
|
-
UNKNOWN_SERVER_ERROR:
|
|
455
|
-
OFFSET_OUT_OF_RANGE:
|
|
456
|
-
CORRUPT_MESSAGE:
|
|
457
|
-
UNKNOWN_TOPIC_OR_PARTITION:
|
|
458
|
-
INVALID_FETCH_SIZE:
|
|
459
|
-
LEADER_NOT_AVAILABLE:
|
|
460
|
-
NOT_LEADER_OR_FOLLOWER:
|
|
461
|
-
REQUEST_TIMED_OUT:
|
|
462
|
-
BROKER_NOT_AVAILABLE:
|
|
463
|
-
REPLICA_NOT_AVAILABLE:
|
|
464
|
-
MESSAGE_TOO_LARGE:
|
|
465
|
-
STALE_CONTROLLER_EPOCH:
|
|
466
|
-
OFFSET_METADATA_TOO_LARGE:
|
|
467
|
-
NETWORK_EXCEPTION:
|
|
468
|
-
COORDINATOR_LOAD_IN_PROGRESS:
|
|
469
|
-
COORDINATOR_NOT_AVAILABLE:
|
|
470
|
-
NOT_COORDINATOR:
|
|
471
|
-
INVALID_TOPIC_EXCEPTION:
|
|
472
|
-
RECORD_LIST_TOO_LARGE:
|
|
473
|
-
NOT_ENOUGH_REPLICAS:
|
|
474
|
-
NOT_ENOUGH_REPLICAS_AFTER_APPEND:
|
|
475
|
-
INVALID_REQUIRED_ACKS:
|
|
476
|
-
ILLEGAL_GENERATION:
|
|
477
|
-
INCONSISTENT_GROUP_PROTOCOL:
|
|
478
|
-
INVALID_GROUP_ID:
|
|
479
|
-
UNKNOWN_MEMBER_ID:
|
|
480
|
-
INVALID_SESSION_TIMEOUT:
|
|
481
|
-
REBALANCE_IN_PROGRESS:
|
|
482
|
-
INVALID_COMMIT_OFFSET_SIZE:
|
|
483
|
-
TOPIC_AUTHORIZATION_FAILED:
|
|
484
|
-
GROUP_AUTHORIZATION_FAILED:
|
|
485
|
-
CLUSTER_AUTHORIZATION_FAILED:
|
|
486
|
-
INVALID_TIMESTAMP:
|
|
487
|
-
UNSUPPORTED_SASL_MECHANISM:
|
|
488
|
-
ILLEGAL_SASL_STATE:
|
|
489
|
-
UNSUPPORTED_VERSION:
|
|
490
|
-
TOPIC_ALREADY_EXISTS:
|
|
491
|
-
INVALID_PARTITIONS:
|
|
492
|
-
INVALID_REPLICATION_FACTOR:
|
|
493
|
-
INVALID_REPLICA_ASSIGNMENT:
|
|
494
|
-
INVALID_CONFIG:
|
|
495
|
-
NOT_CONTROLLER:
|
|
496
|
-
INVALID_REQUEST:
|
|
497
|
-
UNSUPPORTED_FOR_MESSAGE_FORMAT:
|
|
498
|
-
POLICY_VIOLATION:
|
|
499
|
-
OUT_OF_ORDER_SEQUENCE_NUMBER:
|
|
500
|
-
DUPLICATE_SEQUENCE_NUMBER:
|
|
501
|
-
INVALID_PRODUCER_EPOCH:
|
|
502
|
-
INVALID_TXN_STATE:
|
|
503
|
-
INVALID_PRODUCER_ID_MAPPING:
|
|
504
|
-
INVALID_TRANSACTION_TIMEOUT:
|
|
505
|
-
CONCURRENT_TRANSACTIONS:
|
|
506
|
-
TRANSACTION_COORDINATOR_FENCED:
|
|
507
|
-
TRANSACTIONAL_ID_AUTHORIZATION_FAILED:
|
|
508
|
-
SECURITY_DISABLED:
|
|
509
|
-
OPERATION_NOT_ATTEMPTED:
|
|
510
|
-
KAFKA_STORAGE_ERROR:
|
|
511
|
-
LOG_DIR_NOT_FOUND:
|
|
512
|
-
SASL_AUTHENTICATION_FAILED:
|
|
513
|
-
UNKNOWN_PRODUCER_ID:
|
|
514
|
-
REASSIGNMENT_IN_PROGRESS:
|
|
515
|
-
DELEGATION_TOKEN_AUTH_DISABLED:
|
|
516
|
-
DELEGATION_TOKEN_NOT_FOUND:
|
|
517
|
-
DELEGATION_TOKEN_OWNER_MISMATCH:
|
|
518
|
-
DELEGATION_TOKEN_REQUEST_NOT_ALLOWED:
|
|
519
|
-
DELEGATION_TOKEN_AUTHORIZATION_FAILED:
|
|
520
|
-
DELEGATION_TOKEN_EXPIRED:
|
|
521
|
-
INVALID_PRINCIPAL_TYPE:
|
|
522
|
-
NON_EMPTY_GROUP:
|
|
523
|
-
GROUP_ID_NOT_FOUND:
|
|
524
|
-
FETCH_SESSION_ID_NOT_FOUND:
|
|
525
|
-
INVALID_FETCH_SESSION_EPOCH:
|
|
526
|
-
LISTENER_NOT_FOUND:
|
|
527
|
-
TOPIC_DELETION_DISABLED:
|
|
528
|
-
FENCED_LEADER_EPOCH:
|
|
529
|
-
UNKNOWN_LEADER_EPOCH:
|
|
530
|
-
UNSUPPORTED_COMPRESSION_TYPE:
|
|
531
|
-
STALE_BROKER_EPOCH:
|
|
532
|
-
OFFSET_NOT_AVAILABLE:
|
|
533
|
-
MEMBER_ID_REQUIRED:
|
|
534
|
-
PREFERRED_LEADER_NOT_AVAILABLE:
|
|
535
|
-
GROUP_MAX_SIZE_REACHED:
|
|
536
|
-
FENCED_INSTANCE_ID:
|
|
537
|
-
ELIGIBLE_LEADERS_NOT_AVAILABLE:
|
|
538
|
-
ELECTION_NOT_NEEDED:
|
|
539
|
-
NO_REASSIGNMENT_IN_PROGRESS:
|
|
540
|
-
GROUP_SUBSCRIBED_TO_TOPIC:
|
|
541
|
-
INVALID_RECORD:
|
|
542
|
-
UNSTABLE_OFFSET_COMMIT:
|
|
543
|
-
THROTTLING_QUOTA_EXCEEDED:
|
|
544
|
-
PRODUCER_FENCED:
|
|
545
|
-
RESOURCE_NOT_FOUND:
|
|
546
|
-
DUPLICATE_RESOURCE:
|
|
547
|
-
UNACCEPTABLE_CREDENTIAL:
|
|
548
|
-
INCONSISTENT_VOTER_SET:
|
|
549
|
-
INVALID_UPDATE_VERSION:
|
|
550
|
-
FEATURE_UPDATE_FAILED:
|
|
551
|
-
PRINCIPAL_DESERIALIZATION_FAILURE:
|
|
552
|
-
SNAPSHOT_NOT_FOUND:
|
|
553
|
-
POSITION_OUT_OF_RANGE:
|
|
554
|
-
UNKNOWN_TOPIC_ID:
|
|
555
|
-
DUPLICATE_BROKER_REGISTRATION:
|
|
556
|
-
BROKER_ID_NOT_REGISTERED:
|
|
557
|
-
INCONSISTENT_TOPIC_ID:
|
|
558
|
-
INCONSISTENT_CLUSTER_ID:
|
|
559
|
-
TRANSACTIONAL_ID_NOT_FOUND:
|
|
560
|
-
FETCH_SESSION_TOPIC_ID_ERROR:
|
|
561
|
-
INELIGIBLE_REPLICA:
|
|
562
|
-
NEW_LEADER_ELECTED:
|
|
563
|
-
OFFSET_MOVED_TO_TIERED_STORAGE:
|
|
564
|
-
FENCED_MEMBER_EPOCH:
|
|
565
|
-
UNRELEASED_INSTANCE_ID:
|
|
566
|
-
UNSUPPORTED_ASSIGNOR:
|
|
567
|
-
STALE_MEMBER_EPOCH:
|
|
568
|
-
MISMATCHED_ENDPOINT_TYPE:
|
|
569
|
-
UNSUPPORTED_ENDPOINT_TYPE:
|
|
570
|
-
UNKNOWN_CONTROLLER_ID:
|
|
571
|
-
UNKNOWN_SUBSCRIPTION_ID:
|
|
572
|
-
TELEMETRY_TOO_LARGE:
|
|
573
|
-
INVALID_REGISTRATION:
|
|
574
|
-
TRANSACTION_ABORTABLE:
|
|
454
|
+
readonly UNKNOWN_SERVER_ERROR: -1;
|
|
455
|
+
readonly OFFSET_OUT_OF_RANGE: 1;
|
|
456
|
+
readonly CORRUPT_MESSAGE: 2;
|
|
457
|
+
readonly UNKNOWN_TOPIC_OR_PARTITION: 3;
|
|
458
|
+
readonly INVALID_FETCH_SIZE: 4;
|
|
459
|
+
readonly LEADER_NOT_AVAILABLE: 5;
|
|
460
|
+
readonly NOT_LEADER_OR_FOLLOWER: 6;
|
|
461
|
+
readonly REQUEST_TIMED_OUT: 7;
|
|
462
|
+
readonly BROKER_NOT_AVAILABLE: 8;
|
|
463
|
+
readonly REPLICA_NOT_AVAILABLE: 9;
|
|
464
|
+
readonly MESSAGE_TOO_LARGE: 10;
|
|
465
|
+
readonly STALE_CONTROLLER_EPOCH: 11;
|
|
466
|
+
readonly OFFSET_METADATA_TOO_LARGE: 12;
|
|
467
|
+
readonly NETWORK_EXCEPTION: 13;
|
|
468
|
+
readonly COORDINATOR_LOAD_IN_PROGRESS: 14;
|
|
469
|
+
readonly COORDINATOR_NOT_AVAILABLE: 15;
|
|
470
|
+
readonly NOT_COORDINATOR: 16;
|
|
471
|
+
readonly INVALID_TOPIC_EXCEPTION: 17;
|
|
472
|
+
readonly RECORD_LIST_TOO_LARGE: 18;
|
|
473
|
+
readonly NOT_ENOUGH_REPLICAS: 19;
|
|
474
|
+
readonly NOT_ENOUGH_REPLICAS_AFTER_APPEND: 20;
|
|
475
|
+
readonly INVALID_REQUIRED_ACKS: 21;
|
|
476
|
+
readonly ILLEGAL_GENERATION: 22;
|
|
477
|
+
readonly INCONSISTENT_GROUP_PROTOCOL: 23;
|
|
478
|
+
readonly INVALID_GROUP_ID: 24;
|
|
479
|
+
readonly UNKNOWN_MEMBER_ID: 25;
|
|
480
|
+
readonly INVALID_SESSION_TIMEOUT: 26;
|
|
481
|
+
readonly REBALANCE_IN_PROGRESS: 27;
|
|
482
|
+
readonly INVALID_COMMIT_OFFSET_SIZE: 28;
|
|
483
|
+
readonly TOPIC_AUTHORIZATION_FAILED: 29;
|
|
484
|
+
readonly GROUP_AUTHORIZATION_FAILED: 30;
|
|
485
|
+
readonly CLUSTER_AUTHORIZATION_FAILED: 31;
|
|
486
|
+
readonly INVALID_TIMESTAMP: 32;
|
|
487
|
+
readonly UNSUPPORTED_SASL_MECHANISM: 33;
|
|
488
|
+
readonly ILLEGAL_SASL_STATE: 34;
|
|
489
|
+
readonly UNSUPPORTED_VERSION: 35;
|
|
490
|
+
readonly TOPIC_ALREADY_EXISTS: 36;
|
|
491
|
+
readonly INVALID_PARTITIONS: 37;
|
|
492
|
+
readonly INVALID_REPLICATION_FACTOR: 38;
|
|
493
|
+
readonly INVALID_REPLICA_ASSIGNMENT: 39;
|
|
494
|
+
readonly INVALID_CONFIG: 40;
|
|
495
|
+
readonly NOT_CONTROLLER: 41;
|
|
496
|
+
readonly INVALID_REQUEST: 42;
|
|
497
|
+
readonly UNSUPPORTED_FOR_MESSAGE_FORMAT: 43;
|
|
498
|
+
readonly POLICY_VIOLATION: 44;
|
|
499
|
+
readonly OUT_OF_ORDER_SEQUENCE_NUMBER: 45;
|
|
500
|
+
readonly DUPLICATE_SEQUENCE_NUMBER: 46;
|
|
501
|
+
readonly INVALID_PRODUCER_EPOCH: 47;
|
|
502
|
+
readonly INVALID_TXN_STATE: 48;
|
|
503
|
+
readonly INVALID_PRODUCER_ID_MAPPING: 49;
|
|
504
|
+
readonly INVALID_TRANSACTION_TIMEOUT: 50;
|
|
505
|
+
readonly CONCURRENT_TRANSACTIONS: 51;
|
|
506
|
+
readonly TRANSACTION_COORDINATOR_FENCED: 52;
|
|
507
|
+
readonly TRANSACTIONAL_ID_AUTHORIZATION_FAILED: 53;
|
|
508
|
+
readonly SECURITY_DISABLED: 54;
|
|
509
|
+
readonly OPERATION_NOT_ATTEMPTED: 55;
|
|
510
|
+
readonly KAFKA_STORAGE_ERROR: 56;
|
|
511
|
+
readonly LOG_DIR_NOT_FOUND: 57;
|
|
512
|
+
readonly SASL_AUTHENTICATION_FAILED: 58;
|
|
513
|
+
readonly UNKNOWN_PRODUCER_ID: 59;
|
|
514
|
+
readonly REASSIGNMENT_IN_PROGRESS: 60;
|
|
515
|
+
readonly DELEGATION_TOKEN_AUTH_DISABLED: 61;
|
|
516
|
+
readonly DELEGATION_TOKEN_NOT_FOUND: 62;
|
|
517
|
+
readonly DELEGATION_TOKEN_OWNER_MISMATCH: 63;
|
|
518
|
+
readonly DELEGATION_TOKEN_REQUEST_NOT_ALLOWED: 64;
|
|
519
|
+
readonly DELEGATION_TOKEN_AUTHORIZATION_FAILED: 65;
|
|
520
|
+
readonly DELEGATION_TOKEN_EXPIRED: 66;
|
|
521
|
+
readonly INVALID_PRINCIPAL_TYPE: 67;
|
|
522
|
+
readonly NON_EMPTY_GROUP: 68;
|
|
523
|
+
readonly GROUP_ID_NOT_FOUND: 69;
|
|
524
|
+
readonly FETCH_SESSION_ID_NOT_FOUND: 70;
|
|
525
|
+
readonly INVALID_FETCH_SESSION_EPOCH: 71;
|
|
526
|
+
readonly LISTENER_NOT_FOUND: 72;
|
|
527
|
+
readonly TOPIC_DELETION_DISABLED: 73;
|
|
528
|
+
readonly FENCED_LEADER_EPOCH: 74;
|
|
529
|
+
readonly UNKNOWN_LEADER_EPOCH: 75;
|
|
530
|
+
readonly UNSUPPORTED_COMPRESSION_TYPE: 76;
|
|
531
|
+
readonly STALE_BROKER_EPOCH: 77;
|
|
532
|
+
readonly OFFSET_NOT_AVAILABLE: 78;
|
|
533
|
+
readonly MEMBER_ID_REQUIRED: 79;
|
|
534
|
+
readonly PREFERRED_LEADER_NOT_AVAILABLE: 80;
|
|
535
|
+
readonly GROUP_MAX_SIZE_REACHED: 81;
|
|
536
|
+
readonly FENCED_INSTANCE_ID: 82;
|
|
537
|
+
readonly ELIGIBLE_LEADERS_NOT_AVAILABLE: 83;
|
|
538
|
+
readonly ELECTION_NOT_NEEDED: 84;
|
|
539
|
+
readonly NO_REASSIGNMENT_IN_PROGRESS: 85;
|
|
540
|
+
readonly GROUP_SUBSCRIBED_TO_TOPIC: 86;
|
|
541
|
+
readonly INVALID_RECORD: 87;
|
|
542
|
+
readonly UNSTABLE_OFFSET_COMMIT: 88;
|
|
543
|
+
readonly THROTTLING_QUOTA_EXCEEDED: 89;
|
|
544
|
+
readonly PRODUCER_FENCED: 90;
|
|
545
|
+
readonly RESOURCE_NOT_FOUND: 91;
|
|
546
|
+
readonly DUPLICATE_RESOURCE: 92;
|
|
547
|
+
readonly UNACCEPTABLE_CREDENTIAL: 93;
|
|
548
|
+
readonly INCONSISTENT_VOTER_SET: 94;
|
|
549
|
+
readonly INVALID_UPDATE_VERSION: 95;
|
|
550
|
+
readonly FEATURE_UPDATE_FAILED: 96;
|
|
551
|
+
readonly PRINCIPAL_DESERIALIZATION_FAILURE: 97;
|
|
552
|
+
readonly SNAPSHOT_NOT_FOUND: 98;
|
|
553
|
+
readonly POSITION_OUT_OF_RANGE: 99;
|
|
554
|
+
readonly UNKNOWN_TOPIC_ID: 100;
|
|
555
|
+
readonly DUPLICATE_BROKER_REGISTRATION: 101;
|
|
556
|
+
readonly BROKER_ID_NOT_REGISTERED: 102;
|
|
557
|
+
readonly INCONSISTENT_TOPIC_ID: 103;
|
|
558
|
+
readonly INCONSISTENT_CLUSTER_ID: 104;
|
|
559
|
+
readonly TRANSACTIONAL_ID_NOT_FOUND: 105;
|
|
560
|
+
readonly FETCH_SESSION_TOPIC_ID_ERROR: 106;
|
|
561
|
+
readonly INELIGIBLE_REPLICA: 107;
|
|
562
|
+
readonly NEW_LEADER_ELECTED: 108;
|
|
563
|
+
readonly OFFSET_MOVED_TO_TIERED_STORAGE: 109;
|
|
564
|
+
readonly FENCED_MEMBER_EPOCH: 110;
|
|
565
|
+
readonly UNRELEASED_INSTANCE_ID: 111;
|
|
566
|
+
readonly UNSUPPORTED_ASSIGNOR: 112;
|
|
567
|
+
readonly STALE_MEMBER_EPOCH: 113;
|
|
568
|
+
readonly MISMATCHED_ENDPOINT_TYPE: 114;
|
|
569
|
+
readonly UNSUPPORTED_ENDPOINT_TYPE: 115;
|
|
570
|
+
readonly UNKNOWN_CONTROLLER_ID: 116;
|
|
571
|
+
readonly UNKNOWN_SUBSCRIPTION_ID: 117;
|
|
572
|
+
readonly TELEMETRY_TOO_LARGE: 118;
|
|
573
|
+
readonly INVALID_REGISTRATION: 119;
|
|
574
|
+
readonly TRANSACTION_ABORTABLE: 120;
|
|
575
575
|
};
|
|
576
|
+
export declare const handleApiError: (error: unknown) => Promise<void>;
|
package/dist/api/index.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.API_ERROR = exports.getApiName = exports.API = void 0;
|
|
3
|
+
exports.handleApiError = exports.API_ERROR = exports.getApiName = exports.API = void 0;
|
|
4
|
+
const delay_1 = require("../utils/delay");
|
|
5
|
+
const error_1 = require("../utils/error");
|
|
6
|
+
const logger_1 = require("../utils/logger");
|
|
4
7
|
const api_versions_1 = require("./api-versions");
|
|
5
8
|
const create_topics_1 = require("./create-topics");
|
|
6
9
|
const delete_topics_1 = require("./delete-topics");
|
|
@@ -163,3 +166,23 @@ exports.API_ERROR = {
|
|
|
163
166
|
INVALID_REGISTRATION: 119,
|
|
164
167
|
TRANSACTION_ABORTABLE: 120,
|
|
165
168
|
};
|
|
169
|
+
const handleApiError = async (error) => {
|
|
170
|
+
if (error instanceof error_1.KafkaTSApiError) {
|
|
171
|
+
switch (error.errorCode) {
|
|
172
|
+
case exports.API_ERROR.LEADER_NOT_AVAILABLE:
|
|
173
|
+
logger_1.log.debug('Leader not available yet. Retrying...');
|
|
174
|
+
return (0, delay_1.delay)(500);
|
|
175
|
+
case exports.API_ERROR.COORDINATOR_LOAD_IN_PROGRESS:
|
|
176
|
+
logger_1.log.debug('Coordinator load in progress. Retrying...');
|
|
177
|
+
return (0, delay_1.delay)(100);
|
|
178
|
+
case exports.API_ERROR.COORDINATOR_NOT_AVAILABLE:
|
|
179
|
+
logger_1.log.debug('Coordinator not available yet. Retrying...');
|
|
180
|
+
return (0, delay_1.delay)(100);
|
|
181
|
+
case exports.API_ERROR.OFFSET_NOT_AVAILABLE:
|
|
182
|
+
logger_1.log.debug('Offset not available yet. Retrying...');
|
|
183
|
+
return (0, delay_1.delay)(100);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
throw error;
|
|
187
|
+
};
|
|
188
|
+
exports.handleApiError = handleApiError;
|
package/dist/broker.d.ts
CHANGED
package/dist/broker.js
CHANGED
|
@@ -19,16 +19,13 @@ class Broker {
|
|
|
19
19
|
this.sendRequest = this.connection.sendRequest.bind(this.connection);
|
|
20
20
|
}
|
|
21
21
|
async connect() {
|
|
22
|
-
await this.connection.connect();
|
|
23
|
-
await this.validateApiVersions();
|
|
24
|
-
await this.saslHandshake();
|
|
25
|
-
await this.saslAuthenticate();
|
|
26
|
-
return this;
|
|
27
|
-
}
|
|
28
|
-
async ensureConnected() {
|
|
29
22
|
if (!this.connection.isConnected()) {
|
|
30
|
-
await this.connect();
|
|
23
|
+
await this.connection.connect();
|
|
24
|
+
await this.validateApiVersions();
|
|
25
|
+
await this.saslHandshake();
|
|
26
|
+
await this.saslAuthenticate();
|
|
31
27
|
}
|
|
28
|
+
return this;
|
|
32
29
|
}
|
|
33
30
|
async disconnect() {
|
|
34
31
|
await this.connection.disconnect();
|
package/dist/cluster.d.ts
CHANGED
|
@@ -16,13 +16,13 @@ export declare class Cluster {
|
|
|
16
16
|
private brokerMetadata;
|
|
17
17
|
constructor(options: ClusterOptions);
|
|
18
18
|
connect(): Promise<void>;
|
|
19
|
-
refreshBrokerMetadata(): Promise<void>;
|
|
20
|
-
ensureConnected(): Promise<void>;
|
|
21
19
|
disconnect(): Promise<void>;
|
|
20
|
+
ensureConnected: () => Promise<void>;
|
|
22
21
|
setSeedBroker: (nodeId: number) => Promise<void>;
|
|
23
22
|
sendRequest: SendRequest;
|
|
24
23
|
sendRequestToNode: (nodeId: number) => SendRequest;
|
|
25
24
|
acquireBroker(nodeId: number): Promise<Broker>;
|
|
26
25
|
private findSeedBroker;
|
|
26
|
+
private refreshBrokerMetadata;
|
|
27
27
|
}
|
|
28
28
|
export {};
|
package/dist/cluster.js
CHANGED
|
@@ -14,6 +14,7 @@ const api_1 = require("./api");
|
|
|
14
14
|
const broker_1 = require("./broker");
|
|
15
15
|
const error_1 = require("./utils/error");
|
|
16
16
|
const logger_1 = require("./utils/logger");
|
|
17
|
+
const shared_1 = require("./utils/shared");
|
|
17
18
|
const tracer_1 = require("./utils/tracer");
|
|
18
19
|
const trace = (0, tracer_1.createTracer)('Cluster');
|
|
19
20
|
class Cluster {
|
|
@@ -27,24 +28,6 @@ class Cluster {
|
|
|
27
28
|
async connect() {
|
|
28
29
|
this.seedBroker = await this.findSeedBroker();
|
|
29
30
|
this.brokerById = {};
|
|
30
|
-
await this.refreshBrokerMetadata();
|
|
31
|
-
}
|
|
32
|
-
async refreshBrokerMetadata() {
|
|
33
|
-
const metadata = await this.sendRequest(api_1.API.METADATA, { topics: [] });
|
|
34
|
-
this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
|
|
35
|
-
}
|
|
36
|
-
async ensureConnected() {
|
|
37
|
-
if (!this.seedBroker) {
|
|
38
|
-
return this.connect();
|
|
39
|
-
}
|
|
40
|
-
try {
|
|
41
|
-
await Promise.all([this.seedBroker, ...Object.values(this.brokerById)].map((x) => x.ensureConnected()));
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
logger_1.log.warn('Failed to connect to known brokers, reconnecting...');
|
|
45
|
-
await this.disconnect();
|
|
46
|
-
return this.connect();
|
|
47
|
-
}
|
|
48
31
|
}
|
|
49
32
|
async disconnect() {
|
|
50
33
|
await Promise.all([
|
|
@@ -52,11 +35,45 @@ class Cluster {
|
|
|
52
35
|
...Object.values(this.brokerById).map((x) => x.disconnect()),
|
|
53
36
|
]);
|
|
54
37
|
}
|
|
38
|
+
ensureConnected = (0, shared_1.shared)(async () => {
|
|
39
|
+
if (!this.seedBroker) {
|
|
40
|
+
return this.connect();
|
|
41
|
+
}
|
|
42
|
+
const brokers = [
|
|
43
|
+
{
|
|
44
|
+
broker: this.seedBroker,
|
|
45
|
+
handleError: async (error) => {
|
|
46
|
+
logger_1.log.debug(`Failed to connect to seed broker. Reconnecting...`, { reason: error.message });
|
|
47
|
+
await this.seedBroker?.disconnect();
|
|
48
|
+
this.seedBroker = await this.findSeedBroker();
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
...Object.entries(this.brokerById).map(([nodeId, broker]) => ({
|
|
52
|
+
broker,
|
|
53
|
+
handleError: async (error) => {
|
|
54
|
+
logger_1.log.debug(`Failed to connect to broker ${nodeId}. Disconnecting...`, { reason: error.message });
|
|
55
|
+
await broker.disconnect();
|
|
56
|
+
delete this.brokerById[parseInt(nodeId)];
|
|
57
|
+
},
|
|
58
|
+
})),
|
|
59
|
+
];
|
|
60
|
+
await Promise.all(brokers.map(async ({ broker, handleError }) => {
|
|
61
|
+
try {
|
|
62
|
+
await broker.connect();
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
await handleError(error);
|
|
66
|
+
}
|
|
67
|
+
}));
|
|
68
|
+
});
|
|
55
69
|
setSeedBroker = async (nodeId) => {
|
|
70
|
+
const broker = await this.acquireBroker(nodeId);
|
|
56
71
|
await this.seedBroker?.disconnect();
|
|
57
|
-
this.seedBroker =
|
|
72
|
+
this.seedBroker = broker;
|
|
73
|
+
};
|
|
74
|
+
sendRequest = async (...args) => {
|
|
75
|
+
return this.seedBroker.sendRequest(...args);
|
|
58
76
|
};
|
|
59
|
-
sendRequest = (...args) => this.seedBroker.sendRequest(...args);
|
|
60
77
|
sendRequestToNode = (nodeId) => async (...args) => {
|
|
61
78
|
if (!this.brokerById[nodeId]) {
|
|
62
79
|
this.brokerById[nodeId] = await this.acquireBroker(nodeId);
|
|
@@ -64,9 +81,10 @@ class Cluster {
|
|
|
64
81
|
return this.brokerById[nodeId].sendRequest(...args);
|
|
65
82
|
};
|
|
66
83
|
async acquireBroker(nodeId) {
|
|
67
|
-
if (!(nodeId in this.brokerMetadata))
|
|
68
|
-
|
|
69
|
-
|
|
84
|
+
if (!(nodeId in this.brokerMetadata))
|
|
85
|
+
await this.refreshBrokerMetadata();
|
|
86
|
+
if (!(nodeId in this.brokerMetadata))
|
|
87
|
+
throw new error_1.ConnectionError(`Broker ${nodeId} is not available`);
|
|
70
88
|
const broker = new broker_1.Broker({
|
|
71
89
|
clientId: this.options.clientId,
|
|
72
90
|
sasl: this.options.sasl,
|
|
@@ -92,11 +110,17 @@ class Cluster {
|
|
|
92
110
|
return broker;
|
|
93
111
|
}
|
|
94
112
|
catch (error) {
|
|
95
|
-
logger_1.log.
|
|
113
|
+
logger_1.log.debug(`Failed to connect to seed broker ${options.host}:${options.port}`, {
|
|
114
|
+
reason: error.message,
|
|
115
|
+
});
|
|
96
116
|
}
|
|
97
117
|
}
|
|
98
118
|
throw new error_1.KafkaTSError('No seed brokers found');
|
|
99
119
|
}
|
|
120
|
+
async refreshBrokerMetadata() {
|
|
121
|
+
const metadata = await this.sendRequest(api_1.API.METADATA, { topics: [] });
|
|
122
|
+
this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
|
|
123
|
+
}
|
|
100
124
|
}
|
|
101
125
|
exports.Cluster = Cluster;
|
|
102
126
|
__decorate([
|
package/dist/connection.js
CHANGED
|
@@ -71,6 +71,7 @@ class Connection {
|
|
|
71
71
|
async connect() {
|
|
72
72
|
this.queue = {};
|
|
73
73
|
this.chunks = [];
|
|
74
|
+
const { stack } = new Error();
|
|
74
75
|
await new Promise((resolve, reject) => {
|
|
75
76
|
const { ssl, connection } = this.options;
|
|
76
77
|
this.socket = ssl
|
|
@@ -81,14 +82,16 @@ class Connection {
|
|
|
81
82
|
}, resolve)
|
|
82
83
|
: net_1.default.connect(connection, resolve);
|
|
83
84
|
this.socket.setKeepAlive(true, 30_000);
|
|
84
|
-
this.socket.once('error',
|
|
85
|
+
this.socket.once('error', (error) => {
|
|
86
|
+
reject(new error_1.ConnectionError(error.message, stack));
|
|
87
|
+
});
|
|
85
88
|
});
|
|
86
89
|
this.socket.removeAllListeners('error');
|
|
87
90
|
this.socket.on('error', (error) => logger_1.log.debug('Socket error', { error }));
|
|
88
91
|
this.socket.on('data', (data) => this.handleData(data));
|
|
89
92
|
this.socket.once('close', async () => {
|
|
90
93
|
Object.values(this.queue).forEach(({ reject }) => {
|
|
91
|
-
reject(new error_1.ConnectionError('Socket closed unexpectedly'));
|
|
94
|
+
reject(new error_1.ConnectionError('Socket closed unexpectedly', stack));
|
|
92
95
|
});
|
|
93
96
|
this.queue = {};
|
|
94
97
|
});
|
|
@@ -112,18 +115,19 @@ class Connection {
|
|
|
112
115
|
.writeString(this.options.clientId);
|
|
113
116
|
const request = api.request(encoder, body);
|
|
114
117
|
const requestEncoder = new encoder_1.Encoder().writeInt32(request.getBufferLength()).writeEncoder(request);
|
|
118
|
+
const { stack } = new Error();
|
|
115
119
|
let timeout;
|
|
116
120
|
const { responseDecoder, responseSize } = await new Promise(async (resolve, reject) => {
|
|
117
121
|
timeout = setTimeout(() => {
|
|
118
122
|
delete this.queue[correlationId];
|
|
119
|
-
reject(new error_1.ConnectionError(`${apiName} timed out
|
|
123
|
+
reject(new error_1.ConnectionError(`${apiName} timed out`, stack));
|
|
120
124
|
}, this.options.requestTimeout);
|
|
121
125
|
try {
|
|
122
126
|
this.queue[correlationId] = { resolve, reject };
|
|
123
127
|
await this.write(requestEncoder.value());
|
|
124
128
|
}
|
|
125
129
|
catch (error) {
|
|
126
|
-
reject(error);
|
|
130
|
+
reject(new error_1.ConnectionError(error.message, stack));
|
|
127
131
|
}
|
|
128
132
|
});
|
|
129
133
|
clearTimeout(timeout);
|
|
@@ -142,15 +146,7 @@ class Connection {
|
|
|
142
146
|
}
|
|
143
147
|
write(buffer) {
|
|
144
148
|
return new Promise((resolve, reject) => {
|
|
145
|
-
|
|
146
|
-
this.socket.write(buffer, 'binary', (error) => {
|
|
147
|
-
if (error) {
|
|
148
|
-
const err = new error_1.ConnectionError(error.message);
|
|
149
|
-
err.stack += `\n${stack}`;
|
|
150
|
-
return reject(err);
|
|
151
|
-
}
|
|
152
|
-
resolve();
|
|
153
|
-
});
|
|
149
|
+
this.socket.write(buffer, 'binary', (error) => (error ? reject(error) : resolve()));
|
|
154
150
|
});
|
|
155
151
|
}
|
|
156
152
|
handleData(buffer) {
|
|
@@ -28,12 +28,13 @@ export declare class ConsumerGroup {
|
|
|
28
28
|
private startHeartbeater;
|
|
29
29
|
private stopHeartbeater;
|
|
30
30
|
handleLastHeartbeat(): void;
|
|
31
|
-
|
|
31
|
+
findCoordinator(): Promise<void>;
|
|
32
32
|
private joinGroup;
|
|
33
33
|
private syncGroup;
|
|
34
34
|
private offsetFetch;
|
|
35
35
|
offsetCommit(topicPartitions: Record<string, Set<number>>): Promise<void>;
|
|
36
36
|
heartbeat(): Promise<void>;
|
|
37
37
|
leaveGroup(): Promise<void>;
|
|
38
|
+
private handleError;
|
|
38
39
|
}
|
|
39
40
|
export {};
|
|
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
12
12
|
exports.ConsumerGroup = void 0;
|
|
13
13
|
const api_1 = require("../api");
|
|
14
14
|
const find_coordinator_1 = require("../api/find-coordinator");
|
|
15
|
+
const error_1 = require("../utils/error");
|
|
16
|
+
const logger_1 = require("../utils/logger");
|
|
15
17
|
const tracer_1 = require("../utils/tracer");
|
|
16
18
|
const trace = (0, tracer_1.createTracer)('ConsumerGroup');
|
|
17
19
|
class ConsumerGroup {
|
|
@@ -28,7 +30,6 @@ class ConsumerGroup {
|
|
|
28
30
|
}
|
|
29
31
|
async init() {
|
|
30
32
|
await this.findCoordinator();
|
|
31
|
-
await this.options.cluster.setSeedBroker(this.coordinatorId);
|
|
32
33
|
this.memberId = '';
|
|
33
34
|
}
|
|
34
35
|
async join() {
|
|
@@ -61,11 +62,19 @@ class ConsumerGroup {
|
|
|
61
62
|
}
|
|
62
63
|
}
|
|
63
64
|
async findCoordinator() {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
try {
|
|
66
|
+
const { coordinators } = await this.options.cluster.sendRequest(api_1.API.FIND_COORDINATOR, {
|
|
67
|
+
keyType: find_coordinator_1.KEY_TYPE.GROUP,
|
|
68
|
+
keys: [this.options.groupId],
|
|
69
|
+
});
|
|
70
|
+
this.coordinatorId = coordinators[0].nodeId;
|
|
71
|
+
await this.options.cluster.setSeedBroker(this.coordinatorId);
|
|
72
|
+
this.heartbeatError = null;
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
await this.handleError(error);
|
|
76
|
+
return this.findCoordinator();
|
|
77
|
+
}
|
|
69
78
|
}
|
|
70
79
|
async joinGroup() {
|
|
71
80
|
const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
|
|
@@ -86,11 +95,8 @@ class ConsumerGroup {
|
|
|
86
95
|
this.memberIds = response.members.map((member) => member.memberId);
|
|
87
96
|
}
|
|
88
97
|
catch (error) {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
return this.joinGroup();
|
|
92
|
-
}
|
|
93
|
-
throw error;
|
|
98
|
+
await this.handleError(error);
|
|
99
|
+
return this.joinGroup();
|
|
94
100
|
}
|
|
95
101
|
}
|
|
96
102
|
async syncGroup() {
|
|
@@ -108,16 +114,22 @@ class ConsumerGroup {
|
|
|
108
114
|
}, {});
|
|
109
115
|
assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
|
|
110
116
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
117
|
+
try {
|
|
118
|
+
const response = await cluster.sendRequest(api_1.API.SYNC_GROUP, {
|
|
119
|
+
groupId,
|
|
120
|
+
groupInstanceId,
|
|
121
|
+
memberId: this.memberId,
|
|
122
|
+
generationId: this.generationId,
|
|
123
|
+
protocolType: 'consumer',
|
|
124
|
+
protocolName: 'RoundRobinAssigner',
|
|
125
|
+
assignments,
|
|
126
|
+
});
|
|
127
|
+
metadata.setAssignment(response.assignments);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
await this.handleError(error);
|
|
131
|
+
return this.syncGroup();
|
|
132
|
+
}
|
|
121
133
|
}
|
|
122
134
|
async offsetFetch() {
|
|
123
135
|
const { cluster, groupId, topics, metadata, offsetManager } = this.options;
|
|
@@ -135,20 +147,26 @@ class ConsumerGroup {
|
|
|
135
147
|
};
|
|
136
148
|
if (!request.groups.length)
|
|
137
149
|
return;
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
150
|
+
try {
|
|
151
|
+
const response = await cluster.sendRequest(api_1.API.OFFSET_FETCH, request);
|
|
152
|
+
const topicPartitions = {};
|
|
153
|
+
response.groups.forEach((group) => {
|
|
154
|
+
group.topics.forEach((topic) => {
|
|
155
|
+
topicPartitions[topic.name] ??= new Set();
|
|
156
|
+
topic.partitions.forEach(({ partitionIndex, committedOffset }) => {
|
|
157
|
+
if (committedOffset >= 0) {
|
|
158
|
+
topicPartitions[topic.name].add(partitionIndex);
|
|
159
|
+
offsetManager.resolve(topic.name, partitionIndex, committedOffset);
|
|
160
|
+
}
|
|
161
|
+
});
|
|
148
162
|
});
|
|
149
163
|
});
|
|
150
|
-
|
|
151
|
-
|
|
164
|
+
offsetManager.flush(topicPartitions);
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
await this.handleError(error);
|
|
168
|
+
return this.offsetFetch();
|
|
169
|
+
}
|
|
152
170
|
}
|
|
153
171
|
async offsetCommit(topicPartitions) {
|
|
154
172
|
const { cluster, groupId, groupInstanceId, offsetManager, consumer } = this.options;
|
|
@@ -174,7 +192,13 @@ class ConsumerGroup {
|
|
|
174
192
|
if (!request.topics.length) {
|
|
175
193
|
return;
|
|
176
194
|
}
|
|
177
|
-
|
|
195
|
+
try {
|
|
196
|
+
await cluster.sendRequest(api_1.API.OFFSET_COMMIT, request);
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
await this.handleError(error);
|
|
200
|
+
return this.offsetCommit(topicPartitions);
|
|
201
|
+
}
|
|
178
202
|
consumer.emit('offsetCommit');
|
|
179
203
|
}
|
|
180
204
|
async heartbeat() {
|
|
@@ -200,12 +224,27 @@ class ConsumerGroup {
|
|
|
200
224
|
});
|
|
201
225
|
}
|
|
202
226
|
catch (error) {
|
|
203
|
-
if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
|
|
227
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
|
|
204
228
|
return;
|
|
205
229
|
}
|
|
206
|
-
|
|
230
|
+
await this.handleError(error);
|
|
231
|
+
return this.leaveGroup();
|
|
207
232
|
}
|
|
208
233
|
}
|
|
234
|
+
async handleError(error) {
|
|
235
|
+
await (0, api_1.handleApiError)(error).catch(async (error) => {
|
|
236
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_COORDINATOR) {
|
|
237
|
+
logger_1.log.debug('Not coordinator. Searching for new coordinator...');
|
|
238
|
+
await this.findCoordinator();
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.MEMBER_ID_REQUIRED) {
|
|
242
|
+
this.memberId = error.response.memberId;
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
throw error;
|
|
246
|
+
});
|
|
247
|
+
}
|
|
209
248
|
}
|
|
210
249
|
exports.ConsumerGroup = ConsumerGroup;
|
|
211
250
|
__decorate([
|
|
@@ -75,13 +75,12 @@ class Consumer extends events_1.default {
|
|
|
75
75
|
: undefined;
|
|
76
76
|
}
|
|
77
77
|
async start() {
|
|
78
|
-
const { topics, allowTopicAutoCreation, fromTimestamp } = this.options;
|
|
79
78
|
this.stopHook = undefined;
|
|
80
79
|
try {
|
|
81
80
|
await this.cluster.connect();
|
|
82
|
-
await this.
|
|
81
|
+
await this.fetchMetadata();
|
|
83
82
|
this.metadata.setAssignment(this.metadata.getTopicPartitions());
|
|
84
|
-
await this.
|
|
83
|
+
await this.fetchOffsets();
|
|
85
84
|
await this.consumerGroup?.init();
|
|
86
85
|
}
|
|
87
86
|
catch (error) {
|
|
@@ -101,8 +100,8 @@ class Consumer extends events_1.default {
|
|
|
101
100
|
await this.fetchManager?.stop();
|
|
102
101
|
});
|
|
103
102
|
}
|
|
104
|
-
await this.consumerGroup?.leaveGroup().catch((error) => logger_1.log.debug(
|
|
105
|
-
await this.cluster.disconnect().catch((
|
|
103
|
+
await this.consumerGroup?.leaveGroup().catch((error) => logger_1.log.debug('Failed to leave group', { reason: error.message }));
|
|
104
|
+
await this.cluster.disconnect().catch(() => { });
|
|
106
105
|
}
|
|
107
106
|
async startFetchManager() {
|
|
108
107
|
const { groupId } = this.options;
|
|
@@ -129,18 +128,22 @@ class Consumer extends events_1.default {
|
|
|
129
128
|
}
|
|
130
129
|
catch (error) {
|
|
131
130
|
await this.fetchManager?.stop();
|
|
132
|
-
if (error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
|
|
131
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
|
|
133
132
|
logger_1.log.debug('Rebalance in progress...', { apiName: error.apiName, groupId });
|
|
134
133
|
continue;
|
|
135
134
|
}
|
|
136
|
-
if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
|
|
135
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
|
|
137
136
|
logger_1.log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
|
|
138
137
|
this.close();
|
|
139
138
|
break;
|
|
140
139
|
}
|
|
141
|
-
if (error instanceof error_1.
|
|
142
|
-
(
|
|
143
|
-
|
|
140
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_COORDINATOR) {
|
|
141
|
+
logger_1.log.debug('Not coordinator. Searching for new coordinator...');
|
|
142
|
+
await this.consumerGroup?.findCoordinator();
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (error instanceof error_1.ConnectionError) {
|
|
146
|
+
logger_1.log.debug(`${error.message}. Restarting consumer...`, { stack: error.stack });
|
|
144
147
|
this.close().then(() => this.start());
|
|
145
148
|
break;
|
|
146
149
|
}
|
|
@@ -233,13 +236,44 @@ class Consumer extends events_1.default {
|
|
|
233
236
|
});
|
|
234
237
|
}
|
|
235
238
|
catch (error) {
|
|
239
|
+
await this.handleError(error);
|
|
240
|
+
return this.fetch(nodeId, assignment);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
async fetchMetadata() {
|
|
244
|
+
const { topics, allowTopicAutoCreation } = this.options;
|
|
245
|
+
try {
|
|
246
|
+
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
247
|
+
}
|
|
248
|
+
catch (error) {
|
|
249
|
+
await this.handleError(error);
|
|
250
|
+
return this.fetchMetadata();
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
async fetchOffsets() {
|
|
254
|
+
const { fromTimestamp } = this.options;
|
|
255
|
+
try {
|
|
256
|
+
await this.offsetManager.fetchOffsets({ fromTimestamp });
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
await this.handleError(error);
|
|
260
|
+
return this.fetchOffsets();
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
async handleError(error) {
|
|
264
|
+
await (0, api_1.handleApiError)(error).catch(async (error) => {
|
|
265
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
|
|
266
|
+
logger_1.log.debug('Refreshing metadata', { reason: error.message });
|
|
267
|
+
await this.fetchMetadata();
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
236
270
|
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OFFSET_OUT_OF_RANGE) {
|
|
237
271
|
logger_1.log.warn('Offset out of range. Resetting offsets.');
|
|
238
|
-
await this.
|
|
239
|
-
return
|
|
272
|
+
await this.fetchOffsets();
|
|
273
|
+
return;
|
|
240
274
|
}
|
|
241
275
|
throw error;
|
|
242
|
-
}
|
|
276
|
+
});
|
|
243
277
|
}
|
|
244
278
|
}
|
|
245
279
|
exports.Consumer = Consumer;
|
|
@@ -19,8 +19,10 @@ export declare class Producer {
|
|
|
19
19
|
acks?: -1 | 1;
|
|
20
20
|
}): Promise<void>;
|
|
21
21
|
close(): Promise<void>;
|
|
22
|
-
private
|
|
22
|
+
private ensureProducerInitialized;
|
|
23
23
|
private initProducerId;
|
|
24
24
|
private getSequence;
|
|
25
25
|
private updateSequence;
|
|
26
|
+
private fetchMetadata;
|
|
27
|
+
private handleError;
|
|
26
28
|
}
|
|
@@ -14,7 +14,6 @@ const api_1 = require("../api");
|
|
|
14
14
|
const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
|
|
15
15
|
const partitioner_1 = require("../distributors/partitioner");
|
|
16
16
|
const metadata_1 = require("../metadata");
|
|
17
|
-
const delay_1 = require("../utils/delay");
|
|
18
17
|
const error_1 = require("../utils/error");
|
|
19
18
|
const lock_1 = require("../utils/lock");
|
|
20
19
|
const logger_1 = require("../utils/logger");
|
|
@@ -41,7 +40,7 @@ class Producer {
|
|
|
41
40
|
this.partition = this.options.partitioner({ metadata: this.metadata });
|
|
42
41
|
}
|
|
43
42
|
async send(messages, { acks = -1 } = {}) {
|
|
44
|
-
await this.
|
|
43
|
+
await this.ensureProducerInitialized();
|
|
45
44
|
const { allowTopicAutoCreation } = this.options;
|
|
46
45
|
const defaultTimestamp = BigInt(Date.now());
|
|
47
46
|
const topics = new Set(messages.map((message) => message.topic));
|
|
@@ -51,96 +50,74 @@ class Producer {
|
|
|
51
50
|
return message;
|
|
52
51
|
});
|
|
53
52
|
const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
53
|
+
await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
|
|
54
|
+
try {
|
|
55
|
+
await this.lock.acquire([`node:${nodeId}`], async () => {
|
|
56
|
+
const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
|
|
57
|
+
name: topic,
|
|
58
|
+
partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
|
|
59
|
+
const partitionIndex = parseInt(partition);
|
|
60
|
+
let baseTimestamp;
|
|
61
|
+
let maxTimestamp;
|
|
62
|
+
messages.forEach(({ timestamp = defaultTimestamp }) => {
|
|
63
|
+
if (!baseTimestamp || timestamp < baseTimestamp) {
|
|
64
|
+
baseTimestamp = timestamp;
|
|
65
|
+
}
|
|
66
|
+
if (!maxTimestamp || timestamp > maxTimestamp) {
|
|
67
|
+
maxTimestamp = timestamp;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
return {
|
|
71
|
+
index: partitionIndex,
|
|
72
|
+
baseOffset: 0n,
|
|
73
|
+
partitionLeaderEpoch: -1,
|
|
74
|
+
attributes: 0,
|
|
75
|
+
lastOffsetDelta: messages.length - 1,
|
|
76
|
+
baseTimestamp: baseTimestamp ?? 0n,
|
|
77
|
+
maxTimestamp: maxTimestamp ?? 0n,
|
|
78
|
+
producerId: this.producerId,
|
|
79
|
+
producerEpoch: 0,
|
|
80
|
+
baseSequence: this.getSequence(topic, partitionIndex),
|
|
81
|
+
records: messages.map((message, index) => ({
|
|
76
82
|
attributes: 0,
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
attributes: 0,
|
|
85
|
-
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
|
|
86
|
-
offsetDelta: index,
|
|
87
|
-
key: message.key ?? null,
|
|
88
|
-
value: message.value,
|
|
89
|
-
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
|
|
90
|
-
key,
|
|
91
|
-
value,
|
|
92
|
-
})),
|
|
83
|
+
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
|
|
84
|
+
offsetDelta: index,
|
|
85
|
+
key: message.key ?? null,
|
|
86
|
+
value: message.value,
|
|
87
|
+
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
|
|
88
|
+
key,
|
|
89
|
+
value,
|
|
93
90
|
})),
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
91
|
+
})),
|
|
92
|
+
};
|
|
93
|
+
}),
|
|
94
|
+
}));
|
|
95
|
+
await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
|
|
96
|
+
transactionalId: null,
|
|
97
|
+
acks,
|
|
98
|
+
timeoutMs: 30000,
|
|
99
|
+
topicData,
|
|
100
|
+
});
|
|
101
|
+
topicData.forEach(({ name, partitionData }) => {
|
|
102
|
+
partitionData.forEach(({ index, records }) => {
|
|
103
|
+
this.updateSequence(name, index, records.length);
|
|
107
104
|
});
|
|
108
105
|
});
|
|
109
|
-
}
|
|
110
|
-
catch (error) {
|
|
111
|
-
if (error instanceof error_1.BrokerNotAvailableError || (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER)) {
|
|
112
|
-
logger_1.log.debug('Refreshing broker metadata', { reason: error.message, nodeId });
|
|
113
|
-
await this.cluster.refreshBrokerMetadata();
|
|
114
|
-
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
115
|
-
const messages = Object.values(topicPartitionMessages).flatMap(partitionMessages => Object.values(partitionMessages).flat()).map(({ partition, ...message }) => message);
|
|
116
|
-
return this.send(messages, { acks });
|
|
117
|
-
}
|
|
118
|
-
throw error;
|
|
119
|
-
}
|
|
120
|
-
}));
|
|
121
|
-
}
|
|
122
|
-
catch (error) {
|
|
123
|
-
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
|
|
124
|
-
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
125
|
-
}
|
|
126
|
-
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
|
|
127
|
-
await this.initProducerId();
|
|
128
|
-
}
|
|
129
|
-
logger_1.log.warn('Reconnecting producer due to an unhandled error', { error });
|
|
130
|
-
try {
|
|
131
|
-
await this.cluster.disconnect();
|
|
132
|
-
await this.cluster.connect();
|
|
106
|
+
});
|
|
133
107
|
}
|
|
134
108
|
catch (error) {
|
|
135
|
-
|
|
109
|
+
await this.handleError(error);
|
|
110
|
+
const messages = Object.values(topicPartitionMessages)
|
|
111
|
+
.flatMap((partitionMessages) => Object.values(partitionMessages).flat())
|
|
112
|
+
.map(({ partition, ...message }) => message);
|
|
113
|
+
return this.send(messages, { acks });
|
|
136
114
|
}
|
|
137
|
-
|
|
138
|
-
}
|
|
115
|
+
}));
|
|
139
116
|
}
|
|
140
117
|
async close() {
|
|
141
118
|
await this.cluster.disconnect();
|
|
142
119
|
}
|
|
143
|
-
|
|
120
|
+
ensureProducerInitialized = (0, shared_1.shared)(async () => {
|
|
144
121
|
await this.cluster.ensureConnected();
|
|
145
122
|
if (!this.producerId) {
|
|
146
123
|
await this.initProducerId();
|
|
@@ -159,11 +136,8 @@ class Producer {
|
|
|
159
136
|
this.sequences = {};
|
|
160
137
|
}
|
|
161
138
|
catch (error) {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
return this.initProducerId();
|
|
165
|
-
}
|
|
166
|
-
throw error;
|
|
139
|
+
await this.handleError(error);
|
|
140
|
+
return this.initProducerId();
|
|
167
141
|
}
|
|
168
142
|
}
|
|
169
143
|
getSequence(topic, partition) {
|
|
@@ -174,6 +148,31 @@ class Producer {
|
|
|
174
148
|
this.sequences[topic][partition] ??= 0;
|
|
175
149
|
this.sequences[topic][partition] += messagesCount;
|
|
176
150
|
}
|
|
151
|
+
async fetchMetadata(topics, allowTopicAutoCreation) {
|
|
152
|
+
try {
|
|
153
|
+
await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
await this.handleError(error);
|
|
157
|
+
return this.fetchMetadata(topics, allowTopicAutoCreation);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
async handleError(error) {
|
|
161
|
+
await (0, api_1.handleApiError)(error).catch(async (error) => {
|
|
162
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
|
|
163
|
+
logger_1.log.debug('Refreshing metadata', { reason: error.message });
|
|
164
|
+
const topics = Object.keys(this.metadata.getTopicPartitions());
|
|
165
|
+
await this.fetchMetadata(topics, false);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
|
|
169
|
+
logger_1.log.debug('Out of order sequence number. Reinitializing producer ID');
|
|
170
|
+
await this.initProducerId();
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
throw error;
|
|
174
|
+
});
|
|
175
|
+
}
|
|
177
176
|
}
|
|
178
177
|
exports.Producer = Producer;
|
|
179
178
|
__decorate([
|
package/dist/utils/error.d.ts
CHANGED
|
@@ -9,9 +9,6 @@ export declare class KafkaTSApiError<T = any> extends KafkaTSError {
|
|
|
9
9
|
request: unknown | undefined;
|
|
10
10
|
constructor(errorCode: number, errorMessage: string | null, response: T);
|
|
11
11
|
}
|
|
12
|
-
export declare class BrokerNotAvailableError extends KafkaTSError {
|
|
13
|
-
brokerId: number;
|
|
14
|
-
constructor(brokerId: number);
|
|
15
|
-
}
|
|
16
12
|
export declare class ConnectionError extends KafkaTSError {
|
|
13
|
+
constructor(message: string, stack?: string);
|
|
17
14
|
}
|
package/dist/utils/error.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ConnectionError = exports.
|
|
3
|
+
exports.ConnectionError = exports.KafkaTSApiError = exports.KafkaTSError = void 0;
|
|
4
4
|
const api_1 = require("../api");
|
|
5
5
|
class KafkaTSError extends Error {
|
|
6
6
|
constructor(message) {
|
|
@@ -24,14 +24,10 @@ class KafkaTSApiError extends KafkaTSError {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
exports.KafkaTSApiError = KafkaTSApiError;
|
|
27
|
-
class BrokerNotAvailableError extends KafkaTSError {
|
|
28
|
-
brokerId;
|
|
29
|
-
constructor(brokerId) {
|
|
30
|
-
super(`Broker ${brokerId} is not available`);
|
|
31
|
-
this.brokerId = brokerId;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
exports.BrokerNotAvailableError = BrokerNotAvailableError;
|
|
35
27
|
class ConnectionError extends KafkaTSError {
|
|
28
|
+
constructor(message, stack) {
|
|
29
|
+
super(message);
|
|
30
|
+
this.stack += `\n${stack}`;
|
|
31
|
+
}
|
|
36
32
|
}
|
|
37
33
|
exports.ConnectionError = ConnectionError;
|
package/dist/utils/logger.d.ts
CHANGED
|
@@ -4,6 +4,13 @@ export interface Logger {
|
|
|
4
4
|
warn: (message: string, metadata?: unknown) => void;
|
|
5
5
|
error: (message: string, metadata?: unknown) => void;
|
|
6
6
|
}
|
|
7
|
+
export declare enum LogLevel {
|
|
8
|
+
DEBUG = 0,
|
|
9
|
+
INFO = 1,
|
|
10
|
+
WARNING = 2,
|
|
11
|
+
ERROR = 3
|
|
12
|
+
}
|
|
7
13
|
export declare const jsonSerializer: (_: unknown, v: unknown) => unknown;
|
|
8
14
|
export declare let log: Logger;
|
|
9
15
|
export declare const setLogger: (newLogger: Logger) => void;
|
|
16
|
+
export declare const setLogLevel: (newLogLevel: LogLevel) => void;
|
package/dist/utils/logger.js
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.setLogger = exports.log = exports.jsonSerializer = void 0;
|
|
3
|
+
exports.setLogLevel = exports.setLogger = exports.log = exports.jsonSerializer = exports.LogLevel = void 0;
|
|
4
|
+
var LogLevel;
|
|
5
|
+
(function (LogLevel) {
|
|
6
|
+
LogLevel[LogLevel["DEBUG"] = 0] = "DEBUG";
|
|
7
|
+
LogLevel[LogLevel["INFO"] = 1] = "INFO";
|
|
8
|
+
LogLevel[LogLevel["WARNING"] = 2] = "WARNING";
|
|
9
|
+
LogLevel[LogLevel["ERROR"] = 3] = "ERROR";
|
|
10
|
+
})(LogLevel || (exports.LogLevel = LogLevel = {}));
|
|
4
11
|
const jsonSerializer = (_, v) => {
|
|
5
12
|
if (v instanceof Error) {
|
|
6
13
|
return Object.getOwnPropertyNames(v).reduce((acc, key) => {
|
|
@@ -19,16 +26,19 @@ const jsonSerializer = (_, v) => {
|
|
|
19
26
|
exports.jsonSerializer = jsonSerializer;
|
|
20
27
|
class JsonLogger {
|
|
21
28
|
debug(message, metadata) {
|
|
22
|
-
|
|
29
|
+
logLevel <= LogLevel.DEBUG &&
|
|
30
|
+
console.debug(JSON.stringify({ message, metadata, level: 'debug' }, exports.jsonSerializer));
|
|
23
31
|
}
|
|
24
32
|
info(message, metadata) {
|
|
25
|
-
console.info(JSON.stringify({ message, metadata, level: 'info' }, exports.jsonSerializer));
|
|
33
|
+
logLevel <= LogLevel.INFO && console.info(JSON.stringify({ message, metadata, level: 'info' }, exports.jsonSerializer));
|
|
26
34
|
}
|
|
27
35
|
warn(message, metadata) {
|
|
28
|
-
|
|
36
|
+
logLevel <= LogLevel.WARNING &&
|
|
37
|
+
console.warn(JSON.stringify({ message, metadata, level: 'warning' }, exports.jsonSerializer));
|
|
29
38
|
}
|
|
30
39
|
error(message, metadata) {
|
|
31
|
-
|
|
40
|
+
logLevel <= LogLevel.ERROR &&
|
|
41
|
+
console.error(JSON.stringify({ message, metadata, level: 'error' }, exports.jsonSerializer));
|
|
32
42
|
}
|
|
33
43
|
}
|
|
34
44
|
exports.log = new JsonLogger();
|
|
@@ -36,3 +46,8 @@ const setLogger = (newLogger) => {
|
|
|
36
46
|
exports.log = newLogger;
|
|
37
47
|
};
|
|
38
48
|
exports.setLogger = setLogger;
|
|
49
|
+
let logLevel = LogLevel.INFO;
|
|
50
|
+
const setLogLevel = (newLogLevel) => {
|
|
51
|
+
logLevel = newLogLevel;
|
|
52
|
+
};
|
|
53
|
+
exports.setLogLevel = setLogLevel;
|