@platformatic/kafka 1.23.0 → 1.24.0

package/dist/clients/producer/producer.js

@@ -1,12 +1,15 @@
+import { randomUUID } from 'crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
-import { ProduceAcks } from "../../apis/enumerations.js";
-import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel } from "../../diagnostic.js";
+import { FindCoordinatorKeyTypes, ProduceAcks } from "../../apis/enumerations.js";
+import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel, producerTransactionsChannel } from "../../diagnostic.js";
 import { UserError } from "../../errors.js";
 import { murmur2 } from "../../protocol/murmur2.js";
+import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator, kTransactionPrepare } from "../../symbols.js";
 import { NumericMap } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
+import { Transaction } from "./transaction.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
 export function noopSerializer(data) {
@@ -23,6 +26,8 @@ export class Producer extends Base {
     #headerKeySerializer;
     #headerValueSerializer;
     #metricsProducedMessages;
+    #coordinatorId;
+    #transaction;
     constructor(options) {
         if (options.idempotent) {
             options.maxInflights = 1;
@@ -40,6 +45,7 @@ export class Producer extends Base {
         this.#valueSerializer = options.serializers?.value ?? noopSerializer;
         this.#headerKeySerializer = options.serializers?.headerKey ?? noopSerializer;
         this.#headerValueSerializer = options.serializers?.headerValue ?? noopSerializer;
+        this[kOptions].transactionalId ??= randomUUID();
         this[kValidateOptions](options, producerOptionsValidator, '/options');
         if (this[kPrometheus]) {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_producers', 'Number of active Kafka producers').inc();
@@ -53,6 +59,12 @@ export class Producer extends Base {
     get producerEpoch() {
         return this.#producerInfo?.producerEpoch;
     }
+    get transaction() {
+        return this.#transaction;
+    }
+    get coordinatorId() {
+        return this.#coordinatorId;
+    }
     close(callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -84,7 +96,7 @@ export class Producer extends Base {
         }
         const validationError = this[kValidateOptions](options, produceOptionsValidator, '/options', false);
         if (validationError) {
-            callback(validationError, undefined);
+            callback(validationError);
             return callback[kCallbackPromise];
         }
         producerInitIdempotentChannel.traceCallback(this.#initIdempotentProducer, 1, createDiagnosticContext({ client: this, operation: 'initIdempotentProducer', options }), this, options, callback);
@@ -97,36 +109,284 @@ export class Producer extends Base {
         if (this[kCheckNotClosed](callback)) {
             return callback[kCallbackPromise];
         }
+        if (this.#transaction && options[kTransaction] !== this.#transaction[kInstance]) {
+            callback(new UserError(options[kTransaction]
+                ? 'The producer is in use by another transaction.'
+                : 'The producer is in use by a transaction.'));
+            return callback[kCallbackPromise];
+        }
         const validationError = this[kValidateOptions](options, sendOptionsValidator, '/options', false);
         if (validationError) {
-            callback(validationError, undefined);
+            callback(validationError);
             return callback[kCallbackPromise];
         }
+        const idempotent = options.idempotent ?? this[kOptions].idempotent;
+        if (idempotent) {
+            if (typeof options.producerId !== 'undefined' || typeof options.producerEpoch !== 'undefined') {
+                callback(new UserError('Cannot specify producerId or producerEpoch when using idempotent producer.'));
+                return callback[kCallbackPromise];
+            }
+            if (typeof options.acks !== 'undefined' && options.acks !== ProduceAcks.ALL) {
+                callback(new UserError('Idempotent producer requires acks to be ALL (-1).'));
+                return callback[kCallbackPromise];
+            }
+        }
+        options.acks ??= idempotent ? ProduceAcks.ALL : ProduceAcks.LEADER;
         producerSendsChannel.traceCallback(this.#send, 1, createDiagnosticContext({ client: this, operation: 'send', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    beginTransaction(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        if (!options) {
+            options = {};
+        }
+        const validationError = this[kValidateOptions](options, produceOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError);
+            return callback[kCallbackPromise];
+        }
+        if (!this[kOptions].idempotent) {
+            callback(new UserError('Cannot begin a transaction on a non-idempotent producer.'));
+            return callback[kCallbackPromise];
+        }
+        if (this.#transaction) {
+            callback(new UserError('There is already an active transaction.'));
+            return callback[kCallbackPromise];
+        }
+        this.#transaction = new Transaction(this);
+        producerTransactionsChannel.traceCallback(this.#transaction[kTransactionPrepare], 2, createDiagnosticContext({ client: this, operation: 'begin' }), this.#transaction, this.#producerInfo?.transactionalId, options, callback);
+        return callback[kCallbackPromise];
+    }
+    [kTransactionFindCoordinator](callback) {
+        if (this.#coordinatorId !== undefined) {
+            callback(null);
+            return;
+        }
+        const transactionalId = this.#transaction.id;
+        this[kPerformDeduplicated]('findCoordinator', deduplicateCallback => {
+            this[kPerformWithRetry]('findCoordinator', retryCallback => {
+                this[kGetBootstrapConnection]((error, connection) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    this[kGetApi]('FindCoordinator', (error, api) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        api(connection, FindCoordinatorKeyTypes.TRANSACTION, [transactionalId], retryCallback);
+                    });
+                });
+            }, (error, response) => {
+                if (error) {
+                    deduplicateCallback(error);
+                    return;
+                }
+                const groupInfo = response.coordinators.find(coordinator => coordinator.key === transactionalId);
+                this.#coordinatorId = groupInfo.nodeId;
+                deduplicateCallback(null);
+            }, 0);
+        }, callback);
+    }
+    [kTransactionAddPartitions](transactionId, topicsPartitions, callback) {
+        if (transactionId !== this.#transaction?.[kInstance]) {
+            callback(new UserError('The producer is in use by another transaction.'));
+            return;
+        }
+        const transactions = [];
+        for (const [topic, partitionsSet] of topicsPartitions) {
+            transactions.push({
+                name: topic,
+                partitions: Array.from(partitionsSet)
+            });
+        }
+        this[kPerformDeduplicated]('addPartitionsToTransaction', deduplicateCallback => {
+            this[kPerformWithRetry]('addPartitionsToTransaction', retryCallback => {
+                this.#getCoordinatorConnection((error, connection) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    this[kGetApi]('AddPartitionsToTxn', (error, api) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        api(connection, [
+                            {
+                                transactionalId: this.#transaction.id,
+                                producerId: this.#producerInfo.producerId,
+                                producerEpoch: this.#producerInfo.producerEpoch,
+                                topics: transactions,
+                                verifyOnly: false
+                            }
+                        ], error => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
+                            retryCallback(null);
+                        });
+                    });
+                });
+            }, deduplicateCallback);
+        }, callback);
+    }
+    [kTransactionAddOffsets](transactionId, groupId, callback) {
+        if (transactionId !== this.#transaction?.[kInstance]) {
+            callback(new UserError('The producer is in use by another transaction.'));
+            return;
+        }
+        this[kPerformDeduplicated]('addOffsetsToTransaction', deduplicateCallback => {
+            this[kPerformWithRetry]('addOffsetsToTransaction', retryCallback => {
+                this.#getCoordinatorConnection((error, connection) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    this[kGetApi]('AddOffsetsToTxn', (error, api) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        api(connection, this.#transaction.id, this.#producerInfo.producerId, this.#producerInfo.producerEpoch, groupId, error => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
+                            retryCallback(null);
+                        });
+                    });
+                });
+            }, deduplicateCallback);
+        }, callback);
+    }
+    [kTransactionCommitOffset](transactionId, message, callback) {
+        if (transactionId !== this.#transaction?.[kInstance]) {
+            callback(new UserError('The producer is in use by another transaction.'));
+            return;
+        }
+        this[kPerformDeduplicated]('commit', deduplicateCallback => {
+            this[kPerformWithRetry]('commit', retryCallback => {
+                this[kMetadata]({ topics: [message.topic] }, (error, metadata) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    const { groupId, generationId, memberId, coordinatorId } = message.metadata
+                        .consumer;
+                    this[kGetConnection](metadata.brokers.get(coordinatorId), (error, connection) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        this[kGetApi]('TxnOffsetCommit', (error, api) => {
+                            if (error) {
+                                retryCallback(error);
+                                return;
+                            }
+                            const { topic, partition } = message;
+                            api(connection, this.#transaction.id, groupId, this.#producerInfo.producerId, this.#producerInfo.producerEpoch, generationId, memberId, null, [
+                                {
+                                    name: topic,
+                                    partitions: [
+                                        {
+                                            partitionIndex: message.partition,
+                                            committedOffset: message.offset + 1n,
+                                            committedLeaderEpoch: metadata.topics.get(topic).partitions[partition].leaderEpoch
+                                        }
+                                    ]
+                                }
+                            ], error => {
+                                if (error) {
+                                    retryCallback(error);
+                                    return;
+                                }
+                                retryCallback(null);
+                            });
+                        });
+                    });
+                });
+            }, deduplicateCallback);
+        }, callback);
+    }
+    [kTransactionEnd](transactionId, commit, callback) {
+        if (transactionId !== this.#transaction?.[kInstance]) {
+            callback(new UserError('The producer is in use by another transaction.'));
+            return;
+        }
+        this[kPerformDeduplicated]('endTransaction', deduplicateCallback => {
+            this[kPerformWithRetry]('endTransaction', retryCallback => {
+                this.#getCoordinatorConnection((error, connection) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    this[kGetApi]('EndTxn', (error, api) => {
+                        if (error) {
+                            retryCallback(error);
+                            return;
+                        }
+                        api(connection, this.#transaction.id, this.#producerInfo.producerId, this.#producerInfo.producerEpoch, commit, error => {
+                            if (error) {
+                                this.#handleFencingError(error);
+                                retryCallback(error);
+                                return;
+                            }
+                            this.#transaction = undefined;
+                            retryCallback(null);
+                        });
+                    });
+                });
+            }, deduplicateCallback);
+        }, callback);
+    }
+    [kTransactionCancel](transactionId, callback) {
+        if (transactionId !== this.#transaction?.[kInstance]) {
+            callback(new UserError('The producer is in use by another transaction.'));
+            return;
+        }
+        this.#transaction = undefined;
+        callback(null);
+    }
     #initIdempotentProducer(options, callback) {
+        const transactionalId = this.#transaction?.id;
         this[kPerformDeduplicated]('initProducerId', deduplicateCallback => {
             this[kPerformWithRetry]('initProducerId', retryCallback => {
-                this[kGetBootstrapConnection]((error, connection) => {
+                const connector = transactionalId
+                    ? this.#getCoordinatorConnection.bind(this)
+                    : this[kGetBootstrapConnection].bind(this);
+                connector((error, connection) => {
                     if (error) {
-                        retryCallback(error, undefined);
+                        retryCallback(error);
                         return;
                     }
                     this[kGetApi]('InitProducerId', (error, api) => {
                         if (error) {
-                            retryCallback(error, undefined);
+                            retryCallback(error);
                             return;
                         }
-                        api(connection, null, this[kOptions].timeout, options.producerId ?? this[kOptions].producerId ?? 0n, options.producerEpoch ?? this[kOptions].producerEpoch ?? 0, retryCallback);
+                        api(connection, transactionalId, this[kOptions].timeout, options.producerId ?? this[kOptions].producerId ?? 0n, options.producerEpoch ?? this[kOptions].producerEpoch ?? 0, retryCallback);
                     });
                 });
             }, (error, response) => {
                 if (error) {
-                    deduplicateCallback(error, undefined);
+                    this.#handleFencingError(error);
+                    deduplicateCallback(error);
                     return;
                 }
-                this.#producerInfo = { producerId: response.producerId, producerEpoch: response.producerEpoch };
+                this.#producerInfo = {
+                    producerId: response.producerId,
+                    producerEpoch: response.producerEpoch,
+                    transactionalId
+                };
+                this.#sequences.clear();
                 deduplicateCallback(null, this.#producerInfo);
             }, 0);
         }, callback);
@@ -136,39 +396,34 @@ export class Producer extends Base {
         options.repeatOnStaleMetadata ??= this[kOptions].repeatOnStaleMetadata;
         options.partitioner ??= this[kOptions].partitioner;
         const { idempotent, partitioner } = options;
-        if (idempotent) {
-            options.acks ??= ProduceAcks.ALL;
-        }
-        else {
-            options.acks ??= ProduceAcks.LEADER;
+        let producerId = options.producerId;
+        let producerEpoch = options.producerEpoch;
+        // If the producer was transactional and is not anymore, reset
+        if (!this.#transaction && this.#producerInfo?.transactionalId) {
+            this.#producerInfo = undefined;
         }
-        // We still need to initialize the producerId
-        if (idempotent) {
+        // Initialize the idempotent producer if required
+        if (options.idempotent) {
             if (!this.#producerInfo) {
                 const { messages, ...initOptions } = options;
                 this.initIdempotentProducer(initOptions, error => {
                     if (error) {
-                        callback(error, undefined);
+                        callback(error);
                         return;
                     }
                     this.#send(options, callback);
                 });
                 return;
             }
-            if (typeof options.producerId !== 'undefined' || typeof options.producerEpoch !== 'undefined') {
-                callback(new UserError('Cannot specify producerId or producerEpoch when using idempotent producer.'), undefined);
-                return;
-            }
-            if (options.acks !== ProduceAcks.ALL) {
-                callback(new UserError('Idempotent producer requires acks to be ALL (-1).'), undefined);
-                return;
-            }
+            producerId = this.#producerInfo.producerId;
+            producerEpoch = this.#producerInfo.producerEpoch;
         }
         const produceOptions = {
             compression: options.compression ?? this[kOptions].compression,
-            producerId: idempotent ? this.#producerInfo.producerId : options.producerId,
-            producerEpoch: idempotent ? this.#producerInfo.producerEpoch : options.producerEpoch,
-            sequences: idempotent ? this.#sequences : undefined
+            producerId,
+            producerEpoch,
+            sequences: idempotent ? this.#sequences : undefined,
+            transactionalId: this.#transaction?.id
         };
         // Build messages records out of messages
         const topics = new Set();
@@ -214,13 +469,32 @@ export class Producer extends Base {
                 timestamp: message.timestamp
             });
         }
-        this.#performSend(Array.from(topics), messages, options, produceOptions, callback);
+        const topicsArray = Array.from(topics);
+        if (this.#transaction) {
+            // Get the metadata for topics, we need to normalize the partitions
+            this[kMetadata]({ topics: topicsArray, autocreateTopics: options.autocreateTopics }, (error, metadata) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                this.#transaction?.addPartitions(metadata, messages, error => {
+                    if (error) {
+                        callback(error);
+                        return;
+                    }
+                    this.#performSend(topicsArray, messages, options, produceOptions, callback);
+                });
+            });
+        }
+        else {
+            this.#performSend(topicsArray, messages, options, produceOptions, callback);
+        }
     }
     #performSend(topics, messages, sendOptions, produceOptions, callback) {
         // Get the metadata with the topic/partitions informations
         this[kMetadata]({ topics, autocreateTopics: sendOptions.autocreateTopics }, (error, metadata) => {
             if (error) {
-                callback(error, undefined);
+                callback(error);
                 return;
             }
             const messagesByDestination = new Map();
@@ -247,7 +521,7 @@ export class Producer extends Base {
                 this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
             }, (error, apiResults) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(error);
                     return;
                 }
                 this.#metricsProducedMessages?.inc(messages.length);
@@ -286,7 +560,7 @@ export class Producer extends Base {
         // Get the metadata with the topic/partitions informations
         this[kMetadata]({ topics, autocreateTopics }, (error, metadata) => {
             if (error) {
-                callback(error, undefined);
+                callback(error);
                 return;
             }
             const { topic, partition } = messages[0];
@@ -294,12 +568,12 @@ export class Producer extends Base {
             this[kPerformWithRetry]('produce', retryCallback => {
                 this[kGetConnection](metadata.brokers.get(leader), (error, connection) => {
                     if (error) {
-                        retryCallback(error, undefined);
+                        retryCallback(error);
                         return;
                     }
                     this[kGetApi]('Produce', (error, api) => {
                         if (error) {
-                            retryCallback(error, undefined);
+                            retryCallback(error);
                             return;
                         }
                         api(connection, acks, timeout, messages, produceOptions, retryCallback);
@@ -315,13 +589,37 @@ export class Producer extends Base {
                         this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
                         return;
                     }
-                    callback(error, undefined);
+                    callback(error);
                     return;
                 }
                 callback(error, results);
-            }, 0, [], (error) => {
+            }, 0, [], error => {
                 return repeatOnStaleMetadata && !!error.findBy('hasStaleMetadata', true);
             });
         });
     }
+    #getCoordinatorConnection(callback) {
+        // Get a connection to the coordinator
+        this[kMetadata]({}, (error, metadata) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            this[kPerformWithRetry]('getCoordinatorConnection', retryCallback => {
+                this[kGetConnection](metadata.brokers.get(this.#coordinatorId), (error, connection) => {
+                    if (error) {
+                        retryCallback(error);
+                        return;
+                    }
+                    retryCallback(null, connection);
+                });
+            }, callback);
+        });
+    }
+    #handleFencingError(error) {
+        if (error.findBy?.('producerFenced', true)) {
+            this.#producerInfo = undefined;
+            this.#transaction = undefined;
+        }
+    }
 }

package/dist/clients/producer/transaction.d.ts

@@ -0,0 +1,30 @@
+import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import { type Callback } from '../../apis/definitions.ts';
+import { type Message, type MessageRecord } from '../../protocol/records.ts';
+import { kInstance, kTransactionPrepare } from '../../symbols.ts';
+import { type ClusterMetadata } from '../base/types.ts';
+import { type Consumer } from '../consumer/consumer.ts';
+import { type Producer } from './producer.ts';
+import { type ProduceOptions, type ProduceResult, type SendOptions } from './types.ts';
+export declare class Transaction<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
+    #private;
+    id: string;
+    [kInstance]: number;
+    constructor(producer: Producer<Key, Value, HeaderKey, HeaderValue>);
+    get completed(): boolean;
+    [kTransactionPrepare](transactionalId: string | undefined, options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: Callback<Transaction<Key, Value, HeaderKey, HeaderValue>>): void;
+    send(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProduceResult>): void;
+    send(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProduceResult>;
+    addPartitions(metadata: ClusterMetadata, messages: MessageRecord[], callback: CallbackWithPromise<void>): void;
+    addPartitions(metadata: ClusterMetadata, messages: MessageRecord[]): Promise<void>;
+    addConsumer<MessageKey = Buffer, MessageValue = Buffer, MessageHeaderKey = Buffer, MessageHeaderValue = Buffer>(consumer: Consumer<MessageKey, MessageValue, MessageHeaderKey, MessageHeaderValue>, callback: CallbackWithPromise<void>): void;
+    addConsumer<MessageKey = Buffer, MessageValue = Buffer, MessageHeaderKey = Buffer, MessageHeaderValue = Buffer>(consumer: Consumer<MessageKey, MessageValue, MessageHeaderKey, MessageHeaderValue>): Promise<void>;
+    addOffset<MessageKey = Buffer, MessageValue = Buffer, MessageHeaderKey = Buffer, MessageHeaderValue = Buffer>(message: Message<MessageKey, MessageValue, MessageHeaderKey, MessageHeaderValue>, callback: CallbackWithPromise<void>): void;
+    addOffset<MessageKey = Buffer, MessageValue = Buffer, MessageHeaderKey = Buffer, MessageHeaderValue = Buffer>(message: Message<MessageKey, MessageValue, MessageHeaderKey, MessageHeaderValue>): Promise<void>;
+    commit(callback: CallbackWithPromise<void>): void;
+    commit(): Promise<void>;
+    abort(callback: CallbackWithPromise<void>): void;
+    abort(): Promise<void>;
+    cancel(callback: CallbackWithPromise<void>): void;
+    cancel(): Promise<void>;
+}