@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/src/rdkafka_txnmgr.h

@@ -0,0 +1,171 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2019-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_TXNMGR_H_
+#define _RDKAFKA_TXNMGR_H_
+
+/**
+ * @returns true if transaction state allows enqueuing new messages
+ *          (i.e., produce()), else false.
+ *
+ * @locality application thread
+ * @locks none
+ */
+static RD_INLINE RD_UNUSED rd_bool_t rd_kafka_txn_may_enq_msg(rd_kafka_t *rk) {
+        return !rd_kafka_is_transactional(rk) ||
+               rd_atomic32_get(&rk->rk_eos.txn_may_enq);
+}
+
+
+/**
+ * @returns true if transaction state allows sending messages to broker,
+ *          else false.
+ *
+ * @locality broker thread
+ * @locks none
+ */
+static RD_INLINE RD_UNUSED rd_bool_t rd_kafka_txn_may_send_msg(rd_kafka_t *rk) {
+        rd_bool_t ret;
+
+        rd_kafka_rdlock(rk);
+        ret = (rk->rk_eos.txn_state == RD_KAFKA_TXN_STATE_IN_TRANSACTION ||
+               rk->rk_eos.txn_state == RD_KAFKA_TXN_STATE_BEGIN_COMMIT);
+        rd_kafka_rdunlock(rk);
+
+        return ret;
+}
+
+
+/**
+ * @returns true if transaction and partition state allows sending queued
+ *          messages to broker, else false.
+ *
+ * @locality any
+ * @locks toppar_lock MUST be held
+ */
+static RD_INLINE RD_UNUSED rd_bool_t
+rd_kafka_txn_toppar_may_send_msg(rd_kafka_toppar_t *rktp) {
+        if (likely(rktp->rktp_flags & RD_KAFKA_TOPPAR_F_IN_TXN))
+                return rd_true;
+
+        return rd_false;
+}
+
+
+
+void rd_kafka_txn_schedule_register_partitions(rd_kafka_t *rk, int backoff_ms);
+
+
+/**
+ * @brief Add partition to transaction (unless already added).
+ *
+ * The partition will first be added to the pending list (txn_pending_rktps)
+ * awaiting registration on the coordinator with AddPartitionsToTxnRequest.
+ * On successful registration the partition is flagged as IN_TXN and removed
+ * from the pending list.
+ *
+ * @locality application thread
+ * @locks none
+ */
+static RD_INLINE RD_UNUSED void
+rd_kafka_txn_add_partition(rd_kafka_toppar_t *rktp) {
+        rd_kafka_t *rk;
+        rd_bool_t schedule = rd_false;
+
+        rd_kafka_toppar_lock(rktp);
+
+        /* Already added or registered */
+        if (likely(rktp->rktp_flags &
+                   (RD_KAFKA_TOPPAR_F_PEND_TXN | RD_KAFKA_TOPPAR_F_IN_TXN))) {
+                rd_kafka_toppar_unlock(rktp);
+                return;
+        }
+
+        rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_PEND_TXN;
+
+        rd_kafka_toppar_unlock(rktp);
+
+        rk = rktp->rktp_rkt->rkt_rk;
+
+        mtx_lock(&rk->rk_eos.txn_pending_lock);
+        schedule = TAILQ_EMPTY(&rk->rk_eos.txn_pending_rktps);
+
+        /* List is sorted by topic name since AddPartitionsToTxnRequest()
+         * requires it. */
+        TAILQ_INSERT_SORTED(&rk->rk_eos.txn_pending_rktps, rktp,
+                            rd_kafka_toppar_t *, rktp_txnlink,
+                            rd_kafka_toppar_topic_cmp);
+        rd_kafka_toppar_keep(rktp);
+        mtx_unlock(&rk->rk_eos.txn_pending_lock);
+
+        rd_kafka_dbg(rk, EOS, "ADDPARTS",
+                     "Marked %.*s [%" PRId32
+                     "] as part of transaction: "
+                     "%sscheduling registration",
+                     RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+                     rktp->rktp_partition, schedule ? "" : "not ");
+
+
+        /* Schedule registration of partitions by the rdkafka main thread */
+        if (unlikely(schedule))
+                rd_kafka_txn_schedule_register_partitions(rk, 1 /*immediate*/);
+}
+
+
+
+void rd_kafka_txn_idemp_state_change(rd_kafka_t *rk,
+                                     rd_kafka_idemp_state_t state);
+
+void rd_kafka_txn_set_abortable_error0(rd_kafka_t *rk,
+                                       rd_kafka_resp_err_t err,
+                                       rd_bool_t requires_epoch_bump,
+                                       const char *fmt,
+                                       ...) RD_FORMAT(printf, 4, 5);
+#define rd_kafka_txn_set_abortable_error(rk, err, ...)                         \
+        rd_kafka_txn_set_abortable_error0(rk, err, rd_false, __VA_ARGS__)
+
+#define rd_kafka_txn_set_abortable_error_with_bump(rk, err, ...)               \
+        rd_kafka_txn_set_abortable_error0(rk, err, rd_true, __VA_ARGS__)
+
+void rd_kafka_txn_set_fatal_error(rd_kafka_t *rk,
+                                  rd_dolock_t do_lock,
+                                  rd_kafka_resp_err_t err,
+                                  const char *fmt,
+                                  ...) RD_FORMAT(printf, 4, 5);
+
+rd_bool_t rd_kafka_txn_coord_query(rd_kafka_t *rk, const char *reason);
+
+rd_bool_t rd_kafka_txn_coord_set(rd_kafka_t *rk,
+                                 rd_kafka_broker_t *rkb,
+                                 const char *fmt,
+                                 ...) RD_FORMAT(printf, 3, 4);
+
+void rd_kafka_txns_term(rd_kafka_t *rk);
+void rd_kafka_txns_init(rd_kafka_t *rk);
+
+#endif /* _RDKAFKA_TXNMGR_H_ */

package/deps/librdkafka/src/rdkafka_zstd.c

@@ -0,0 +1,226 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2018-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdkafka_int.h"
+#include "rdkafka_zstd.h"
+
+#if WITH_ZSTD_STATIC
+/* Enable advanced/unstable API for initCStream_srcSize */
+#define ZSTD_STATIC_LINKING_ONLY
+#endif
+
+#include <zstd.h>
+#include <zstd_errors.h>
+
+rd_kafka_resp_err_t rd_kafka_zstd_decompress(rd_kafka_broker_t *rkb,
+                                             char *inbuf,
+                                             size_t inlen,
+                                             void **outbuf,
+                                             size_t *outlenp) {
+        unsigned long long out_bufsize = ZSTD_getFrameContentSize(inbuf, inlen);
+
+        switch (out_bufsize) {
+        case ZSTD_CONTENTSIZE_UNKNOWN:
+                /* Decompressed size cannot be determined, make a guess */
+                out_bufsize = inlen * 2;
+                break;
+        case ZSTD_CONTENTSIZE_ERROR:
+                /* Error calculating frame content size */
+                rd_rkb_dbg(rkb, MSG, "ZSTD",
+                           "Unable to begin ZSTD decompression "
+                           "(out buffer is %llu bytes): %s",
+                           out_bufsize, "Error in determining frame size");
+                return RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+        default:
+                break;
+        }
+
+        /* Increase output buffer until it can fit the entire result,
+         * capped by message.max.bytes */
+        while (out_bufsize <=
+               (unsigned long long)rkb->rkb_rk->rk_conf.recv_max_msg_size) {
+                size_t ret;
+                char *decompressed;
+
+                decompressed = rd_malloc((size_t)out_bufsize);
+                if (!decompressed) {
+                        rd_rkb_dbg(rkb, MSG, "ZSTD",
+                                   "Unable to allocate output buffer "
+                                   "(%llu bytes for %" PRIusz
+                                   " compressed bytes): %s",
+                                   out_bufsize, inlen, rd_strerror(errno));
+                        return RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
+                }
+
+
+                ret = ZSTD_decompress(decompressed, (size_t)out_bufsize, inbuf,
+                                      inlen);
+                if (!ZSTD_isError(ret)) {
+                        *outlenp = ret;
+                        *outbuf  = decompressed;
+                        return RD_KAFKA_RESP_ERR_NO_ERROR;
+                }
+
+                rd_free(decompressed);
+
+                /* Check if the destination size is too small */
+                if (ZSTD_getErrorCode(ret) == ZSTD_error_dstSize_tooSmall) {
+
+                        /* Grow quadratically */
+                        out_bufsize += RD_MAX(out_bufsize * 2, 4000);
+
+                        rd_atomic64_add(&rkb->rkb_c.zbuf_grow, 1);
+
+                } else {
+                        /* Fail on any other error */
+                        rd_rkb_dbg(rkb, MSG, "ZSTD",
+                                   "Unable to begin ZSTD decompression "
+                                   "(out buffer is %llu bytes): %s",
+                                   out_bufsize, ZSTD_getErrorName(ret));
+                        return RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                }
+        }
+
+        rd_rkb_dbg(rkb, MSG, "ZSTD",
+                   "Unable to decompress ZSTD "
+                   "(input buffer %" PRIusz
+                   ", output buffer %llu): "
+                   "output would exceed message.max.bytes (%d)",
+                   inlen, out_bufsize, rkb->rkb_rk->rk_conf.max_msg_size);
+
+        return RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+}
+
+
+rd_kafka_resp_err_t rd_kafka_zstd_compress(rd_kafka_broker_t *rkb,
+                                           int comp_level,
+                                           rd_slice_t *slice,
+                                           void **outbuf,
+                                           size_t *outlenp) {
+        ZSTD_CStream *cctx;
+        size_t r;
+        rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+        size_t len              = rd_slice_remains(slice);
+        ZSTD_outBuffer out;
+        ZSTD_inBuffer in;
+
+        *outbuf  = NULL;
+        out.pos  = 0;
+        out.size = ZSTD_compressBound(len);
+        out.dst  = rd_malloc(out.size);
+        if (!out.dst) {
+                rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                           "Unable to allocate output buffer "
+                           "(%" PRIusz " bytes): %s",
+                           out.size, rd_strerror(errno));
+                return RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
+        }
+
+
+        cctx = ZSTD_createCStream();
+        if (!cctx) {
+                rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                           "Unable to create ZSTD compression context");
+                err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
+                goto done;
+        }
+
+#if defined(WITH_ZSTD_STATIC) &&                                               \
+    ZSTD_VERSION_NUMBER >= (1 * 100 * 100 + 2 * 100 + 1) /* v1.2.1 */
+        r = ZSTD_initCStream_srcSize(cctx, comp_level, len);
+#else
+        /* libzstd not linked statically (or zstd version < 1.2.1):
+         * decompression in consumer may be more costly due to
+         * decompressed size not included in header by librdkafka producer */
+        r = ZSTD_initCStream(cctx, comp_level);
+#endif
+        if (ZSTD_isError(r)) {
+                rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                           "Unable to begin ZSTD compression "
+                           "(out buffer is %" PRIusz " bytes): %s",
+                           out.size, ZSTD_getErrorName(r));
+                err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                goto done;
+        }
+
+        while ((in.size = rd_slice_reader(slice, &in.src))) {
+                in.pos = 0;
+                r      = ZSTD_compressStream(cctx, &out, &in);
+                if (unlikely(ZSTD_isError(r))) {
+                        rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                                   "ZSTD compression failed "
+                                   "(at of %" PRIusz
+                                   " bytes, with "
+                                   "%" PRIusz
+                                   " bytes remaining in out buffer): "
+                                   "%s",
+                                   in.size, out.size - out.pos,
+                                   ZSTD_getErrorName(r));
+                        err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                        goto done;
+                }
+
+                /* No space left in output buffer,
+                 * but input isn't fully consumed */
+                if (in.pos < in.size) {
+                        err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                        goto done;
+                }
+        }
+
+        if (rd_slice_remains(slice) != 0) {
+                rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                           "Failed to finalize ZSTD compression "
+                           "of %" PRIusz " bytes: %s",
+                           len, "Unexpected trailing data");
+                err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                goto done;
+        }
+
+        r = ZSTD_endStream(cctx, &out);
+        if (unlikely(ZSTD_isError(r) || r > 0)) {
+                rd_rkb_dbg(rkb, MSG, "ZSTDCOMPR",
+                           "Failed to finalize ZSTD compression "
+                           "of %" PRIusz " bytes: %s",
+                           len, ZSTD_getErrorName(r));
+                err = RD_KAFKA_RESP_ERR__BAD_COMPRESSION;
+                goto done;
+        }
+
+        *outbuf  = out.dst;
+        *outlenp = out.pos;
+
+done:
+        if (cctx)
+                ZSTD_freeCStream(cctx);
+
+        if (err)
+                rd_free(out.dst);
+
+        return err;
+}

package/deps/librdkafka/src/rdkafka_zstd.h

@@ -0,0 +1,57 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2018-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDZSTD_H_
+#define _RDZSTD_H_
+
+/**
+ * @brief Decompress ZSTD framed data.
+ *
+ * @returns allocated buffer in \p *outbuf, length in \p *outlenp on success.
+ */
+rd_kafka_resp_err_t rd_kafka_zstd_decompress(rd_kafka_broker_t *rkb,
+                                             char *inbuf,
+                                             size_t inlen,
+                                             void **outbuf,
+                                             size_t *outlenp);
+
+/**
+ * Allocate space for \p *outbuf and compress all \p iovlen buffers in \p iov.
+ * @param MessageSetSize indicates (at least) full uncompressed data size,
+ *                       possibly including MessageSet fields that will not
+ *                       be compressed.
+ *
+ * @returns allocated buffer in \p *outbuf, length in \p *outlenp.
+ */
+rd_kafka_resp_err_t rd_kafka_zstd_compress(rd_kafka_broker_t *rkb,
+                                           int comp_level,
+                                           rd_slice_t *slice,
+                                           void **outbuf,
+                                           size_t *outlenp);
+
+#endif /* _RDZSTD_H_ */